汽车市场数据分析与商业洞察项目（Python）——数据爬取

【【汽车市场数据分析与商业洞察项目（Python）】1 –  CSDN App】https://blog.csdn.net/m0_74617873/article/details/155430688?sharetype=blogdetail&shareId=155430688&sharerefer=APP&sharesource=m0_74617873&sharefrom=link

【【汽车市场数据分析与商业洞察项目（Python）】2 –  CSDN App】https://blog.csdn.net/m0_74617873/article/details/155430773?sharetype=blogdetail&shareId=155430773&sharerefer=APP&sharesource=m0_74617873&sharefrom=link

【【汽车市场数据分析与商业洞察项目（Python）】3 –  CSDN App】
https://blog.csdn.net/m0_74617873/article/details/155430928?sharetype=blogdetail&shareId=155430928&sharerefer=APP&sharesource=m0_74617873&sharefrom=link

爬取内容亮点✨

✅ 核心数据全覆盖：汽车 ID、排名、销量、价格区间（最低 / 最高价）、车名、品牌厂商✅ 关键配置信息：车型分类、能源类型（油车 / 电车 / 混动）、上市时间✅ 附加资源：车型图片直链（可直接下载使用）✅ 数据规模：支持爬取 100 页共 1000 款车型数据，覆盖主流在售车型

技术栈拆解🔧

数据请求：requests 库抓取 JSON 接口数据，高效稳定动态页面解析：Selenium+ChromeDriver 处理车型参数详情页反爬优化：禁用自动化标识、配置真实请求头 + Cookie，避免被封 IP数据存储：CSV 格式存储，方便 Excel、Python Pandas 直接分析异常处理：等待页面加载完成再解析，保证数据完整性

import csv
import datetime
import os.path
import time
import requests
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from selenium.webdriver import ChromeOptions
 
 
def init_csv():
    # 初始化操作，判断csv文件是否存在，如不存在就创建一个
    if not os.path.exists('car_data.csv'):
        with open('car_data.csv', 'w', encoding='utf8', newline='') as file:
            writer = csv.writer(file)
            writer.writerow([
                '汽车id',
                '排名',
                '最低价',
                '最高价',
                '车名',
                '销量',
                '图片链接',
                '厂商',
                '车型',
                '能源类型',
                '上市时间'
            ])
 
 
def getJsonHtml(url, params):
    """
    请求获取html内容，json数据格式
    :param url:
    :param params:
    :return:
    """
    cookies = {....}
 
    headers = {
        'accept': '*/*',
        'accept-language': 'zh-CN,zh;q=0.9',
        'priority': 'u=1, i',
        'referer': 'https://www.dongchedi.com/sales',
        'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'same-origin',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
        # 'cookie': 'ttwid=1%7C_NsfbhztMTk5kN5SgrexFZlF3v2vDcJwGC5w9sSf824%7C1764501362%7C05d193e68a71aae576bf19d85d4ef9543b79a731d98c0296287519ed5cf165ac; tt_webid=7578475579789116952; tt_web_version=new; is_dev=false; is_boe=false; Hm_lvt_3e79ab9e4da287b5752d8048743b95e6=1763277655,1763450460,1764501374; Hm_lpvt_3e79ab9e4da287b5752d8048743b95e6=1764501374; HMACCOUNT=927167EF89990893; city_name=%E6%B9%98%E8%A5%BF; s_v_web_id=verify_milmk67y_gMcJLBIo_JeWW_4fm7_AJ8n_MdWFUf4TR6js; _ga=GA1.2.1176853007.1764501375; _gid=GA1.2.1813262875.1764501375; _gat_gtag_UA_138671306_1=1; _ga_YB3EWSDTGF=GS2.1.s1764501374$o1$g1$t1764501380$j54$l0$h0',
    }
 
    response = requests.get(
        url,
        params=params,
        cookies=cookies,
        headers=headers,
    )
    if response.status_code == 200:
        return response.json()
    else:
        return None
 
 
def writerToCsv(row):
    """
    写入csv文件
    :param row:
    :return:
    """
    with open('car_data.csv', 'a', encoding='utf8',
              newline='') as file:
        writer = csv.writer(file)
        writer.writerow(row)
 
 
def parseJson(json, driver):
    """
    解析json数据
    :param json:
    :return:
    """
    cars = json['data']['list']
    for car in cars:
        series_id = car['series_id']  #汽车id
        rank = car['rank']  #排名
        min_price = car['min_price'] #最低价
        max_price = car['max_price'] #最高价
        series_name = car['series_name'] #车名
        count = car['count']  #销量
        image = car['image'] #图片链接
 
        # 访问参数页
        driver.get(f'https://www.dongchedi.com/auto/params-carIds-x-{series_id}')
        WebDriverWait(driver, 20).until(
            EC.presence_of_element_located((By.XPATH, '//*[@id="__next"]/div/div/div/div[2]/div[2]/div[2]/div[3]/div[2]/div'))
        )
        # time.sleep(3)
        #厂商
        manufacturer = driver.find_element(By.XPATH,'//*[@id="__next"]/div/div/div/div[2]/div[2]/div[2]/div[3]/div[2]/div').text
        #车型
        model = driver.find_element(By.XPATH, '//*[@id="__next"]/div/div/div/div[2]/div[2]/div[2]/div[4]/div[2]/div').text
        #能源类型
        EnergyType = driver.find_element(By.XPATH, '//*[@id="__next"]/div/div/div/div[2]/div[2]/div[2]/div[5]/div[2]/div').text
        #上市时间
        create_at = driver.find_element(By.XPATH, '//*[@id="__next"]/div/div/div/div[2]/div[2]/div[2]/div[6]/div[2]/div').text
        time.sleep(2)
        # print(create_at)
        # break
        writerToCsv([
            series_id,
            rank,
            min_price,
            max_price,
            series_name,
            count,
            image,
            manufacturer,
            model,
            EnergyType,
            create_at
        ])
 
 
def start():
    init_csv()
    url = 'https://www.dongchedi.com/motor/pc/car/rank_data'
    service = Service('D:/Software/chromedriver-win64/chromedriver.exe')
    options = ChromeOptions()
    options.add_experimental_option('excludeSwitches', ['enable_automation', 'enable-logging'])
    options.add_experimental_option('useAutomationExtension', False)
    driver = webdriver.Chrome(service=service, options=options)
 
    driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
        "source": "Object.defineProperty(navigator, 'webdriver', {get:()=> false})"
    })
    for page in range(36, 100):
        params = {
            'aid': '1839',
            'app_name': 'auto_web_pc',
            'city_name': '湘西',
            'count': '10',
            'offset': int(page*10),
            'month': '',
            'new_energy_type': '',
            'rank_data_type': '11',
            'brand_id': '',
            'price': '',
            'manufacturer': '',
            'series_type': '',
            'nation': '0',
        }
        print("正在爬取第%d页" % page)
        jsonHtml = getJsonHtml(url, params)
        # print(jsonHtml)
        parseJson(jsonHtml, driver)
        print("爬取第%d页结束" % page)
        time.sleep(2)
        # break
 
 
if __name__ == '__main__':
    start()