动态渲染爬虫：Selenium抓取京东关键字搜索结果

from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager

def search_jd(keyword, pages=3):
    base_url = "https://search.jd.com/Search?keyword="
    all_products = []
    
    for page in range(1, pages+1):
        url = f"{base_url}{keyword}&page={page}"
        driver.get(url)
        
        # 模拟滚动加载所有商品
        for i in range(5):
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(2)  # 等待加载
            
        # 等待商品列表加载完成
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, ".gl-item"))
        )
        
        # 获取页面源码并用BeautifulSoup解析
        page_source = driver.page_source
        products = parse_page(page_source)
        all_products.extend(products)
        
        time.sleep(3)  # 避免请求过于频繁
    
    return all_products

from bs4 import BeautifulSoup

def parse_page(html):
    soup = BeautifulSoup(html, 'html.parser')
    items = soup.select('.gl-item')
    products = []
    
    for item in items:
        try:
            name = item.select('.p-name em')[0].get_text(strip=True)
            price = item.select('.p-price i')[0].get_text(strip=True)
            shop = item.select('.p-shop a')[0].get_text(strip=True) if item.select('.p-shop a') else "自营"
            comment = item.select('.p-commit a')[0].get_text(strip=True) if item.select('.p-commit a') else "0"
            link = "https:" + item.select('.p-img a')[0]['href']
            
            products.append({
                'name': name,
                'price': price,
                'shop': shop,
                'comment': comment,
                'link': link
            })
        except Exception as e:
            print(f"解析商品出错: {e}")
            continue
            
    return products

import pandas as pd

def save_to_csv(products, filename='jd_products.csv'):
    df = pd.DataFrame(products)
    df.to_csv(filename, index=False, encoding='utf_8_sig')
    print(f"数据已保存到 {filename}")

if __name__ == "__main__":
    try:
        keyword = input("请输入要搜索的商品关键字: ")
        products = search_jd(keyword, pages=3)
        save_to_csv(products)
    finally:
        driver.quit()  # 确保浏览器关闭

import sqlite3

def save_to_db(products, db_name='jd_products.db'):
    conn = sqlite3.connect(db_name)
    c = conn.cursor()
    
    c.execute('''CREATE TABLE IF NOT EXISTS products
                 (name TEXT, price TEXT, shop TEXT, comment TEXT, link TEXT)''')
    
    for p in products:
        c.execute("INSERT INTO products VALUES (?,?,?,?,?)",
                 (p['name'], p['price'], p['shop'], p['comment'], p['link']))
    
    conn.commit()
    conn.close()