Python爬虫实战：利用短效代理IP爬取京东母婴纸尿裤数据，多线程池并行处理方案详解

pip install requests
pip install pandas
pip install lxml
pip install concurrent.futures

import requests
def get_proxy():
    """获取青果网络短效代理IP"""
    proxy_url = "https://api.qg.net/get_proxy"  
    response = requests.get(proxy_url)
    proxy = response.json().get("proxy")
    return {"http": proxy, "https": proxy}

from lxml import etree
def crawl_page(keyword, page, proxies):
    """爬取指定页面信息"""
    url = f"https://search.jd.com/Search?keyword={keyword}&page={page}"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36",
    }
    # 尝试发送请求并获取页面内容
    try:
        response = requests.get(url, headers=headers, proxies=proxies, timeout=10)
        response.raise_for_status()
        html = etree.HTML(response.text)
        # 提取商品数据
        goods_list = html.xpath('//li[contains(@class, "gl-item")]')
        data = []
        for item in goods_list:
            try:
                title = ''.join(item.xpath('.//div[@class="p-name"]//em//text()')).strip()
                price = ''.join(item.xpath('.//div[@class="p-price"]//i/text()')).strip()
                shop = ''.join(item.xpath('.//div[@class="p-shop"]//a/text()')).strip()
                link = 'https:' + ''.join(item.xpath('.//div[@class="p-name"]//a/@href')).strip()
                data.append([title, price, shop, link])
            except Exception as e:
                print(f"Error extracting data: {e}")
        return data
    except Exception as e:
        print(f"Error fetching page {page}: {e}")
        return []

from concurrent.futures import ThreadPoolExecutor, as_completed
import pandas as pd
import time
def main(keyword):
    page_count = 10  # 爬取的总页数
    thread_count = 5  # 并行线程数量
    all_data = []
    # 初始化青果代理池
    proxies = get_proxy()
    with ThreadPoolExecutor(max_workers=thread_count) as executor:
        futures = [executor.submit(crawl_page, keyword, page, proxies) for page in range(1, page_count+1)]
        for future in as_completed(futures):
            try:
                result = future.result()
                all_data.extend(result)
            except Exception as e:
                print(f"Error in thread: {e}")
    # 存储数据到Excel
    df = pd.DataFrame(all_data, columns=['标题', '价格', '店铺', '链接','评论'])
    df.to_excel("京东母婴纸尿裤数据.xlsx", index=False)
    print("数据采集完成！")

if __name__ == "__main__":
    start_time = time.time()
    main(keyword="纸尿裤")
    end_time = time.time()
    print(f"总耗时：{round(end_time - start_time, 2)} 秒")