用Selenium模拟登录淘宝并采集商品信息：从基础到实战

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time

# 代理配置（以站大爷为例）
PROXY_HOST = "***.***.com"
PROXY_PORT = "24000"
PROXY_USER = "your_username"
PROXY_PASS = "your_password"

chrome_options = Options()
# 设置代理（方式一：直接配置）
chrome_options.add_argument(f'--proxy-server=http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}')

# 启动浏览器（添加无头模式可隐藏窗口）
driver = webdriver.Chrome(
    executable_path='./chromedriver',  # 替换为你的驱动路径
    options=chrome_options
)

def login_taobao(driver):
    driver.get("https://login.***.com")
    time.sleep(3)  # 等待页面加载
    
    # 示例：通过账号密码登录（实际需处理验证码）
    username = driver.find_element("id", "fm-login-id")
    password = driver.find_element("id", "fm-login-password")
    username.send_keys("your_username")
    password.send_keys("your_password")
    
    # 点击登录按钮（需根据实际页面调整选择器）
    driver.find_element("xpath", "//button[@class='fm-btn']").click()
    time.sleep(5)  # 等待登录完成

from bs4 import BeautifulSoup
import pandas as pd

def search_and_scrape(driver, keyword="手机"):
    driver.get(f"https://s.***.com/search?q={keyword}")
    time.sleep(5)  # 等待搜索结果加载
    
    # 获取页面源码并解析
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    
    # 提取商品信息（示例选择器，需根据实际页面调整）
    products = []
    items = soup.find_all("div", class_="m-itemlist")
    
    for item in items:
        try:
            title = item.find("div", class_="row-2").find("a").get("title")
            price = item.find("strong").text
            sales = item.find("div", class_="row-3").find_all("span")[-1].text
            products.append({
                "title": title,
                "price": price,
                "sales": sales
            })
        except AttributeError:
            continue
    
    # 导出为CSV
    df = pd.DataFrame(products)
    df.to_csv("taobao_products.csv", index=False, encoding="utf-8-sig")
    print("数据已保存")

import random
from zdaye_api import get_proxy  # 假设的代理API

def rotate_ip(driver):
    # 获取新代理（实际需调用代理服务商API）
    new_proxy = get_proxy()  
    
    # 重新配置代理（需重启WebDriver）
    driver.quit()
    chrome_options.add_argument(f'--proxy-server=http://{new_proxy}')
    driver = webdriver.Chrome(options=chrome_options)
    return driver

def set_user_agent(driver):
    user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36..."
    chrome_options.add_argument(f'user-agent={user_agent}')

def save_cookies(driver, path="cookies.pkl"):
    import pickle
    pickle.dump(driver.get_cookies(), open(path, "wb"))

def load_cookies(driver, path="cookies.pkl"):
    import pickle
    cookies = pickle.load(open(path, "rb"))
    for cookie in cookies:
        driver.add_cookie(cookie)

if __name__ == "__main__":
    # 初始化
    driver = init_driver()
    set_user_agent(driver)
    
    try:
        # 登录
        login_taobao(driver)
        
        # 搜索商品
        search_and_scrape(driver)
        
        # 轮换IP示例（实际按需调用）
        # driver = rotate_ip(driver)
        
    finally:
        driver.quit()

from seleniumwire import webdriver  # 需安装selenium-wire

options = {
    'proxy': {
        'http': 'http://user:pass@proxy:port',
        'ssl': 'http://user:pass@proxy:port'
    },
    'browser_fingerprint': 'custom_hash_value'  # 模拟不同设备
}
driver = webdriver.Chrome(seleniumwire_options=options)