12306旅游产品数据抓取：Python+API逆向分析

import requests

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    "Referer": "https://kyfw.12306.cn/otn/leftTicket/init",
    "Cookie": "JSESSIONID=...; RAIL_EXPIRATION=...; RAIL_DEVICEID=...",  # 需替换为有效Cookie
}

url = "https://kyfw.12306.cn/otn/travel/product/list?page=1"
response = requests.get(url, headers=headers)

if response.status_code == 200:
    data = response.json()
    print(data)
else:
    print("请求失败:", response.status_code)

from selenium import webdriver
import time

def get_12306_cookie():
    driver = webdriver.Chrome()
    driver.get("https://kyfw.12306.cn/otn/resources/login.html")
    
    # 手动扫码登录（或自动填充账号密码）
    time.sleep(20)  # 留时间扫码
    
    # 获取Cookie
    cookies = driver.get_cookies()
    cookie_str = "; ".join([f"{c['name']}={c['value']}" for c in cookies])
    
    driver.quit()
    return cookie_str

cookie = get_12306_cookie()
print("获取的Cookie:", cookie)

import requests
import pandas as pd
from fake_useragent import UserAgent
import time
import random

# 代理配置
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

# 构造代理字典
proxies = {
    "http": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
    "https": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
}

def get_random_headers():
    """生成随机请求头"""
    ua = UserAgent()
    return {
        "User-Agent": ua.random,
        "Referer": "https://kyfw.12306.cn/otn/leftTicket/init",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Connection": "keep-alive",
        "X-Requested-With": "XMLHttpRequest",
    }

def fetch_12306_travel_products(page=1, max_pages=5):
    """爬取12306旅游产品数据（带代理IP）"""
    all_products = []
    base_url = "https://kyfw.12306.cn/otn/travel/product/list"
    
    for page in range(1, max_pages + 1):
        try:
            # 每次请求使用随机请求头
            headers = get_random_headers()
            headers["Cookie"] = get_12306_cookie()  # 替换为你的Cookie
            
            params = {"page": page}
            
            # 添加随机延迟（1-3秒）
            time.sleep(random.uniform(1, 3))
            
            # 发送带代理的请求
            response = requests.get(
                base_url,
                headers=headers,
                params=params,
                proxies=proxies,
                timeout=10
            )
            
            if response.status_code == 200:
                data = response.json()
                products = data.get("data", {}).get("list", [])
                all_products.extend(products)
                print(f"✅ 成功爬取第 {page} 页，共 {len(products)} 条数据")
            else:
                print(f"❌ 第 {page} 页请求失败，状态码: {response.status_code}")
                
        except Exception as e:
            print(f"⚠️ 第 {page} 页请求异常: {str(e)}")
            continue
    
    return all_products

# 示例：获取Cookie（需替换为实际实现）
def get_12306_cookie():
    """获取12306登录Cookie（示例函数，需替换为实际实现）"""
    return "JSESSIONID=...; RAIL_EXPIRATION=...; RAIL_DEVICEID=..."

# 主程序
if __name__ == "__main__":
    print("🚀 开始爬取12306旅游产品数据...")
    
    # 爬取数据（3页示例）
    travel_data = fetch_12306_travel_products(max_pages=3)
    
    # 数据处理
    if travel_data:
        df = pd.DataFrame(travel_data)
        
        # 提取关键字段
        df = df[[
            "productName",  # 产品名称
            "fromStation",  # 出发地
            "toStation",    # 目的地
            "price",        # 价格
            "days",         # 行程天数
            "productUrl"    # 详情链接
        ]]
        
        # 保存数据
        df.to_csv("12306_travel_products.csv", index=False, encoding='utf_8_sig')
        print(f"🎉 数据已保存至 12306_travel_products.csv，共 {len(df)} 条记录")
    else:
        print("❌ 未获取到有效数据")

import pandas as pd

# 示例数据整理
df = pd.DataFrame(travel_data)
df = df[[
    "productName",  # 产品名称
    "fromStation",  # 出发地
    "toStation",    # 目的地
    "price",        # 价格
    "days",         # 行程天数
    "productUrl"    # 详情链接
]]
print(df.head())