处理动态Token：Python爬虫应对AJAX授权请求的策略

import requests
from lxml import html
import time
import random
from urllib.parse import urljoin  # 用于处理相对URL

# ========== 代理配置 ==========
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

proxyMeta = f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
proxies = {
    "http": proxyMeta,
    "https": proxyMeta,
}

# ========== 爬虫配置 ==========
BASE_DOMAIN = "example.com"
BASE_URL = f"https://{BASE_DOMAIN}/users"
AJAX_URL = f"https://{BASE_DOMAIN}/api/get_users"

# 更加真实的浏览器 User-Agent 列表
USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0'
]

def create_session():
    """创建并配置会话"""
    session = requests.Session()
    session.proxies.update(proxies)
    session.headers.update({
        'User-Agent': random.choice(USER_AGENTS),
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
    })
    return session

def extract_csrf_token(html_content):
    """从HTML内容中提取CSRF Token，支持多种可能的定位方式"""
    tree = html.fromstring(html_content)
    
    # 尝试多种常见的CSRF Token存放位置
    selectors = [
        '//meta[@name="csrf-token"]/@content',
        '//meta[@name="_token"]/@content',
        '//input[@name="csrf_token"]/@value',
        '//input[@name="_token"]/@value',
        '//input[@name="csrf-token"]/@value',
    ]
    
    for selector in selectors:
        tokens = tree.xpath(selector)
        if tokens:
            return tokens[0]
    
    raise ValueError("CSRF Token not found in the HTML")

def make_request_with_retry(session, url, method='get', max_retries=3, **kwargs):
    """带重试机制的请求函数"""
    for attempt in range(max_retries):
        try:
            if method.lower() == 'get':
                response = session.get(url, timeout=15, **kwargs)
            elif method.lower() == 'post':
                response = session.post(url, timeout=15, **kwargs)
            else:
                raise ValueError(f"Unsupported HTTP method: {method}")
            
            response.raise_for_status()
            return response
            
        except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
            if attempt == max_retries - 1:
                raise e
            print(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
            time.sleep(2 ** attempt)  # 指数退避策略

def scrape_ajax_with_token_enhanced():
    """增强版的爬虫函数，包含更好的错误处理和重试机制"""
    session = create_session()
    
    try:
        # 1. 首次请求获取初始页面和CSRF Token
        print("🔍 正在通过代理请求初始页面...")
        response = make_request_with_retry(session, BASE_URL)
        
        # 2. 提取CSRF Token
        csrf_token = extract_csrf_token(response.text)
        print(f"✅ CSRF Token 获取成功: {csrf_token[:20]}...")  # 只显示部分Token

        # 3. 设置AJAX请求的公共头部
        ajax_headers = {
            'X-Requested-With': 'XMLHttpRequest',
            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'Origin': f'https://{BASE_DOMAIN}',
            'Referer': BASE_URL,
        }

        total_pages = 5
        successful_pages = 0
        
        for page in range(2, total_pages + 1):
            print(f"\n📄 正在请求第 {page} 页数据...")
            
            payload = {
                'page': page,
                'size': 20,  # 通常分页API会有size参数
                'csrf_token': csrf_token
            }

            try:
                # 4. 发送AJAX请求
                ajax_response = make_request_with_retry(
                    session, AJAX_URL, method='post', 
                    data=payload, headers=ajax_headers
                )
                
                # 5. 解析响应数据
                data = ajax_response.json()
                
                # 更健壮的数据提取
                users = data.get('data', {}).get('list', [])
                if not users:
                    users = data.get('list', [])
                
                if users:
                    print(f"✅ 第 {page} 页获取成功，共 {len(users)} 条数据")
                    successful_pages += 1
                    
                    # 数据处理逻辑
                    process_users(users, page)
                else:
                    print(f"⚠️  第 {page} 页无数据，可能已到末页")
                    break
                    
                # 6. 随机延迟，模拟人类行为
                time.sleep(random.uniform(1, 3))
                
            except ValueError as e:
                print(f"❌ 第 {page} 页JSON解析失败: {e}")
                break
            except Exception as e:
                print(f"❌ 第 {page} 页请求失败: {e}")
                # 可以选择继续尝试下一页或跳出循环
                continue

        print(f"\n🎉 爬取完成！成功获取 {successful_pages} 页数据")

    except requests.exceptions.ProxyError as e:
        print(f"❌ 代理连接失败: {e}")
        print("请检查代理配置或联系代理服务商")
    except requests.exceptions.SSLError as e:
        print(f"❌ SSL证书错误: {e}")
    except Exception as e:
        print(f"❌ 爬虫执行失败: {e}")
    finally:
        session.close()
        print("会话已关闭")

def process_users(users, page_num):
    """处理获取到的用户数据"""
    # 这里实现您的具体业务逻辑
    for i, user in enumerate(users, 1):
        # 示例：打印用户信息
        user_id = user.get('id', 'N/A')
        user_name = user.get('name', 'N/A')
        # print(f"  用户 {i}: ID={user_id}, Name={user_name}")
    
    # 实际应用中，您可能会：
    # 1. 保存到数据库
    # 2. 写入CSV或JSON文件
    # 3. 进行数据清洗和转换
    pass

if __name__ == '__main__':
    start_time = time.time()
    scrape_ajax_with_token_enhanced()
    end_time = time.time()
    print(f"⏱️  总耗时: {end_time - start_time:.2f} 秒")