三种常见网站结构的解析方式对比——信息结构与处理路径图谱

网站结构解析策略图谱
|
├── 静态型页面
│   └── requests + BeautifulSoup 方式
│
├── 动态型页面
│   └── Playwright 自动化方式（支持模拟交互）
│
└── 接口驱动型页面
    └── requests + JSON 格式数据提取

import random
from curl_cffi import requests
from bs4 import BeautifulSoup

# 亿牛云代理参数（参考示例）
proxy_host = "proxy.16yun.cn"
proxy_port = "3111"
proxy_user = "16YUN"
proxy_pass = "16IP"

# 构建代理配置
proxies = {
    "http": f"http://{proxy_user}:{proxy_pass}@{proxy_host}:{proxy_port}",
    "https": f"http://{proxy_user}:{proxy_pass}@{proxy_host}:{proxy_port}",
}

# 浏览器身份标识（部分网站要求真实UA）
user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64)...",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)...",
]
headers = {
    "User-Agent": random.choice(user_agents)
}

url = "https://example.com/static-page"

# 发送请求
resp = requests.get(url, headers=headers, proxies=proxies)
soup = BeautifulSoup(resp.text, "html.parser")

# 提取标题和段落信息
title = soup.find("h1").text
paragraphs = [p.text for p in soup.find_all("p")]

print(f"页面标题: {title}")
print("主要内容段落:", paragraphs)

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True, proxy={
        "server": f"http://{proxy_host}:{proxy_port}",
        "username": proxy_user,
        "password": proxy_pass
    })
    page = browser.new_page()
    page.set_extra_http_headers(headers)

    page.goto("https://example.com/dynamic-content")
    page.wait_for_timeout(3000)  # 等待前端逻辑渲染完成

    content = page.locator("div.comment").all_inner_texts()
    print("页面中提取的评论：", content)

    browser.close()

graphql_url = "https://www.expedia.com/graphql"

query_body = {
    "query": "query{ hotelOffers(city:\"Tokyo\"){ name price rating } }"
}

# 发起接口请求
resp = requests.post(
    graphql_url,
    headers={**headers, "Content-Type": "application/json"},
    json=query_body,
    proxies=proxies
)

data = resp.json()
offers = data["data"]["hotelOffers"]
for offer in offers:
    print(f"酒店名称: {offer['name']}, 价格: {offer['price']}, 评分: {offer['rating']}")