面向电商的多语言页面抓取策略

import requests
from bs4 import BeautifulSoup

url = "https://www.amazon.com/dp/B08N5WRWNW"
headers = {"User-Agent": "my-bot/1.0"}
resp = requests.get(url, headers=headers, timeout=10)

# 問題点：直接用 resp.text，不管编码是什么
soup = BeautifulSoup(resp.text, "html.parser")
title = soup.select_one("#productTitle").get_text(strip=True)
print("Title:", title)

pip install requests bs4 chardet

# robust_amazon_scraper.py
import time
import random
import requests
import chardet
from bs4 import BeautifulSoup

# === 代理配置（以亿牛云为例） ===
proxy_host = "proxy.16yun.cn"
proxy_port = 3100
proxy_user = "16YUN"
proxy_pass = "16IP"

proxy_url = f"http://{proxy_user}:{proxy_pass}@{proxy_host}:{proxy_port}"
proxies = {"http": proxy_url, "https": proxy_url}

# === 请求头 ===
HEADERS = {
    "User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                   "AppleWebKit/537.36 (KHTML, like Gecko) "
                   "Chrome/120.0.0.0 Safari/537.36"),
    "Accept-Language": "en-US,en;q=0.9",
}

# === 小工具函数 ===
def check_robots():
    """简单检查 robots.txt"""
    try:
        r = requests.get("https://www.amazon.com/robots.txt", timeout=8)
        print("robots.txt status:", r.status_code)
        print("\n".join(r.text.splitlines()[:10]))
    except Exception as e:
        print("robots.txt 读取失败:", e)

def detect_encoding(resp):
    """更聪明的编码检测"""
    if resp.encoding and resp.encoding.lower() != "iso-8859-1":
        return resp.encoding
    detected = chardet.detect(resp.content)
    return detected.get("encoding") or "utf-8"

def safe_get(url, max_retries=3):
    """带重试与随机等待的请求函数"""
    for i in range(max_retries):
        try:
            r = requests.get(url, headers=HEADERS, proxies=proxies, timeout=15)
            if r.status_code == 200 and len(r.content) > 200:
                return r
        except Exception as e:
            print("请求失败:", e)
        time.sleep(random.uniform(1, 3))
    raise RuntimeError("多次请求失败")

def parse_amazon_page(html):
    """针对不同语言/布局容错"""
    soup = BeautifulSoup(html, "html.parser")
    title = None
    for sel in ["#productTitle", "h1 span#productTitle", ".a-size-large.a-color-base.a-text-normal"]:
        node = soup.select_one(sel)
        if node:
            title = node.get_text(strip=True)
            break

    price = None
    for sel in ["#priceblock_ourprice", ".a-price .a-offscreen", "#corePriceDisplay_desktop_feature_div .a-offscreen"]:
        node = soup.select_one(sel)
        if node:
            price = node.get_text(strip=True)
            break

    return title, price

def scrape_product(url):
    check_robots()
    resp = safe_get(url)
    encoding = detect_encoding(resp)
    html = resp.content.decode(encoding, errors="replace")
    title, price = parse_amazon_page(html)
    return {"title": title, "price": price, "encoding": encoding}

if __name__ == "__main__":
    result = scrape_product("https://www.amazon.com/dp/B08N5WRWNW")
    print(result)