浅析爬虫技术更迭：静态请求与浏览器渲染采集能力对比

import requests
from bs4 import BeautifulSoup
import time

# 请求头：模拟浏览器，绕过基础校验
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": "https://book.douban.com/"
}

def static_crawl_douban_notes(url):
    try:
        # 1. 发送静态请求，获取原始HTML
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()  # 抛出请求异常
        
        # 2. 解析静态HTML
        soup = BeautifulSoup(response.text, "lxml")
        
        # 3. 提取数据（仅能获取静态展示的基础信息）
        book_title = soup.find("h1", class_="article-title").get_text(strip=True) if soup.find("h1", class_="article-title") else "未获取到标题"
        author = soup.find("span", class_="author").get_text(strip=True) if soup.find("span", class_="author") else "未获取到作者"
        
        # 关键问题：读书笔记内容为JS渲染，静态HTML中无数据
        note_content = soup.find("div", class_="note-content")
        content = note_content.get_text(strip=True) if note_content else "静态爬虫无法获取动态笔记内容"

        # 输出结果
        print("="*50)
        print("静态请求爬虫结果")
        print(f"书籍标题：{book_title}")
        print(f"作者信息：{author}")
        print(f"笔记内容：{content}")
        print("="*50)
        
    except Exception as e:
        print(f"爬取失败：{str(e)}")

if __name__ == "__main__":
    # 目标豆瓣读书笔记链接
    target_url = "https://book.douban.com/annotation/123456789"
    static_crawl_douban_notes(target_url)

from playwright.sync_api import sync_playwright
import time

def browser_render_crawl(url):
    with sync_playwright() as p:
        # 1. 启动无头Chrome浏览器（后台运行，无界面）
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
        )
        page = context.new_page()
        
        try:
            # 2. 访问页面并等待完整渲染
            page.goto(url, timeout=30000)
            page.wait_for_load_state("networkidle")  # 等待网络空闲，确保JS执行完成
            time.sleep(1)  # 预留渲染缓冲时间
            
            # 3. 提取渲染后的完整数据
            book_title = page.locator("h1.article-title").inner_text().strip()
            author = page.locator("span.author").inner_text().strip()
            note_content = page.locator("div.note-content").inner_text().strip()  # 成功获取动态渲染的笔记
            
            # 输出完整结果
            print("="*50)
            print("浏览器渲染采集结果")
            print(f"书籍标题：{book_title}")
            print(f"作者信息：{author}")
            print(f"笔记内容：{note_content}")
            print("="*50)
            
        except Exception as e:
            print(f"爬取失败：{str(e)}")
        finally:
            # 关闭浏览器，释放资源
            context.close()
            browser.close()

if __name__ == "__main__":
    target_url = "https://book.douban.com/annotation/123456789"
    browser_render_crawl(target_url)