Fandom Wiki 网站爬取文本信息踩坑实录

options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-gpu')
options.add_argument('--remote-debugging-port=9222')

import os
import time
import re
import requests
import opencc
import undetected_chromedriver as uc
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.options import Options

# --- 配置部分 ---
INPUT_FILE = "哆啦A梦漫画道具.txt"
OUTPUT_DIR = "哆啦A梦漫画道具"
HEADLESS_MODE = True  # 默认为 False (可见模式)，测试通过。如需后台运行改为 True

# 伪装请求头
REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

def sanitize_filename(name):
    return re.sub(r'[\\/*?:"<>|]', "", name).strip()

def get_meta_description_via_bs4(html_text):
    """使用 BeautifulSoup 解析 requests 返回的 HTML 文本"""
    soup = BeautifulSoup(html_text, 'html.parser')
    tag = soup.find('meta', attrs={'name': lambda x: x and x.lower() == 'description'})
    if tag and tag.get('content'):
        return tag.get('content')
    return None

def get_section_content(soup, keyword_simp, keyword_trad):
    """提取 h2 章节"""
    for h2 in soup.find_all('h2'):
        headline_span = h2.find('span', class_='mw-headline')
        if headline_span:
            headline_text = headline_span.get_text(strip=True)
            if keyword_simp in headline_text or keyword_trad in headline_text:
                content_parts = []
                for sibling in h2.find_next_siblings():
                    if sibling.name == 'h2':
                        break
                    if sibling.name in ['p', 'div']:
                        text = sibling.get_text(strip=True)
                        if text and len(text) > 2:
                            content_parts.append(text)
                return "\n\n".join(content_parts) if content_parts else None
    return None

def main():
    if not os.path.exists(INPUT_FILE):
        print(f"错误：未找到文件 {INPUT_FILE}")
        return

    # 1. 检查并创建输出文件夹 (如果不存在)
    if not os.path.exists(OUTPUT_DIR):
        os.makedirs(OUTPUT_DIR)
        print(f"已创建输出文件夹: {OUTPUT_DIR}")
    else:
        print(f"输出文件夹已存在: {OUTPUT_DIR}")

    print("正在初始化繁简转换器...")
    try:
        converter = opencc.OpenCC('t2s')
    except Exception as e:
        print(f"OpenCC 初始化失败: {e}")
        return

    # --- 初始化 Undetected ChromeDriver ---
    print("正在初始化反爬虫浏览器...")
    options = Options()
    options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
    options.add_argument('--start-maximized')
    options.add_argument('--disable-blink-features=AutomationControlled')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    
    if HEADLESS_MODE:
        options.add_argument('--headless')

    driver = None
    try:
        driver = uc.Chrome(options=options)
        driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
        print("浏览器启动成功。")

        with open(INPUT_FILE, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        remaining_lines = []

        for i, line in enumerate(lines):
            line = line.strip()
            if not line:
                continue

            if '：' in line:
                parts = line.split('：', 1)
            elif ':' in line:
                parts = line.split(':', 1)
            else:
                remaining_lines.append(line)
                continue

            prop_name = parts[0].strip()
            prop_url = parts[1].strip()

            # --- 新增逻辑：检查文件是否已存在 ---
            safe_name = sanitize_filename(prop_name)
            output_file = os.path.join(OUTPUT_DIR, f"{safe_name}.txt")

            if os.path.exists(output_file):
                print(f"[{i+1}/{len(lines)}] [跳过] 文件已存在: {safe_name}.txt")
                # 文件已存在，跳过处理，并且不加入 remaining_lines (即删除源文件中的该行)
                continue
            # -----------------------------------

            print(f"\n正在处理 [{prop_name}] ({i+1}/{len(lines)})...")

            # --- 策略 A: Requests (快速通道) ---
            meta_via_requests = None
            print("  -> [尝试1] 正在使用 Requests 静态抓取...")
            try:
                response = requests.get(prop_url, headers=REQUEST_HEADERS, timeout=10)
                if response.status_code == 200:
                    meta_via_requests = get_meta_description_via_bs4(response.text)
            except:
                pass

            final_text = ""
            has_content = False
            use_selenium = False

            if meta_via_requests:
                print("  -> [成功] Requests 抓取成功，跳过浏览器。")
                final_text = f"【道具介绍】\n{meta_via_requests}\n\n(注：内容来源为网页 Meta 描述)"
                has_content = True
            else:
                # --- 策略 B: Undetected Chrome (强力通道) ---
                print("  -> [尝试2] Requests 无效，正在使用反爬虫浏览器...")
                use_selenium = True
                
                try:
                    driver.get(prop_url)
                    time.sleep(10)

                    if "challenge" in driver.current_url:
                        print("  -> [警告] 遇到 Cloudflare 挑战，再等待 10 秒...")
                        time.sleep(10)

                    soup = BeautifulSoup(driver.page_source, 'html.parser')

                    intro = get_section_content(soup, "道具介绍", "道具介紹")
                    story = get_section_content(soup, "故事内容", "故事內容")

                    if intro or story:
                        if intro:
                            final_text += f"【道具介绍】\n{intro}\n\n"
                        else:
                            final_text += "【道具介绍】\n(未找到该章节)\n\n"
                        
                        if story:
                            final_text += f"【故事内容】\n{story}"
                        else:
                            final_text += "【故事内容】\n(未找到该章节)"
                        has_content = True
                    else:
                        desc = get_meta_description_via_bs4(driver.page_source)
                        if desc:
                            final_text = f"【道具介绍】\n{desc}\n\n(注：内容来源为网页 Meta 描述)"
                            has_content = True
                        else:
                            print("  -> [失败] 浏览器页面也无法获取内容。")

                except Exception as e:
                    print(f"  -> [错误] 浏览器访问出错: {e}")
                    remaining_lines.append(line)
                    continue

            # 保存文件
            if has_content:
                final_text_simplified = converter.convert(final_text)
                
                with open(output_file, 'w', encoding='utf-8') as f_out:
                    f_out.write(final_text_simplified)
                
                print(f"成功！(已转简体) 保存至: {output_file}")
                # 成功则不加入 remaining_lines
            else:
                remaining_lines.append(line)

            if use_selenium:
                time.sleep(2)

        print("\n任务完成，正在更新列表文件...")
        with open(INPUT_FILE, 'w', encoding='utf-8') as f:
            f.write('\n'.join(remaining_lines))
            if remaining_lines:
                f.write('\n')
        print(f"列表已更新，剩余 {len(remaining_lines)} 条未处理。")

    except Exception as e:
        print(f"程序发生严重错误: {e}")
    finally:
        if driver:
            driver.quit()
            print("浏览器已关闭。")

if __name__ == '__main__':
    main()