Python爬取小说网站全本下载：从入门到反爬实战

# 安装必要库
pip install requests beautifulsoup4 fake_useragent

import requests
from bs4 import BeautifulSoup

def get_chapter(url):
    headers = {'User-Agent': 'Mozilla/5.0'}
    try:
        response = requests.get(url, headers=headers, timeout=10)
        soup = BeautifulSoup(response.text, 'html.parser')
        # 笔趣阁内容通常在id="content"的div中
        content = soup.find('div', id='content').text.strip()
        return content
    except Exception as e:
        print(f"获取章节失败: {e}")
        return None

# 测试获取第一章
print(get_chapter("https://www.example.com/book/1.html"))

def get_chapter_list(book_url):
    headers = {'User-Agent': 'Mozilla/5.0'}
    response = requests.get(book_url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 笔趣阁目录通常在class="listmain"的ul中
    chapters = []
    for li in soup.select('.listmain ul li a'):
        chapters.append({
            'title': li.text.strip(),
            'url': li['href']
        })
    return chapters

# 获取某本书所有章节链接
chapters = get_chapter_list("https://www.example.com/book/")
for i, chap in enumerate(chapters[:5], 1):  # 打印前5章
    print(f"{i}. {chap['title']}: {chap['url']}")

import concurrent.futures

def download_book(chapters, output_file):
    all_content = []
    
    def fetch_chapter(chapter):
        content = get_chapter(chapter['url'])
        if content:
            return f"\n\n{chapter['title']}\n\n{content}"
        return ""
    
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = executor.map(fetch_chapter, chapters)
        all_content = [content for content in results if content]
    
    with open(output_file, 'w', encoding='utf-8') as f:
        f.write("\n".join(all_content))
    print(f"小说已保存至 {output_file}")

# 使用示例
chapters = get_chapter_list("https://www.example.com/book/")
download_book(chapters, "小说全本.txt")

from ebooklib import epub

def create_epub(chapters, book_name):
    book = epub.EpubBook()
    book.set_title(book_name)
    book.set_language('zh')
    
    # 添加章节
    for chapter in chapters:
        content = get_chapter(chapter['url'])
        if content:
            c1 = epub.EpubHtml(
                title=chapter['title'],
                file_name=f"{chapter['title'].replace('/', '_')}.xhtml",
                lang='zh',
                content=f"<h1>{chapter['title']}</h1><p>{content}</p>"
            )
            book.add_item(c1)
            book.toc.append((chapter['title'], [c1], False))
    
    # 添加元数据
    book.add_author('匿名')
    
    # 生成epub
    epub.write_epub(f"{book_name}.epub", book, {})
    print(f"EPUB文件已生成: {book_name}.epub")

# 使用示例
chapters = get_chapter_list("https://www.example.com/book/")
create_epub(chapters, "我的小说")

# 随机User-Agent
from fake_useragent import UserAgent

def get_random_ua():
    ua = UserAgent()
    return ua.random

headers = {'User-Agent': get_random_ua()}

import random

class ProxyPool:
    def __init__(self):
        # 购买站大爷代理IP
        self.proxies = [
            {'http': 'http://1.1.1.1:8080'},
            {'http': 'http://2.2.2.2:8080'},
            # 更多代理...
        ]
    
    def get_proxy(self):
        return random.choice(self.proxies)

# 使用代理
proxy = ProxyPool().get_proxy()
response = requests.get(url, headers=headers, proxies=proxy, timeout=10)

from PIL import Image
import pytesseract
import io

def solve_captcha(img_bytes):
    img = Image.open(io.BytesIO(img_bytes))
    # 转换为灰度图提高识别率
    img = img.convert('L')
    # 二值化处理
    threshold = 140
    table = []
    for i in range(256):
        if i < threshold:
            table.append(0)
        else:
            table.append(1)
    img = img.point(table, '1')
    return pytesseract.image_to_string(img)

# 使用示例（需要先获取验证码图片）
# captcha_text = solve_captcha(captcha_img_bytes)

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

def selenium_get(url):
    chrome_options = Options()
    chrome_options.add_argument("--headless")  # 无头模式
    chrome_options.add_argument(f"user-agent={get_random_ua()}")
    
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(url)
    
    # 等待页面加载（可根据实际情况调整）
    driver.implicitly_wait(10)
    
    # 获取内容（示例获取body文本）
    content = driver.find_element_by_tag_name('body').text
    driver.quit()
    return content

# main.py
from parser import NovelParser
from downloader import Downloader
from converter import EpubConverter

def main():
    book_url = input("请输入小说目录页URL: ")
    parser = NovelParser()
    chapters = parser.get_chapters(book_url)
    
    downloader = Downloader()
    downloader.download_all(chapters, "小说.txt")
    
    converter = EpubConverter()
    converter.to_epub(chapters, "我的小说")

if __name__ == "__main__":
    main()