Python爬虫面试：requests、BeautifulSoup与Scrapy详解

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

def fetch_data(url, retries=3, backoff_factor=0.5):
    session = requests.Session()
    retry_strategy = Retry(
        total=retries,
        status_forcelist=[429, 500, 502, 503, 504],
        method_whitelist=["GET", "POST"],
        backoff_factor=backoff_factor
    )
    adapter = HTTPAdapter(max_retries=retry_strategy)
    session.mount("http://", adapter)
    session.mount("https://", adapter)

    try:
        response = session.get(url, timeout=10)
        response.raise_for_status()  # Raise for non-2xx status codes
        return response.json()  # Assuming JSON response
    except requests.exceptions.RequestException as e:
        print(f"Request failed: {e}")
        return None

from bs4 import BeautifulSoup
import requests

def parse_html(html):
    soup = BeautifulSoup(html, 'lxml')  # 使用lxml解析器提高效率

    title = soup.find('title').get_text().strip()  # 获取页面标题
    article_links = [a['href'] for a in soup.select('.article-list a')]  # 使用CSS选择器提取文章链接

    return title, article_links

import scrapy

class ExampleSpider(scrapy.Spider):
    name = 'example'
    start_urls = ['https://example.com/articles']

    def parse(self, response):
        for article in response.css('.article'):
            title = article.css('.article-title::text').get()
            author = article.css('.article-author::text').get()
            link = article.css('.article-link::attr(href)').get()

            yield {
                'title': title,
                'author': author,
                'link': response.urljoin(link),  # 正确处理相对链接
            }

        next_page = response.css('.pagination a.next::attr(href)').get()
        if next_page is not None:
            yield response.follow(next_page, self.parse)