如何使用分页来抓取网页

import requests
from bs4 import BeautifulSoup

def fetch_page(url):
    response = requests.get(url)
    if response.status_code == 200:
        return response.text
    else:
        raise Exception(f"Failed to fetch {url}")

def parse_page(html):
    soup = BeautifulSoup(html, 'html.parser')
    # 假设我们要抓取的数据在class为'item'的div标签中
    items = soup.find_all('div', class_='item')
    data = []
    for item in items:
        title = item.find('h2').text.strip()
        description = item.find('p').text.strip()
        data.append({'title': title, 'description': description})
    return data

def scrape_with_pagination(base_url, page_param='page', start_page=1, end_page=5):
    all_data = []
    for page in range(start_page, end_page + 1):
        url = f"{base_url}?{page_param}={page}"
        try:
            html = fetch_page(url)
            data = parse_page(html)
            all_data.extend(data)
            print(f"Successfully scraped page {page}")
        except Exception as e:
            print(f"Error scraping page {page}: {e}")
    return all_data

base_url = 'https://example.com/items'
data = scrape_with_pagination(base_url, start_page=1, end_page=10)
for item in data:
    print(item)