盘点一个Python网络爬虫的实战问题

# encoding: UTF-8
# create time: 2024/05/30/0030 16:26:03

# ################### 导入模块 #################### 分隔的开始
import time
from urllib.parse import urljoin

import openpyxl
import requests
import parsel
from fake_useragent import UserAgent
# ################### 导入模块 #################### 分隔的结束


# TODO 列表下载
def get_page(pages: int):
    """
    发送请求，获取页面数据
    :param pages: 翻页参数
    :return: 请求到的数据
    """
    url = f"https://www.xiachufang.com/category/40071/?page={pages}"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0'
    }
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            parse_page(response)
        else:
            return "请求失败，状态响应码:\t" + str(response.status_code)
    except requests.ReadTimeout as e:
        print("请求超时", e.args[0])
        time.sleep(2)


def parse_page(response):
    """
    解析页面数据
    :param response: 响应的内容
    :return: 返回一个列表，并交给存储的函数
    """
    items = []
    lst = []
    base_url = 'https://www.xiachufang.com'
    html = parsel.Selector(response.text)
    foods_list = html.css('div.info p.name')
    count = 0
    for li in range(0, len(foods_list)):
        count += 1
        lst.append([
            count,    # 计数器
            foods_list[li].css('a::text').extract()[0][16:-14].strip(),   # 获取标题
            urljoin(base_url, foods_list[li].css('a::attr(href)').extract()[0])   # 获取连接并对连接做处理
        ])
    print(lst)
    items.append(lst)
    save1(items)
    save_data(items, current_page)


def save1(items):
    with open('data.txt', 'w', encoding='utf-8') as f:
        for item in items:
            for item in item:
                f.write(str(item) + '\n')


def save_data(items):
    """
    存储数据
    :param lst: 解析数据得到的列表
    :return: 无返回
    """
    # wb = openpyxl.Workbook()
    # sheet = wb.active
    # for item in lst:
    #     sheet.append(item)
    # wb.save('下厨房早餐.xlsx')
    wb = openpyxl.Workbook()
    # 选择默认的工作表
    ws = wb.active

    # 给工作表添加标题行
    ws.append(['ID', '菜名', '链接'])

    # 遍历列表数据并添加到工作表中
    for item in items:
        for item in item:
            ws.append(item)

    # 保存工作簿为Excel文件
    wb.save(f'下厨房早餐{current_page}.xlsx')


def main() -> None:
    total_pages = 3
    for i in range(total_pages):
        current_page = i + 1
        get_page(current_page)
        print("当前页:\t" + str(current_page))
        time.sleep(2)


if __name__ == '__main__':
    main()