爬取微博热榜并将其存储为csv文件

import requests
from bs4 import BeautifulSoup
import pandas as pd

cookie = '你自己的cookie'

# 获取网页响应，对网页响应进行编码
response = requests.get(url, headers=headers)
response.encoding = response.apparent_encoding
html = response.text

# 将网页文本使用Beautifulsoup解析
soup = BeautifulSoup(html, 'html.parser')

# allnews存放热搜前50的新闻和热度,形式为{'新闻':'热度'}字典
all_news = {}

# 定位网页元素，观察到热搜新闻位于'td'元素下，class为'td-02'
for news in soup.find_all('td', class_='td-02')[1:]:
    text = news.text.split('\n')[1].strip()
    if news.text.split('\n')[2].strip() == '':
        continue
    elif news.text.split('\n')[2].strip()[0].isdigit():
        hot = news.text.split('\n')[2].strip()
    else:
        hot = news.text.split('\n')[2].strip()[2:]
    all_news[text] = hot

# 将字典转为DataFrame，并将DataFrame保存为csv文件
df = pd.DataFrame.from_dict(all_news, orient='index', columns=['热度'])
df.index.name = '新闻'
df.to_csv('weibo_hot.csv', encoding='utf-8-sig')