Python爬虫实战：如何利用代理IP高效爬取Bilibili视频评论数据（附完整代码）

pip install requests
pip install pandas
pip install lxml

https://api.bilibili.com/x/v2/reply/main?oid={video_id}&type=1&next={page}

def get_ip():
    url = "https://share.proxy.qg.net/get?key=yourkey&num=1&area=&isp=0&format=txt&seq=\r\n&distinct=false"
    while 1:
        try:
            r = requests.get(url, timeout=10)
        except:
            continue

        ip = r.text.strip()
        if '请求过于频繁' in ip:
            print('IP请求频繁')
            time.sleep(1)
            continue
        break
    proxies = {
        'https': '%s' % ip
    }

    return proxies

import requests
from lxml import etree
import json
def crawl_comments(video_id, page, proxies):
    """爬取指定视频的评论数据"""
    url = "https://api.bilibili.com/x/v2/reply/main"
    params = {
        "oid": video_id,
        "type": 1,
        "mode": 3,
        "next": page
    }
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"
    }
    try:
        response = requests.get(url, params=params, headers=headers, proxies=proxies, timeout=10)
        response.raise_for_status()
        data = response.json()
        comments = data.get("data", {}).get("replies", [])
        result = []
        for comment in comments:
            content = comment.get("content", {}).get("message", "")
            nickname = comment.get("member", {}).get("uname", "")
            timestamp = comment.get("ctime", "")
            likes = comment.get("like", 0)
            result.append({
                "评论内容": content,
                "用户昵称": nickname,
                "评论发布时间": timestamp,
                "点赞数量": likes
            })
        return result
    except Exception as e:
        print(f"Error fetching comments for page {page}: {e}")
        return []

from concurrent.futures import ThreadPoolExecutor, as_completed
import pandas as pd
import time
def main(video_id):
    max_pages = 20  # 需要爬取的评论页数
    thread_count = 10  # 最大并行线程数量
    all_comments = []
    # 初始化青果网络代理池
    proxies = get_proxy()
    with ThreadPoolExecutor(max_workers=thread_count) as executor:
        futures = [executor.submit(crawl_comments, video_id, page, proxies) for page in range(1, max_pages + 1)]
        for future in as_completed(futures):
            try:
                comments = future.result()
                all_comments.extend(comments)
            except Exception as e:
                print(f"Error in thread：{e}")
    # 存储数据到Excel
    df = pd.DataFrame(all_comments, columns=["用户昵称", "评论内容", "点赞数量", "评论发布时间"])
    df.to_excel(f"Bilibili视频_{video_id}_评论数据.xlsx", index=False)
    print(f"爬取完成！")

if __name__ == "__main__":
    start_time = time.time()
    video_id = "xxxxx"  # 替换为目标视频的ID
    main(video_id)
    end_time = time.time()
    print(f"总耗时：{round(end_time - start_time, 2)} 秒")