高效爬取某易云音乐：Python JS 逆向与多线程结合实践

import base64
from Crypto.Cipher import AES
from Crypto.PublicKey import RSA
from Crypto.Util.number import bytes_to_long, long_to_bytes
import random

# AES加密函数
def aes_encrypt(text, key):
    iv = '0102030405060708'  # 偏移量固定为16位
    pad = 16 - len(text) % 16
    text = text + pad * chr(pad)
    cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC, iv.encode('utf-8'))
    encrypted = cipher.encrypt(text.encode('utf-8'))
    return base64.b64encode(encrypted).decode('utf-8')

# RSA加密函数
def rsa_encrypt(text, pub_key, modulus):
    text = text[::-1]
    rs = int(bytes_to_long(text.encode('utf-8'))) ** int(pub_key, 16) % int(modulus, 16)
    return format(rs, 'x').zfill(256)

# 生成加密参数
def get_enc_params(data):
    # 固定公钥和模数
    pub_key = '010001'
    modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7'
    # 随机16位密钥
    secret_key = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz0123456789', 16))
    # AES加密数据
    params = aes_encrypt(aes_encrypt(data, '0CoJUm6Qyw8W8jud'), secret_key)
    # RSA加密密钥
    enc_sec_key = rsa_encrypt(secret_key, pub_key, modulus)
    return {'params': params, 'encSecKey': enc_sec_key}

import requests
import json
from queue import Queue
from threading import Thread
import time
import base64
from Crypto.Cipher import AES
from Crypto.PublicKey import RSA
from Crypto.Util.number import bytes_to_long, long_to_bytes
import random

# 代理配置
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"
proxyAuth = f"{proxyUser}:{proxyPass}"
proxyAuthEncoded = base64.b64encode(proxyAuth.encode()).decode()

# 构建代理字典
proxies = {
    "http": f"http://{proxyHost}:{proxyPort}",
    "https": f"https://{proxyHost}:{proxyPort}"
}

# 构建代理头部（如果需要认证）
proxy_headers = {
    "Proxy-Authorization": f"Basic {proxyAuthEncoded}"
}

# AES加密函数
def aes_encrypt(text, key):
    iv = '0102030405060708'  # 偏移量固定为16位
    pad = 16 - len(text) % 16
    text = text + pad * chr(pad)
    cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC, iv.encode('utf-8'))
    encrypted = cipher.encrypt(text.encode('utf-8'))
    return base64.b64encode(encrypted).decode('utf-8')

# RSA加密函数
def rsa_encrypt(text, pub_key, modulus):
    text = text[::-1]
    rs = int(bytes_to_long(text.encode('utf-8'))) ** int(pub_key, 16) % int(modulus, 16)
    return format(rs, 'x').zfill(256)

# 生成加密参数
def get_enc_params(data):
    # 固定公钥和模数
    pub_key = '010001'
    modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7'
    # 随机16位密钥
    secret_key = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz0123456789', 16))
    # AES加密数据
    params = aes_encrypt(aes_encrypt(data, '0CoJUm6Qyw8W8jud'), secret_key)
    # RSA加密密钥
    enc_sec_key = rsa_encrypt(secret_key, pub_key, modulus)
    return {'params': params, 'encSecKey': enc_sec_key}

# 全局配置
session = requests.Session()
session.headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
    'Referer': 'https://music.163.com/',
    "Proxy-Authorization": f"Basic {proxyAuthEncoded}"  # 添加代理认证头部
}
# 设置session的代理
session.proxies = proxies

comment_queue = Queue()  # 评论任务队列
result_list = []  # 存储爬取结果

# 消费者线程：爬取评论
def crawl_comment():
    while not comment_queue.empty():
        try:
            page, resource_id = comment_queue.get()
            # 构造请求数据
            data = json.dumps({
                'rid': f'R_SO_4_{resource_id}',
                'offset': (page - 1) * 20,
                'limit': 20,
                'csrf_token': ''
            })
            enc_params = get_enc_params(data)
            # 发送请求（使用session的代理配置）
            response = session.post(
                'https://music.163.com/weapi/comment/resource/comments/get', 
                data=enc_params,
                timeout=15  # 添加超时设置
            )
            result = response.json()
            if 'comments' in result:
                comments = [{'content': c['content'], 'likeCount': c['likedCount'], 'user': c['user']['nickname']} for c in result['comments']]
                result_list.extend(comments)
                print(f'第{page}页评论爬取完成，共{len(comments)}条')
            comment_queue.task_done()
            time.sleep(0.5)  # 避免请求过快
        except Exception as e:
            print(f'爬取失败：{e}')
            comment_queue.task_done()

# 生产者线程：生成任务
def produce_task(resource_id, total_page):
    for page in range(1, total_page + 1):
        comment_queue.put((page, resource_id))

# 主函数
def main(resource_id, total_page):
    # 生成任务
    produce_task(resource_id, total_page)
    # 创建消费者线程
    threads = []
    for _ in range(5):  # 启动5个线程
        t = Thread(target=crawl_comment)
        t.start()
        threads.append(t)
    # 等待所有任务完成
    comment_queue.join()
    for t in threads:
        t.join()
    print(f'爬取完成，共获取{len(result_list)}条评论')

if __name__ == '__main__':
    # 爬取歌曲ID为186016的评论，共10页
    main(186016, 10)

import csv

def save_to_csv(data, filename):
    with open(filename, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=['content', 'likeCount', 'user'])
        writer.writeheader()
        writer.writerows(data)

# 在main函数中添加保存逻辑
save_to_csv(result_list, 'music_comments.csv')