用Python采集CBC新闻：如何借助青果网络海外代理IP构建稳定采集方案

import requests

def get_proxy():
"""获取青果网络代理IP"""
proxy_url = "https://api.qg.net/get_proxy" 

# 测试代理是否生效
try:
    response = requests.get("https://www.cbc.ca/news", proxies=proxies, timeout=10)
    print("代理已生效，站点内容如下：")
    print(response.text[:500])  # 输出部分HTML文件内容
except Exception as e:
    print(f"代理设置失败，错误信息：{e}")

# 新增代理中间件
class QgProxyMiddleware:
    def process_request(self, request, spider):
        request.meta['proxy'] = 'https://api.qg.net/get_proxy'# 获取青果网络代理IP API
        request.headers['Proxy-Authorization'] = basic_auth_header('用户名', '密码')

# 优化后的爬虫核心
def parse(self, response):
    # 添加随机延迟（5-10秒）
    time.sleep(random.uniform(5, 10))
    
    
    # 动态时间筛选（采集近1个月数据）
    current_year = datetime.now().month
    for item in items:
        if current_month - int(item['month']) <= 1:
            yield scrapy.Request(
                url=response.urljoin(item['href']),  # 智能URL拼接
                callback=self.parse_detail,
                meta={
                    "item": item,
                    "proxy": "http://青果网络加拿大代理IP"  # 指定地区代理
                }
            )
    
    # 添加分页逻辑
    next_page = response.xpath('//a[@data-testid="pagination-next"]/@href').get()
    if next_page:
        yield response.follow(next_page, self.parse)

from bs4 import BeautifulSoup
import pandas as pd
import re

def clean_cbc_data(html):
    soup = BeautifulSoup(html, 'lxml')
    article_data = {
        "title": soup.select_one('h1[data-testid="story-title"]').text.strip(),
        "date": pd.to_datetime(soup.find('time')['datetime']),  # 标准化时间
        "section": soup.find('a', {'data-testid': 'section'}).text,
        "content": '\n'.join([p.text for p in soup.select('div[data-testid="story-body"] > p')]),
        "keywords": [tag.text for tag in soup.select('li[data-testid="tag-item"]')]
    }
    # 移除广告文本等噪音
    article_data['content'] = re.sub(r'^\s*(Advertisement|Related Stories)\s*$', '', 
                                    article_data['content'], flags=re.MULTILINE)
    return article_data

# 构建结构化数据集
articles = [clean_cbc_data(html) for html in raw_htmls]
df = pd.DataFrame(articles)

import matplotlib.pyplot as plt
df['month'] = df['date'].dt.to_period('M')
topic_counts = df.groupby(['month', 'section']).size().unstack()
topic_counts.plot(kind='line', title='CBC各主题新闻趋势')

from wordcloud import WordCloud
text = ' '.join(df['content'].sample(1000))  # 避免内存过载
wordcloud = WordCloud(width=1200, height=600).generate(text)
plt.imshow(wordcloud)