Python爬虫与数据可视化：构建完整的数据采集与分析流程

import requests
from bs4 import BeautifulSoup
from requests.auth import HTTPBasicAuth

# 代理设置
proxyHost = "cfgfrgtr"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

# 构建代理字典
proxies = {
    "http": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
    "https": f"https://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
}

def fetch_article(url):
    # 使用代理发送请求
    response = requests.get(url, proxies=proxies)
    response.encoding = 'utf-8'
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 提取文章标题和内容
    title = soup.find('h1').text
    content = soup.find('div', class_='article-content').text
    
    return title, content

# 示例URL
url = 'https://www.duanwenxue.com/example-article-url' 
title, content = fetch_article(url)
print(f'Title: {title}\nContent: {content}')

import csv

def save_to_csv(data, filename):
    with open(filename, 'w', newline='', encoding='utf-8') as file:
        writer = csv.writer(file)
        writer.writerow(['Title', 'Content'])
        for item in data:
            writer.writerow(item)

# 假设data是一个包含标题和内容的列表
data = [(title, content)]
save_to_csv(data, 'articles.csv')

import matplotlib.pyplot as plt

def plot_bar_chart(data):
    titles = [item[0] for item in data]
    contents = [len(item[1]) for item in data]  # 文章内容长度作为数量指标
    
    plt.figure(figsize=(10, 6))
    plt.bar(titles, contents, color='blue')
    plt.xlabel('Article Titles')
    plt.ylabel('Content Length')
    plt.title('Article Content Length Distribution')
    plt.show()

plot_bar_chart(data)

import jieba
from wordcloud import WordCloud

def generate_word_cloud(text):
    # 分词
    words = jieba.cut(text)
    words = ' '.join(words)
    
    # 生成词云
    wordcloud = WordCloud(font_path='simhei.ttf', background_color='white').generate(words)
    
    # 显示词云图
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.show()

# 使用文章内容生成词云
generate_word_cloud(content)