实战：爬取某联招聘职位需求并生成词云——从零开始的完整指南

pip install requests beautifulsoup4 pandas wordcloud jieba matplotlib

import requests
from bs4 import BeautifulSoup

def get_job_list(keyword, page):
    url = f"https://sou.***.com/?kw={keyword}&page={page}"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }
    try:
        response = requests.get(url, headers=headers, timeout=10)
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, 'html.parser')
            jobs = []
            for li in soup.select('.job-list ul li'):
                title = li.select_one('.job-title').text.strip()
                company = li.select_one('.company-name').text.strip()
                salary = li.select_one('.salary').text.strip() if li.select_one('.salary') else "面议"
                requirement = li.select_one('.job-requirements').text.strip() if li.select_one('.job-requirements') else ""
                jobs.append({
                    'title': title,
                    'company': company,
                    'salary': salary,
                    'requirement': requirement
                })
            return jobs
        else:
            print(f"请求失败，状态码：{response.status_code}")
            return []
    except Exception as e:
        print(f"发生错误：{e}")
        return []

def crawl_all_pages(keyword, max_pages=5):
    all_jobs = []
    for page in range(1, max_pages+1):
        print(f"正在爬取第{page}页...")
        jobs = get_job_list(keyword, page)
        if not jobs:
            break
        all_jobs.extend(jobs)
        time.sleep(2)  # 礼貌性延迟
    return all_jobs

# 执行爬取
jobs_data = crawl_all_pages("Python开发", 3)

import pandas as pd

df = pd.DataFrame(jobs_data)
# 去除空值
df = df.dropna(subset=['requirement'])
# 保存原始数据（可选）
df.to_csv('zhaopin_raw.csv', index=False, encoding='utf_8_sig')

all_requirements = ' '.join(df['clean_require'].tolist())

import jieba

# 添加自定义词典（可选）
jieba.load_userdict("tech_terms.txt")  # 可自行准备技术术语词典

# 分词并过滤停用词
stopwords = set()
with open('stopwords.txt', 'r', encoding='utf-8') as f:  # 停用词表
    for line in f:
        stopwords.add(line.strip())

words = [word for word in jieba.cut(all_requirements) 
         if len(word) > 1 and word not in stopwords and not word.isspace()]
word_str = ' '.join(words)

from wordcloud import WordCloud
import matplotlib.pyplot as plt

# 设置词云参数
wc = WordCloud(
    font_path='simhei.ttf',  # 中文字体文件路径
    background_color='white',
    width=800,
    height=600,
    max_words=100,
    max_font_size=100
)

# 生成词云
wc.generate(word_str)

# 显示词云
plt.figure(figsize=(10, 8))
plt.imshow(wc, interpolation='bilinear')
plt.axis('off')
plt.savefig('zhaopin_wordcloud.png', dpi=300, bbox_inches='tight')
plt.show()

import requests
from bs4 import BeautifulSoup
import pandas as pd
import jieba
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import time
import random

# 1. 数据采集
def get_job_list(keyword, page):
    url = f"https://sou.***.com/?kw={keyword}&page={page}"
    headers = {
        "User-Agent": "Mozilla/5.0...",
        "Referer": "https://www.***.com/"
    }
    try:
        response = requests.get(url, headers=headers, timeout=10)
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, 'html.parser')
            jobs = []
            for li in soup.select('.job-list ul li'):
                # 解析逻辑同上...
            return jobs
        return []
    except:
        return []

# 2. 数据处理
def process_data(jobs_data):
    df = pd.DataFrame(jobs_data)
    df = df.dropna(subset=['requirement'])
    
    def extract_req(text):
        # 提取逻辑同上...
        pass
    
    df['clean_require'] = df['requirement'].apply(extract_req)
    all_text = ' '.join(df['clean_require'].tolist())
    return all_text

# 3. 生成词云
def generate_wordcloud(text):
    stopwords = set(['可以', '能够', '具有', '等'])  # 示例停用词
    words = [word for word in jieba.cut(text) 
             if len(word) > 1 and word not in stopwords]
    
    wc = WordCloud(
        font_path='simhei.ttf',
        width=800,
        height=600
    ).generate(' '.join(words))
    
    plt.figure(figsize=(10, 8))
    plt.imshow(wc)
    plt.axis('off')
    plt.savefig('output.png')
    plt.show()

# 主流程
if __name__ == "__main__":
    keyword = "Python开发"
    jobs = []
    for page in range(1, 4):
        jobs.extend(get_job_list(keyword, page))
        time.sleep(2)
    
    text = process_data(jobs)
    generate_wordcloud(text)