Kubernetes上的爬虫排队术——任务调度与弹性扩缩容实战

pip install scrapy pika requests

# task_producer.py
import pika

connection = pika.BlockingConnection(pika.ConnectionParameters(host='rabbitmq-service'))
channel = connection.channel()

channel.queue_declare(queue='news_tasks', durable=True)

# 示例：推送任务
for page in range(1, 11):
    url = f"https://news.yiche.com/qichexinwen/p{page}/"
    channel.basic_publish(exchange='',
                          routing_key='news_tasks',
                          body=url,
                          properties=pika.BasicProperties(delivery_mode=2))  # 持久化消息

print("✅ 已推送任务")
connection.close()

# news_spider.py
import pika
import requests
from bs4 import BeautifulSoup
import random

# 亿牛云代理配置 
PROXY = "http://16YUN:16IP@proxy.16yun.cn:3100"

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64)...",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)...",
    # 更多 User-Agent 可加入
]

def fetch_and_parse(url):
    headers = {
        "User-Agent": random.choice(USER_AGENTS)
    }
    proxies = {
        "http": PROXY,
        "https": PROXY
    }

    resp = requests.get(url, headers=headers, proxies=proxies, timeout=10)
    if resp.status_code != 200:
        print(f"❌ 请求失败：{url}")
        return []

    soup = BeautifulSoup(resp.text, "html.parser")
    articles = []
    for item in soup.select(".module .item"):
        title = item.select_one("h3")
        link = item.select_one("a")
        if title and link:
            articles.append({
                "title": title.get_text(strip=True),
                "url": link["href"]
            })
    return articles

def callback(ch, method, properties, body):
    url = body.decode()
    print(f"📥 正在处理：{url}")
    articles = fetch_and_parse(url)

    # 将数据写入本地归档
    with open("news_data.txt", "a", encoding="utf-8") as f:
        for a in articles:
            f.write(f"{a['title']} - {a['url']}\n")

    print(f"✅ 完成：{len(articles)}条")
    ch.basic_ack(delivery_tag=method.delivery_tag)

# 连接 RabbitMQ
connection = pika.BlockingConnection(pika.ConnectionParameters(host='rabbitmq-service'))
channel = connection.channel()
channel.queue_declare(queue='news_tasks', durable=True)

channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue='news_tasks', on_message_callback=callback)

print("🔄 等待任务中...")
channel.start_consuming()

# Dockerfile
FROM python:3.10-slim

WORKDIR /app
COPY . /app

RUN pip install -r requirements.txt

CMD ["python", "news_spider.py"]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: news-spider
spec:
  replicas: 1
  selector:
    matchLabels:
      app: news-spider
  template:
    metadata:
      labels:
        app: news-spider
    spec:
      containers:
      - name: spider
        image: your-registry/news-spider:latest
        env:
        - name: PROXY
          value: "http://testuser:testpass@proxy.enewcloud.cc:10001"
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: spider-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: news-spider
  minReplicas: 1
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 50

k8s-news-spider/
├── Dockerfile
├── news_spider.py
├── task_producer.py
├── requirements.txt
├── deployment.yaml

requests
pika
beautifulsoup4