农业爬虫实战：惠农网农产品价格行情抓取全解析

[定时任务] → [代理调度] → [请求发送] → [页面解析] → [数据清洗] → [存储分析]

POST https://www.cnhnb.com/api/price/search
Form Data: 
{
  "categoryId": 1001,
  "provinceId": 37,
  "pageSize": 20,
  "pageNum": 1
}

import requests
from bs4 import BeautifulSoup
import random
import time
from fake_useragent import UserAgent

# 代理池配置（示例使用免费代理，实际项目建议购买企业级代理）
PROXY_POOL = [
    {"http": "http://123.123.123.123:8080"},
    # 更多代理IP...
]

def get_random_proxy():
    return random.choice(PROXY_POOL)

def get_page_data(url):
    headers = {"User-Agent": UserAgent().random}
    proxy = get_random_proxy()
    
    try:
        response = requests.get(url, headers=headers, proxies=proxy, timeout=10)
        if response.status_code == 200:
            return response.text
        else:
            print(f"请求失败，状态码：{response.status_code}")
            return None
    except Exception as e:
        print(f"请求异常：{e}")
        return None

def parse_price_data(html):
    soup = BeautifulSoup(html, 'html.parser')
    price_items = []
    
    # 示例解析逻辑（需根据实际页面结构调整）
    for item in soup.select('.price-item'):
        name = item.select_one('.product-name').text.strip()
        price = item.select_one('.current-price').text.strip()
        spec = item.select_one('.spec-info').text.strip() if item.select_one('.spec-info') else ""
        origin = item.select_one('.origin-place').text.strip() if item.select_one('.origin-place') else ""
        
        price_items.append({
            "name": name,
            "price": float(price[1:]),  # 去除人民币符号
            "spec": spec,
            "origin": origin,
            "update_time": time.strftime("%Y-%m-%d %H:%M:%S")
        })
    
    return price_items

def main():
    base_url = "https://www.cnhnb.com/price/apple/"
    html = get_page_data(base_url)
    
    if html:
        data = parse_price_data(html)
        # 此处可添加数据存储逻辑（如写入CSV/数据库）
        print(f"成功抓取{len(data)}条价格数据")
    else:
        print("数据抓取失败")

if __name__ == "__main__":
    main()

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

def get_dynamic_data(url):
    chrome_options = Options()
    chrome_options.add_argument("--headless")  # 无头模式
    chrome_options.add_argument(f"user-agent={UserAgent().random}")
    
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(url)
    
    # 模拟点击"加载更多"按钮
    try:
        load_more = driver.find_element_by_css_selector(".load-more-btn")
        load_more.click()
        time.sleep(3)  # 等待数据加载
    except:
        pass
    
    html = driver.page_source
    driver.quit()
    return html

import pandas as pd
import matplotlib.pyplot as plt

# 加载数据
df = pd.read_csv("apple_prices.csv")

# 价格分布分析
plt.figure(figsize=(10,6))
df['price'].hist(bins=30)
plt.title("苹果价格分布图")
plt.xlabel("价格（元/斤）")
plt.ylabel("频数")
plt.savefig("price_distribution.png")

# 产地对比分析
top_origins = df['origin'].value_counts().head(5)
top_origins.plot(kind='barh')
plt.title("主要产地供应量对比")
plt.savefig("origin_comparison.png")