基于Scrapy-Redis的分布式景点数据爬取与热力图生成

# 启用 Scrapy-Redis 调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"

# Redis 连接配置
REDIS_URL = 'redis://localhost:6379/0'

# 数据存储到 MongoDB
ITEM_PIPELINES = {
    'scrapy_redis.pipelines.RedisPipeline': 300,
    'scenic_spider.pipelines.MongoPipeline': 400,
}

MONGO_URI = 'mongodb://localhost:27017'
MONGO_DB = 'scenic_data'

import scrapy
from scenic_spider.items import ScenicItem
from urllib.parse import urlencode

# 代理配置
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

class MeituanSpider(scrapy.Spider):
    name = 'meituan_spider'
    redis_key = 'meituan:start_urls'  # Redis 初始任务队列

    # 自定义请求头（可选）
    custom_headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }

    def start_requests(self):
        # 从Redis获取初始URL时添加代理
        for url in self.get_start_urls():
            yield scrapy.Request(
                url=url,
                callback=self.parse,
                headers=self.custom_headers,
                meta={
                    'proxy': f'http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}'
                }
            )

    def parse(self, response):
        for scenic in response.css('.scenic-item'):
            item = ScenicItem()
            item['name'] = scenic.css('.title::text').get()
            item['rating'] = scenic.css('.score::text').get()
            item['reviews'] = scenic.css('.review-num::text').get()
            item['location'] = scenic.css('.address::text').get()
            yield item

        # 翻页逻辑（同样添加代理）
        next_page = response.css('.next-page::attr(href)').get()
        if next_page:
            yield scrapy.Request(
                url=next_page,
                callback=self.parse,
                headers=self.custom_headers,
                meta={
                    'proxy': f'http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}'
                }
            )

import pymongo

class MongoPipeline:
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DB')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db['scenic_spots'].insert_one(dict(item))
        return item

import pandas as pd
from pymongo import MongoClient
import requests

client = MongoClient('mongodb://localhost:27017')
db = client['scenic_data']
collection = db['scenic_spots']

# 获取数据并转为 DataFrame
data = list(collection.find())
df = pd.DataFrame(data)

# 调用高德地图API获取经纬度
def get_geocode(address):
    url = f"https://restapi.amap.com/v3/geocode/geo?key=YOUR_AMAP_KEY&address={address}"
    res = requests.get(url).json()
    if res['status'] == '1' and res['geocodes']:
        lng, lat = res['geocodes'][0]['location'].split(',')
        return float(lng), float(lat)
    return None, None

df['lng'], df['lat'] = zip(*df['location'].apply(get_geocode))
df.to_csv('scenic_data.csv', index=False)

import folium
from folium.plugins import HeatMap

df = pd.read_csv('scenic_data.csv')
df = df.dropna(subset=['lat', 'lng'])

# 创建地图
m = folium.Map(location=[df['lat'].mean(), df['lng'].mean()], zoom_start=12)

# 生成热力图
heat_data = [[row['lat'], row['lng'], row['reviews']] for _, row in df.iterrows()]
HeatMap(heat_data, radius=15).add_to(m)

# 保存为HTML
m.save('heatmap.html')