Python爬虫案例：Scrapy+XPath解析当当网网页结构

import scrapy

class DangdangItem(scrapy.Item):
    title = scrapy.Field()       # 商品标题
    price = scrapy.Field()       # 商品价格
    author = scrapy.Field()      # 作者
    publisher = scrapy.Field()   # 出版社
    publish_date = scrapy.Field() # 出版日期
    detail = scrapy.Field()      # 商品详情
    comments = scrapy.Field()    # 用户评价数量
    isbn = scrapy.Field()        # ISBN号
    url = scrapy.Field()         # 商品链接

import scrapy
from dangdang.items import DangdangItem
from scrapy.http import Request

class DdSpiderSpider(scrapy.Spider):
    name = 'dd_spider'
    allowed_domains = ['dangdang.com']
    start_urls = ['http://category.dangdang.com/cp01.54.06.00.00.00.html']  # 从图书分类页开始
    
    # 解析分类页，获取商品详情页链接
    def parse(self, response):
        book_links = response.xpath('//a[@name="itemlist-picture"]/@href').extract()
        for link in book_links:
            yield Request(url=link, callback=self.parse_book)
            
        # 处理分页
        next_page = response.xpath('//li[@class="next"]/a/@href').extract_first()
        if next_page:
            yield Request(url=next_page, callback=self.parse)
    
    # 解析商品详情页
    def parse_book(self, response):
        item = DangdangItem()
        
        # 使用XPath提取数据
        item['title'] = response.xpath('//div[@class="name_info"]/h1/@title').extract_first()
        item['price'] = response.xpath('//p[@id="dd-price"]/text()').extract_first().strip()
        
        # 作者信息可能有多人
        authors = response.xpath('//span[@id="author"]/a/text()').extract()
        item['author'] = ' '.join(authors) if authors else None
        
        item['publisher'] = response.xpath('//a[@dd_name="出版社"]/text()').extract_first()
        item['publish_date'] = response.xpath('//div[@class="messbox_info"]/span[2]/text()').extract_first()
        
        # 处理详情信息
        details = []
        detail_nodes = response.xpath('//div[@class="detail_content"]//text()').extract()
        for detail in detail_nodes:
            if detail.strip():
                details.append(detail.strip())
        item['detail'] = '\n'.join(details)
        
        item['comments'] = response.xpath('//a[@dd_name="单品页点击评论"]/text()').extract_first()
        item['isbn'] = response.xpath('//div[@class="messbox_info"]/span[last()]/text()').extract_first()
        item['url'] = response.url
        
        yield item

# 代理配置
PROXY_HOST = "www.16yun.cn"
PROXY_PORT = "5445"
PROXY_USER = "16QMSOML"
PROXY_PASS = "280651"

# 设置下载延迟
DOWNLOAD_DELAY = 2

# 启用AutoThrottle扩展
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 5
AUTOTHROTTLE_MAX_DELAY = 60

# 设置User-Agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'

# 启用Cookies
COOKIES_ENABLED = True

# 配置中间件
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    'scrapy_user_agents.middlewares.RandomUserAgentMiddleware': 400,
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 100,
    'dangdang.middlewares.ProxyMiddleware': 110,  # 自定义代理中间件
}

import pymongo

class MongoDBPipeline(object):
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
    
    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'dangdang')
        )
    
    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]
    
    def close_spider(self, spider):
        self.client.close()
    
    def process_item(self, item, spider):
        collection_name = item.__class__.__name__
        self.db[collection_name].insert_one(dict(item))
        return item

ITEM_PIPELINES = {
    'dangdang.pipelines.MongoDBPipeline': 300,
}

MONGO_URI = 'mongodb://localhost:27017'
MONGO_DATABASE = 'dangdang'