【Lighthouse教程】网页内容抓取入门

sudo apt install python3-venv

python3 -m venv scrapy_examples
cd scrapy_examples
source bin/activate

pip install scrapy
scrapy startproject movie
cd movie

(scrapy_examples) ➜  movie tree
.
├── movie
│   ├── __init__.py
│   ├── items.py
│   ├── middlewares.py
│   ├── pipelines.py
│   ├── __pycache__
│   ├── settings.py
│   └── spiders
│       ├── __init__.py
│       └── __pycache__
└── scrapy.cfg

import scrapy

class MovieSpider1(scrapy.Spider):
    name = "movie_1"

    def start_requests(self):
        urls = [
            'https://v.qq.com/channel/movie?listpage=1&channel=movie&itype=100012', # 科幻
            'https://v.qq.com/channel/movie?listpage=1&channel=movie&itype=100005', # 爱情
            'https://v.qq.com/channel/movie?listpage=1&channel=movie&itype=100004', # 喜剧
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        movie_list = response.css('body > div.mod_row_box > div > div.mod_figure.mod_figure_v_default.mod_figure_list_box > div')
        for m in movie_list:
            yield {
                '电影标题': m.css('a::attr(title)').get(),
                '播放地址': m.css('a::attr(href)').get(),
                '海报链接': 'https:{}'.format(m.css('a > img.figure_pic::attr(src)').get()),
                '播放时长': m.css('a > div.figure_caption::text').get(),
                '影评分数': m.css('a > div.figure_score::text').get(),
            }

scrapy crawl movie_1 -o movies.csv

(scrapy_examples) ➜  movie head movies.csv 
电影标题,播放地址,海报链接,播放时长,影评分数
僵尸世界大战,https://v.qq.com/x/cover/r5trbf8xs5uwok1.html,https://puui.qpic.cn/vcover_vt_pic/0/r5trbf8xs5uwok11590989398026/220,02:03:02,7.7 
复仇者联盟4：终局之战,https://v.qq.com/x/cover/v2098lbuihuqs11.html,https://puui.qpic.cn/vcover_vt_pic/0/v2098lbuihuqs111587100715029/220,02:54:05,9.2 
美人鱼,https://v.qq.com/x/cover/xg95sxi4q7zc4uo.html,https://puui.qpic.cn/vcover_vt_pic/0/xg95sxi4q7zc4uot1460107848.jpg/220,01:33:45,8.3 
环太平洋：雷霆再起,https://v.qq.com/x/cover/f7pqur8uhmzltps.html,https://puui.qpic.cn/vcover_vt_pic/0/f7pqur8uhmzltps1559809738/220,01:50:59,7.6 
钢铁飞龙之奥特曼崛起,https://v.qq.com/x/cover/380idj4s3fxn1mz.html,https://puui.qpic.cn/vcover_vt_pic/0/380idj4s3fxn1mz1543994759/220,01:30:43,6.5 
地心历险记2：神秘岛,https://v.qq.com/x/cover/ny2pykcofj3tljs.html,https://puui.qpic.cn/vcover_vt_pic/0/ny2pykcofj3tljs1567405217/220,01:34:10,8.1 
速度与激情：特别行动,https://v.qq.com/x/cover/mu01sbah4rauf44.html,https://puui.qpic.cn/vcover_vt_pic/0/mu01sbah4rauf441573116092/220,02:13:52,8 
透明人,https://v.qq.com/x/cover/l3u3m6tzeddnnhk.html,https://puui.qpic.cn/vcover_vt_pic/0/l3u3m6tzeddnnhkt1444906881.jpg/220,01:46:34,8.2 
星际大逃亡,https://v.qq.com/x/cover/mzc00200a20krmb.html,https://puui.qpic.cn/vcover_vt_pic/0/mzc00200a20krmb1590481722077/220,01:28:25,7.7 

body > div.mod_row_box > div > div.mod_figure.mod_figure_v_default.mod_figure_list_box > div:nth-child(1)

import scrapy
from urllib.parse import parse_qs

class MoviesSpider2(scrapy.Spider):
    name = "movie_2"
    # download_delay = 1.0

    base_url_template = 'https://v.qq.com/x/bu/pagesheet/list?append=2&channel=movie&itype=100012&listpage=1&pagesize={}&offset={}'
    page_size = 30

    def start_requests(self):
        url = self.base_url_template.format(self.page_size, 0)
        yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        movie_list = response.css('body > div > div > div.mod_figure.mod_figure_v_default.mod_figure_list_box > div')
        for m in movie_list:
            yield {
                '电影标题': m.css('a::attr(title)').get(),
                '播放地址': m.css('a::attr(href)').get(),
                '海报链接': 'https:{}'.format(m.css('a > img.figure_pic::attr(src)').get()),
                '播放时长': m.css('a > div.figure_caption::text').get(),
                '影评分数': m.css('a > div.figure_score::text').get(),
            }
        # 如果还有，更新url参数继续请求
        if len(movie_list) > 0:
            current_offset = int(parse_qs(response.url)['offset'][0])
            next_url = self.base_url_template.format(self.page_size, current_offset + self.page_size)
            yield scrapy.Request(next_url, callback=self.parse)