爬虫入门经典(九) | 简单一文教你如何爬取扇贝单词

    etree_obj = etree.HTML(html)
    word_list = etree_obj.xpath('//strong/text()')
    explain_list = etree_obj.xpath('//td[@class="span10"]/text()')
    item_zip = zip(word_list,explain_list)
    for item in item_zip:
        items.append(item)

base_url = "https://www.shanbay.com/wordlist/110521/232414/?page={}"

for i in range(1, 4):
    url = base_url.format(i)
    print(url)

# encoding: utf-8
'''
  @author 李华鑫
  @create 2020-10-08 8:10
  Mycsdn：https://buwenbuhuo.blog.csdn.net/
  @contact: 459804692@qq.com
  @software: Pycharm
  @file: 作业：爬扇贝Python必背词汇.py
  @Version：1.0
  
'''
import csv
import requests
from lxml import etree

"""
https://www.shanbay.com/wordlist/110521/232414/?page=1
https://www.shanbay.com/wordlist/110521/232414/?page=2
https://www.shanbay.com/wordlist/110521/232414/?page=3

//strong                         # en
//td[@class="span10"]            # cn
"""
base_url = "https://www.shanbay.com/wordlist/110521/232414/?page={}"

headers={
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36',
}

items =[]

def parse_url(url):
    """解析url，得到响应内容"""
    response = requests.get(url=url,headers=headers)
    return response.content.decode("utf-8")

def parse_html(html):
    """使用xpath解析html"""
    etree_obj = etree.HTML(html)
    word_list = etree_obj.xpath('//strong/text()')
    explain_list = etree_obj.xpath('//td[@class="span10"]/text()')
    item_zip = zip(word_list,explain_list)
    for item in item_zip:
        items.append(item)

def svae():
    """将数据保存到csv中"""
    with open("./shanbei.csv", "a", encoding="utf-8") as file:
       writer = csv.writer(file)
       for item in items:
            writer.writerow(item)

def start():
    """开始爬虫"""
    for i in range(1, 4):
        url = base_url.format(i)
        html = parse_url(url)
        parse_html(html)
    svae()

if __name__ == '__main__':
    start()