从登陆到爬取：Python反反爬获取某宝成千上万条公开商业数据

pip install selenium

# 从 selenium 里面导入 webdriver
from selenium import webdriver

# 指定 chrom 驱动（下载到本地的浏览器驱动器，地址定位到它）
driver = webdriver.Chrome('E:/software/chromedriver_win32/chromedriver.exe')

# get 方法打开指定网址
driver.get('http://www.baidu.com')

from selenium.webdriver import ActionChains # 导入动作链
from selenium import webdriver
import time
import csv
import re

# 传入浏览器驱动本地地址
driver = webdriver.Chrome('E:/software/chromedriver_win32/chromedriver.exe')
# 传入目标页面地址
driver.get('https://www.taobao.com/')

driver.maximize_window() # 最大化浏览器

keyword = input('请输入您要搜索的商品名字：')
driver.find_element_by_id('q').send_keys(keyword)  # 根据“检查”的id值精确定位淘宝网搜索框并传入关键字
driver.find_element_by_class_name('btn-search').click() # 根据class标签'btn-search'定位到搜索按钮并点击

driver.find_element_by_xpath('//*[@id="fm-login-id"]').send_keys('账号')
driver.find_element_by_xpath('//*[@id="fm-login-password"]').send_keys('密码')

login = driver.find_element_by_xpath('//*[@id="nc_1_n1z"]') # 通过xpath找到滑块
action = ActionChains(driver)  # 创造出一个动作链
action.click_and_hold(on_element=login) # 点击不松开
action.move_by_offset(xoffset=300-42, yoffset=0) # 通过坐标轴滑动
action.pause(0.5).release().perform() # 设置链式调用时间（滑动滑块时间）,并松开鼠标   perform()执行动作链

divs = driver.find_elements_by_xpath('//div[@class="items"]/div[@class="item J_MouserOnverReq  "]')
for div in divs:
    info = div.find_element_by_xpath('.//div[@class="row row-2 title"]/a').text
    price = div.find_element_by_xpath('.//strong').text
    deal = div.find_element_by_xpath('.//div[@class="deal-cnt"]').text
    shop = div.find_element_by_xpath('.//div[@class="shop"]/a').text

with open('data.csv', mode='a', newline="") as csvfile:
    csvWriter = csv.writer(csvfile, delimiter=',')
    csvWriter.writerow([info, price, deal, shop])

page = driver.find_element_by_xpath('//*[@id="mainsrp-pager"]/div/div/div/div[1]').text # 获取总页数标签
page_list = re.findall('(\d+)', page)  # 正则表达式获取多个精确数字数据[返回的是列表]
page_num = page_list[0]   # 字符串类型数据

driver.get('https://s.taobao.com/search?q={}&s={}'.format(keyword, page_num*44))
page_num += 1

from selenium.webdriver import ActionChains # 导入动作链
from selenium import webdriver
import time
import csv
import re

# 寻找关键字并登录淘宝
def search_product(key):
    driver.get('https://www.taobao.com/')
    driver.find_element_by_id('q').send_keys(key)  # 根据“检查”的id值精确定位淘宝网搜索框并传入关键字
    driver.find_element_by_class_name('btn-search').click() # 根据class标签'btn-search'定位到搜索按钮并点击

    driver.implicitly_wait(10) # 隐式等待（单位是秒） 等到页面渲染完之后就不再等待
    driver.maximize_window() # 最大化浏览器

    # 解决登陆（登录防爬：例如有滑块）
    driver.find_element_by_xpath('//*[@id="fm-login-id"]').send_keys('这里填写账户名称/手机号码')
    time.sleep(1)
    driver.find_element_by_xpath('//*[@id="fm-login-password"]').send_keys('这里填写账户密码')
    time.sleep(2)

    # 解决滑块
    login = driver.find_element_by_xpath('//*[@id="nc_1_n1z"]') # 通过xpath找到滑块
    action = ActionChains(driver)  # 创造出一个动作链
    action.click_and_hold(on_element=login) # 点击不松开
    action.move_by_offset(xoffset=300-42, yoffset=0) # 通过坐标轴滑动
    action.pause(0.5).release().perform() # 设置链式调用时间（滑动滑块时间）,并松开鼠标   perform()执行动作链
    driver.find_element_by_xpath('//*[@id="login-form"]/div[4]/button').click() # 点击登录并重定向到前面的关键字
    driver.implicitly_wait(10) # 隐式等待

    page = driver.find_element_by_xpath('//*[@id="mainsrp-pager"]/div/div/div/div[1]').text # 获取总页数标签
    page_list = re.findall('(\d+)', page)  # 正则表达式获取多个精确数字数据[返回的是列表]
    page_num = page_list[0]   # 字符串类型数据

    return int(page_num)

# 爬取数据并保存
def get_data():
    divs = driver.find_elements_by_xpath('//div[@class="items"]/div[@class="item J_MouserOnverReq  "]')
    for div in divs:
        info = div.find_element_by_xpath('.//div[@class="row row-2 title"]/a').text
        price = div.find_element_by_xpath('.//strong').text
        deal = div.find_element_by_xpath('.//div[@class="deal-cnt"]').text
        shop = div.find_element_by_xpath('.//div[@class="shop"]/a').text
        print(info, price, deal, shop, sep='|')

        # 保存
        with open('data.csv', mode='a', newline="") as csvfile:
            csvWriter = csv.writer(csvfile, delimiter=',')
            csvWriter.writerow([info, price, deal, shop])

def main():
    print('正在爬取第1页数据...')
    page = search_product(keyword)
    get_data()

    # 第2页之后数据获取
    page_num = 1   # page_num * 44
    while page_num != page:
        print('*' * 100)
        print('正在爬取第{}页数据'.format(page_num+1))
        print('*' * 100)
        driver.get('https://s.taobao.com/search?q={}&s={}'.format(keyword, page_num*44))
        driver.implicitly_wait(10)  # 隐式等待
        get_data()
        page_num += 1

    driver.quit()

if __name__ == '__main__':
    driver = webdriver.Chrome('E:/software/chromedriver_win32/chromedriver.exe')
    # keyword = '电脑'
    keyword = input('请输入您要搜索的商品名字：')
    main()