【道高一尺，魔高一丈】Python爬虫之如何应对网站反爬虫策略

agent_list = [
  "Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
  "Avant Browser/1.2.789rel1 (http://www.avantbrowser.com)",
  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5",
  "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9",
  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7",
  "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.601.0 Safari/534.14",
  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14",
  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20",
  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",
  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",
  "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
  "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre",
  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10"
  ]

'User-Agent': random.choice(agent_list)

# 随机等待时间是0.5秒和1秒之间的一个小数
time.sleep(random.uniform(0.5, 1))

def get_ip_pool(cnt):
  """获取代理ip的函数"""
  url_api = '获取代理IP的API地址'
  try:
    r = requests.get(url_api)
    res_text = r.text
    res_status = r.status_code
    print('获取代理ip状态码：', res_status)
    print('返回内容是：', res_text)
    res_json = json.loads(res_text)
    ip_pool = random.choice(res_json['RESULT'])
    ip = ip_pool['ip']
    port = ip_pool['port']
    ret = str(ip) + ':' + str(port)
    print('获取代理ip成功 -> ', ret)
    return ret
  except Exception as e:
    print('get_ip_pool except:', str(e))
proxies = get_ip_pool() # 调用获取代理ip的函数
requests.get(url=url, headers=headers, proxies={'HTTPS': proxies}) # 发送请求

def f_solve_captcha(v_infile, offset_x, offset_y, multiple=0.55):
  """
  利用超级鹰识别验证码
  :param offset_x: x轴偏移量
  :param offset_y: y轴偏移量
  :param v_infile: 验证码图片
  :param multiple: 图片缩小系数
  :return: 验证码识别结果坐标list
  """
  outfile = 'new-' + v_infile
  # 1、图片缩小到超级鹰要求:宽不超过460px,高不超过310px
  img = Image.open(v_infile)
  w, h = img.size
  w, h = round(w * multiple), round(h * multiple)  # 去掉浮点，防报错
  img = img.resize((w, h), Image.ANTIALIAS)
  img.save(outfile, optimize=True, quality=85)  # 质量为85效果最好
  print('pic smaller done!')
  # 2、调用超级鹰识别
  chaojiying = Chaojiying_Client(cjy_username, cjy_password, cjy_soft_id)
  im = open(outfile, 'rb').read()
  ret = chaojiying.PostPic(im, 9008)
  print(ret)
  loc_list2 = []
  if ret['err_no'] == 0:  # 返回码0代表成功
    loc_list = ret['pic_str'].split('|')
    for loc in loc_list:
      loc_x = round(int(loc.split(',')[0]) / multiple)
      loc_y = round(int(loc.split(',')[1]) / multiple)
      loc_list2.append((loc_x + offset_x, loc_y + offset_y))
    print('超级鹰返回正确,loc_list2 is:')
    print(loc_list2)
    print('长度是:{}'.format(len(loc_list2)))
  else:
    print('超级鹰返回错误!错误码：{}，错误内容：{}'.format(ret['err_no'], ret['err_str']))
  return loc_list2