使用Python快速获取公众号文章定制电子书(二)

url ="https://..." \
          "action=getmsg&" \
          "__biz=MjM5ODIyMTE0MA==&" \
          "f=json&" \
          "offset=11&" \
          "count=10&" \
          "is_ok=1&" \
          "scene=124&" \
          "uin=777&" \
          "key=777&" \
          "pass_ticket=Pu%2FH3aPR7f%2FzgA52T%2Bv4fU9wSWkY5tgGGgAXWewji2dqpMDrjaxUbBR%2Fmo2e%2FaMX&" \
          "wxtoken=&" \
          "appmsg_token=956_cnSiifKearMa6Um6cS3fcmXnu1AfKSYN5dSOSA~~&" \
          "x5=1&" \
          "f=json"

def crawl_msg(offset=0):
    """
    爬取更多文章
    """
    url = "https://..." \
          "action=getmsg&" \
          "__biz=MjM5ODIyMTE0MA==&" \
          "f=json&" \
          "offset={offset}&" \
          "count=10&" \
          "is_ok=1&" \
          "scene=124&" \
          "uin=777&" \
          "key=777&" \
          "pass_ticket=Pu%2FH3aPR7f%2FzgA52T%2Bv4fU9wSWkY5tgGGgAXWewji2dqpMDrjaxUbBR%2Fmo2e%2FaMX&" \
          "wxtoken=&" \
          "appmsg_token=956_cnSiifKearMa6Um6cS3fcmXnu1AfKSYN5dSOSA~~&" \
          "x5=1&" \
          "f=json".format(offset=offset)
    # url 反转义
    url = html.unescape(url)

    headers = """
Host: ...
Connection: keep-alive
X-Requested-With: XMLHttpRequest
User-Agent: Mozilla/5.0 (Linux; Android 6.0.1; NX531J Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/6.2 TBS/044030 Mobile Safari/537.36 MicroMessenger/6.6.5.1280(0x26060532) NetType/WIFI Language/zh_CN
Accept: */*
Referer: ...action=home&__biz=MjM5ODIyMTE0MA==&scene=124&devicetype=android-23&version=26060532&lang=zh_CN&nettype=WIFI&a8scene=3&pass_ticket=Pu%2FH3aPR7f%2FzgA52T%2Bv4fU9wSWkY5tgGGgAXWewji2dqpMDrjaxUbBR%2Fmo2e%2FaMX&wx_header=1
Accept-Encoding: gzip, deflate
Accept-Language: zh-CN,en-US;q=0.8
Cookie: sd_userid=96301522585723838; sd_cookie_crttime=1522585723838; pgv_pvid=1151171120; tvfe_boss_uuid=92a513a6354c3896; rewardsn=; wxtokenkey=777; wxuin=1336178621; devicetype=android-23; version=26060532; lang=zh_CN; pass_ticket=Pu/H3aPR7f/zgA52T+v4fU9wSWkY5tgGGgAXWewji2dqpMDrjaxUbBR/mo2e/aMX; wap_sid2=CL3vkf0EEnBSWHFYWmVoZVpOMjU0cnBpSUhiLWF2cmZHVVVLVWZrWUp4QVRlclVVOTRwS1hmMGNUR0VJaXp1RlVzbGViM2wtZnVfakZVd21RcGxxbzI3U3R3cmtYYlUycXpLU0FzcGJFSm1ESkZsYVhzSzhBd0FBMLbQ5dcFOA1AlU4=
Q-UA2: QV=3&PL=ADR&PR=WX&PP=com.tencent.mm&PPVN=6.6.5&TBSVC=43603&CO=BK&COVC=044030&PB=GE&VE=GA&DE=PHONE&CHID=0&LCID=9422&MO= NX531J &RL=1080*1920&OS=6.0.1&API=23
Q-GUID: 6a875f18ea5ba76bb6afb9ca13b788cb
Q-Auth: 31045b957cf33acf31e40be2f3e71c5217597676a9729f1b
"""
    headers = headers_to_dict(headers)
    response = requests.get(url, headers=headers, verify=False)
    if '<title>验证</title>' in response.text:
        raise Exception("获取微信公众号文章失败，可能是因为你的请求参数有误，请重新获取")
    result = response.json()
    if result.get("ret") == 0:
        msg_list = result.get("general_msg_list")
        data = json.loads(msg_list)
        articles = data.get("list")
        articles_lists = dict()
        for item in articles:
            if item.get("app_msg_ext_info"):
                articles_lists[item["app_msg_ext_info"]["title"]] = item["app_msg_ext_info"]["content_url"]
        rex = r'\\/'
        for item in articles_lists:
            pattern = re.sub(rex, '/', html.unescape(articles_lists.get(item)))
            response = requests.get(pattern, headers=headers, verify=False)
            parser_text_to_file(item, response.text)
        print("抓取数据: offset=%s, data=%s" % (offset, msg_list))
        has_next = result.get("can_msg_continue")
        if has_next == 1:
            next_offset = result.get("next_offset")
            time.sleep(2) # 每两秒爬取一次
            crawl_msg(next_offset)
    else:
        print("无法正确获取内容，请重新从Fiddler/Charles获取请求参数和请求头")
        exit()