从无到有的“糗事百科”分析

#encoding:utf-8
__author__ = 'herain'
import sys
import urllib
import urllib2
import codecs
import re
import thread
import time
#糗事百科爬虫
class BaikeCrawler:
  #init mothed
  def __init__(self):
    self.pageNum = 1
    self.user_agent = '''Mozilla/5.0 
    (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36'''
    #init headers
    self.headers = {'User-Agent': self.user_agent}
    #save msg for page
    self.stories = []
    #program on-off
    self.enable = False
    self.__out_file = codecs.open("qiuxi24_data",'w','utf-8')
  #afferent pageNum
  def forCrawler(self, pageNum):
    try:
      url ='http://www.qiushibaike.com/hot/page/' + str(pageNum)
      request = urllib2.Request(url, headers = self.headers)
      #use urlopen get the html code
      response = urllib2.urlopen(request)
      pageHtmlCode = response.read().decode('utf-8')
      return pageHtmlCode
    except urllib2.URLError, e:
      if hasattr(e, 'reason'):
        print u"链接糗事百科失败，错误原因",e.reason
        return None

  def clearHtmlTag(self,ctx):
    pat = re.compile(r"<[^>]+>|\s",re.S)
    return str(pat.sub('',ctx)).encode('utf-8')
  def clearOtherChar(self, ctx):
    #处理非中文和英文字符
    pat = re.compile(r"<[^>]+>|\s",re.S)
    return str(pat.sub('',ctx)).encode('utf-8')

  def write_line(self, datalist):
    div = '\001'
    res = div.join("%s"%str(i).encode('utf-8') for i in datalist)
    res = "%s\n"%(res)
    self.__out_file.writelines(res)

  def backPageItems(self,pageNum):
    pageHtmlCode = self.forCrawler(pageNum)
    if not pageHtmlCode:
      return None
    pageStories = []
    pattern =re.compile('''<div class="author clearfix">.*?href.*?<img src.*?title=.*?<h2>(.*?)</h2>.*?
      <div class="articleGender(.*?)">(.*?)</div>.*?<div class="content">.*?<span>(.*?)</span>.*?</div>
      .*?</a>(.*?)<div class="stats">.*?
      <i class="number">(.*?)</i>.*?class="stats-comments".*?<i class="number">(.*?)</i>''',re.S)
    items = re.findall(pattern,pageHtmlCode)
    for item in items:
      name = str(item[0]).encode('utf-8')
      sex = str(('男性' if item[1].strip()=='manIcon' else '女性')).encode('utf-8')
      age =  str(item[2]).encode('utf-8')
      ctx = self.clearHtmlTag(item[3])
      ishaveImg = str(('有' if item[4].strip() else '无')).encode('utf-8')
      likeNum =  str(item[5]).encode('utf-8')
      comments =  str(item[6]).encode('utf-8')
      #print '用户名:',name,'性别:',sex,'年龄:',age, '\n内容:',ctx, '\n是否有图:' ,ishaveImg,'\n点赞数:' ,likeNum, '评论数:',comments ,'\n\n'
      self.write_line((name,sex,age,ctx,ishaveImg,likeNum,comments))
    return pageStories
         
  def loadPage(self):
    if self.enable == True:
      if len(self.stories) < 2:
        pageStories = self.backPageItems(self.pageNum)
        if pageStories:
          self.stories.append(pageStories)
          self.pageNum += 1

  def getOneStory(self, pageStories, page):
    for story in pageStories:
      input = raw_input()
      self.loadPage()
      if input == 'Q':
        self.enable = False
        return 
      print '''第%d页\t用户名:%s性别:%s\年龄:%s\t内容:%s\t是否有图:%s\t点赞数:%s
      \t评论数:%s'''%(page,story[0],story[1],story[2],story[3],story[4],story[5],story[6])

  #start mothed
  def start(self):
    print u"正在读取糗事百科,按回车查看新段子，Q退出"
    self.enable = True
    self.loadPage()
    nowPage = 0;
    while self.enable:
      if len(self.stories)>0:
        pageStories = self.stories[0]
        nowPage += 1
        del self.stories[0]
        self.getOneStory(pageStories,nowPage)

if __name__ == "__main__":
  baike = BaikeCrawler()
  #baike.start() #该方法启动阅读模式
  user_define = raw_input('n,m从第n开始读取到第页') #0,10表示从0 开始读取到第10页
  pagelist = user_define.split(',')
  print int(pagelist[0]),int((pagelist[1] if pagelist[1] else 10))
  for i in range(int(pagelist[0]),int((pagelist[1] if pagelist[1] else 10))+1):
    baike.backPageItems(i)