抓取第一批北京积分落户名单并做简要分析抓取分析&作图

# 抓取北京市2018年积分落户公示名单
# http://www.bjrbj.gov.cn/integralpublic/settlePerson

import csv
import json
import requests

fw = open('luohu.csv', 'w')
writer = csv.writer(fw)
writer.writerow(['id','name','birthday','company','score'])

def get_publicity(page_number):
    url = 'http://www.bjrbj.gov.cn/integralpublic/settlePerson/settlePersonJson?sort=pxid&order=asc&limit=100&offset=0&name=&rows=100&page={}'.format(page_number*100)

    header = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                          '(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
        }

    response = requests.get(url,
                            headers=header,
                            timeout=5
                            )

    result = json.loads(response.text)

    for item in result['rows']:
        id = item['pxid']
        name = item['name']
        birthday = item['csrq']
        company = item['unit']
        score = item['score']
        print(id,name,birthday,company,score)
        writer.writerow([id, name, birthday, company, score])

def main():
    for i in range(0,61):
        get_publicity(i)


if __name__ == '__main__':
    main()

import pandas as pd
import matplotlib.pyplot as plt

%matplotlib inline

df_luohu = pd.read_csv('luohu.csv')
print(df_luohu.describe())

                id        score
count  6019.000000  6019.000000
mean   3010.000000    95.654552
std    1737.679967     4.354445
min       1.000000    90.750000
25%    1505.500000    92.330000
50%    3010.000000    94.460000
75%    4514.500000    97.750000
max    6019.000000   122.590000

df_luohu_grouped = df_luohu.groupby('company', as_index=False, sort=False).count()[['company','id']]
df_luohu_grouped_sorted = df_luohu_grouped.sort_values('id', ascending=False)
df_luohu_grouped_sorted.head(20)

bins = [90,95,100,105,110,115,120,125]
bins = pd.cut(df_luohu['score'], bins)

def get_stats(group):
    return {'count': group.count()}

grouped = df_luohu['score'].groupby(bins)
bin_counts = grouped.apply(get_stats).unstack()

bin_counts.index = ['90~95', '95~100', '100~105', '105~110', '110~115', '115~120', '120~125']
bin_counts.index.name = 'score'
bin_counts.plot(kind='bar', alpha=1, rot=0)

df_luohu['age'] = ((pd.to_datetime('2018-11') - pd.to_datetime(df_luohu['birthday'])) / pd.Timedelta('365 days'))

bins = [25,30,35,40,45,50,55,60,65,70]
bins = pd.cut(df_luohu['age'], bins)

def get_stats(group):
    return {'count': group.count()}

grouped = df_luohu['age'].groupby(bins)
bin_counts = grouped.apply(get_stats).unstack()

bin_counts.index = ['25~30',  '30~35', '35~40', '40~45', '45~50', '50~55', '55~60', '60~65', '65~70',]
bin_counts.index.name = 'age'
bin_counts.plot(kind='bar', alpha=1, rot=0)