一、需求分析
模擬登陸新浪微博,爬取新浪微博的熱門話題版塊的24小時內的前TOP500的話題名稱、該話題的閱讀數、討論數、粉絲數、話題主持人，以及對應話題主持人的關注數、粉絲數和微博數。

二、開發語言
python2.7

三、需要匯入模組
import requests
import json
import base64
import re
import time
import pandas as pd

四、抓取流程
首先先手機端模擬登陸新浪
然後傳送請求得到網頁原始碼資料
之後用正則表示式解析資料

五、欄位說明

話題名稱：topic_name
閱讀數：topic_reading
討論數：topic_discuss
話題粉絲數：topic_fans
話題主持人：host_name
主持人關注數：host_follow
主持人粉絲數：host_fans
主持人微博數：host_weibo

六、抓取步驟
1、模擬登入新浪微博（手機版), 解密，用requests傳送post請求,session方法記住登入狀態。

###########模擬登入新浪
def login(username, password):
    username = base64.b64encode(username.encode('utf-8')).decode('utf-8')
    postData = {
        "entry": "sso",
        "gateway": "1",
        "from": "null",
        "savestate": "30"
 
,
        "useticket": "0",
        "pagerefer": "",
        "vsnf": "1",
        "su": username,
        "service": "sso",
        "sp": password,
        "sr": "1440*900",
        "encoding": "UTF-8",
        "cdult": "3",
        "domain": "sina.com.cn",
        "prelt": "0",
        "returntype": "TEXT"
 
,
    }
    loginURL = r'https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.15)'
    session = requests.Session()
    res = session.post(loginURL, data = postData)
    jsonStr = res.content.decode('gbk')
    info = json.loads(jsonStr)
    if info["retcode"] == "0":
        print(U"登入成功")
        # 把cookies新增到headers中，必須寫這一步，否則後面呼叫API失敗
        cookies = session.cookies.get_dict()
        cookies = [key + "=" + value for key, value in cookies.items()]
        cookies = "; ".join(cookies)
        session.headers["cookie"] = cookies
    else:
        print(U"登入失敗，原因： %s" % info["reason"])
    return session

session = login('新浪微博賬號', '新浪微博密碼')

2、定義列表資料結構儲存資料。

##################定義資料結構列表儲存資料
top_name = []
top_reading = []
top_discuss = []
top_fans = []
host_name = []
host_follow = []
host_fans = []
host_weibo = []
url_new1=[]
url_new2=[]

3、請求翻頁實現，用Fiddle 抓包工具 抓取請求url,點點選下一頁時,發現真正請求url,通過拼接 page 來實現翻頁

4、解析網頁採用正則表示式re.findall()

#####正則表示式匹配

    name=re.findall("Pl_Discover_Pt6Rank__5(.*?)</script>",html,re.S)
    for each in name:
        # print each

返回的是json ,資料，這裡有使用者的關注數，微博數，粉絲數。
通過解析json資料，得到這三個欄位。

方法如下：

url="http://m.weibo.cn/api/container/getIndex?type=uid&value=5710151998"
html=session.get(url).content
html=json.loads(html)
userInfo=html['userInfo']
statuses_count=userInfo['statuses_count']
followers_count=userInfo['followers_count']
follow_count=userInfo['follow_count']

print statuses_count,followers_count,follow_count

6、利用迴圈抓取資料之後，把資料組織成資料框的形式（表格）寫入excel

7、通過 time模組 time.sleep(4) 方法控制網路請求速度。

七、結果圖

八、實現原始碼

# encoding: utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import requests
import json
import base64
import re
import time
import pandas as pd

time1=time.time()


###########模擬登入新浪
def login(username, password):
    username = base64.b64encode(username.encode('utf-8')).decode('utf-8')
    postData = {
        "entry": "sso",
        "gateway": "1",
        "from": "null",
        "savestate": "30",
        "useticket": "0",
        "pagerefer": "",
        "vsnf": "1",
        "su": username,
        "service": "sso",
        "sp": password,
        "sr": "1440*900",
        "encoding": "UTF-8",
        "cdult": "3",
        "domain": "sina.com.cn",
        "prelt": "0",
        "returntype": "TEXT",
    }
    loginURL = r'https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.15)'
    session = requests.Session()
    res = session.post(loginURL, data = postData)
    jsonStr = res.content.decode('gbk')
    info = json.loads(jsonStr)
    if info["retcode"] == "0":
        print(U"登入成功")
        # 把cookies新增到headers中，必須寫這一步，否則後面呼叫API失敗
        cookies = session.cookies.get_dict()
        cookies = [key + "=" + value for key, value in cookies.items()]
        cookies = "; ".join(cookies)
        session.headers["cookie"] = cookies
    else:
        print(U"登入失敗，原因： %s" % info["reason"])
    return session


session = login('此處填寫你的微博賬號', '此處填寫你的微博密碼')
##################定義資料結構列表儲存資料
top_name = []
top_reading = []
top_discuss = []
top_fans = []
host_name = []
host_follow = []
host_fans = []
host_weibo = []
url_new1=[]
url_new2=[]


#########################開始迴圈抓取
for i in range(1,501):
    try:
        print "正在抓取第"+str(i)+"頁。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。。"
        url2="http://d.weibo.com/100803?pids=Pl_Discover_Pt6Rank__5&cfs=920&Pl_Discover_Pt6Rank__5_filter=hothtlist_type=1&Pl_Discover_Pt6Rank__5_page="+str(i)+"&ajaxpagelet=1&__ref=/100803&_t=FM_149273744327929"
        html=session.get(url2).content

        ###########正則表示式匹配#######################
        name=re.findall("Pl_Discover_Pt6Rank__5(.*?)</script>",html,re.S)
        for each in name:
            # print each
            k=re.findall('"html":"(.*?)"}',each,re.S)
            for each1 in k:
                k1=str(each1).replace('\\t',"").replace('\\n','').replace('\\','').replace('#','')
                # print k1
                k2=re.findall('alt="(.*?)" class="pic">',str(k1),re.S)
                for each2 in k2:
                    print each2
                    top_name.append(each2)

                k3=re.findall('</span><a target="_blank" href="(.*?)" class="S_txt1"  >',str(k1),re.S)
                for each3 in k3:
                    print each3
                    url_new1.append(each3)
                    heads={
                    "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
                    "Accept-Encoding":"gzip, deflate, sdch",
                    "Accept-Language":"zh-CN,zh;q=0.8",
                    "Cache-Control":"max-age=0",
                    "Connection":"keep-alive",
                    "Host":"weibo.com",
                    "Upgrade-Insecure-Requests":"1",
                    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36"}

                    html2=session.get(each3,headers=heads).content
                    time.sleep(4)
                    p1=re.findall('Pl_Core_T8CustomTriColumn__12(.*?)</script>',str(html2),re.S)
                    for each3_1 in p1:
                        p2=str(each3_1).replace('\\t',"").replace('\\n','').replace('\\','').replace('#','')
                        # print p2
                        p3=re.findall('閱讀</span>r<td class="S_line1">r<strong (.*?)</strong><span class="S_txt2">討論</span>',str(p2),re.S)
                        for each3_2 in p3:
                            print str(each3_2).replace('class="">','').replace('class="W_f12">','').replace('class="W_f16">','').replace('class="W_f14">','').replace('class="W_f18">','')
                            top_discuss.append(str(each3_2).replace('class="">','').replace('class="W_f12">','').replace('class="W_f16">','').replace('class="W_f14">','').replace('class="W_f18">',''))

                        p4=re.findall('><strong class(.*?)</strong><span class="S_txt2">粉絲',str(p2),re.S)
                        for each3_3 in p4:
                            print str(each3_3).replace('="">','').replace('="W_f12">','').replace('="W_f16">','').replace('="W_f14">','').replace('="W_f18">','')
                            top_fans.append(str(each3_3).replace('="">','').replace('="W_f12">','').replace('="W_f16">','').replace('="W_f14">','').replace('="W_f18">',''))



                k4=re.findall('閱讀數：<span><span class="number">(.*?) </span></div> <div class="sub_box W_fl">',str(k1),re.S)
                for each4 in k4:
                    print each4
                    top_reading.append(each4)

                k5=re.findall('主持人:<span><a target="_blank" href="(.*?)" class="tlink S_txt1"',str(k1),re.S)
                for each5 in k5:
                    print each5
                    mm=re.findall('\d+',str(each5),re.S)
                    for mm_1 in mm:
                        pp1="http://m.weibo.cn/api/container/getIndex?type=uid&value="+str(mm_1)
                        html3=session.get(pp1).content
                        html3=json.loads(html3)
                        userInfo=html3['userInfo']
                        statuses_count=userInfo['statuses_count']
                        followers_count=userInfo['followers_count']
                        follow_count=userInfo['follow_count']
                        print statuses_count,followers_count,follow_count
                        host_follow.append(follow_count)
                        host_fans.append(followers_count)
                        host_weibo.append(statuses_count)
                        url_new2.append(pp1)



                k6 = re.findall('" class="tlink S_txt1"   >(.*?)</a></div> </div><div class="opt_box"', str(k1), re.S)
                for each6 in k6:
                    print each6
                    host_name.append(each6)

    except:
        pass





print len(top_name),len(top_reading),len(top_discuss),len(top_fans),len(host_name),len(url_new2),len(host_follow),len(host_fans),len(host_weibo)



data = pd.DataFrame({"top_name":top_name[0:501], "top_reading": top_reading[0:501],"top_discuss":top_discuss[0:501],"top_fans":top_fans[0:501],"host_name":host_name[0:501],\
                     "host_follow":host_follow[0:501],"host_fans":host_fans[0:501],"host_weibo":host_weibo[0:501]})


print len(data)
# 寫出excel
writer = pd.ExcelWriter(r'C:\\sina_weibo_topic500.xlsx', engine='xlsxwriter', options={'strings_to_urls': False})
data.to_excel(writer, index=False)
writer.close()

time2 = time.time()
print u'ok,爬蟲結束!'
print u'總共耗時：' + str(time2 - time1) + 's'