import os
import urllib.request
from  urllib import parse

#https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3&pn=50

def writePage(filename,html):
    """
    :param filename:
    """
    with open(filename, "wb") as f:
        f.write(html)

def request(url,begPage,endPage):
    for index in range(begPage, endPage + 1):
        ua_url = fullurl + "&pn=" + str((index - 1) * 50)
        print(ua_url)
        request = urllib.request.Request(ua_url, headers=headers)
        response = urllib.request.urlopen(request).read()
        filename = "第" + str(index) + "頁.html"
        writePage(filename,response)

if __name__ == "__main__":
    url = "https://tieba.baidu.com/f?"
    key = input("please input keywords to query >>")
    begPage = int(input("請輸入開始頁數 >> "))
    endPage = int(input("請輸入結束頁數 >> "))
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0"}
    kw = parse.urlencode({"kw": key})
    fullurl = url + kw
    request(fullurl,begPage,endPage)
 

抓取貼吧