利用到的庫

• time, requests, lxml, queue, threading

功能

• 爬取不得姐網站中前二十頁的段子資料

import time
import requests
from lxml import etree
from queue import Queue
import threading


class bsSpider:
    def __init__(self):
        self.baseUrl = "http://www.budejie.com/"
        self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"}
        # URL佇列
        self.urlQueue = Queue()
        # 相應html佇列
        self.resQueue = Queue()

    # 生成url佇列
    def getUrl(self):
        for pNumber in range(1, 01):
            url = self.baseUrl + str(pNumber)
            self.urlQueue.put(url)

    # 請求，得到相應的html，放到解析佇列
    def getHtml(self):
        while True:
            # 1.從url佇列中get值
            url = self.urlQueue.get()
            # 2.發請求，得響應，put到響應佇列中
            res = requests.get(url, headers=self.headers)
            res.encoding = "utf-8"
            html = res.text
            self.resQueue.put(html)
            # 清除此任務
            self.urlQueue.task_done()

    # 解析頁面方法
    def getText(self):
        while True:
            html = self.resQueue.get()
            parseHtml = etree.HTML(html)
            r_list = parseHtml.xpath('//div[@class="j-r-list-c-desc"]/a/text()')
            for r in r_list:
                print(r+"\n")
            self.resQueue.task_done()

    def run(self):
        # 空列表，用來存放
        thList = []
        # 生成URL佇列
        self.getUrl()
        # 建立請求執行緒，放到列表中
        for i in range(10):
            thRes = threading.Thread(target=self.getHtml)
            thList.append(thRes)
        # 建立解析執行緒，放到列表中
        for i in range(3):
            thParse = threading.Thread(target=self.getText)
            thList.append(thParse)
        # 所有執行緒開始幹活
        for th in thList:
            th.setDaemon(True)
            th.start()

        # 如果佇列為空，則執行其他程式
        self.urlQueue.join()
        self.resQueue.join()


if __name__ == '__main__':
    begin = time.time()
    spider = bsSpider()
    spider.run()
    end = time.time()
    print(end - begin)