import re
import time
from multiprocessing import Pool
import requests

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:61.0) Gecko/20100101 Firefox/61.0'
}

def re_scraper(url):
    res = requests.get(url,headers=headers)
    names = re.findall('<h2>(.*?)</h2>',res.text,re.S)
    contents = re.findall('<div class="content">.*?</div',res.text,re.S)
    laughs = re.findall('<span class="stats-vote">.*?<i class="number">(\d+)</i>',res.text,re.S)
    comments = re.findall('<i class="number">(\d+)</i> 評論',res.text,re.S)

    infos = list()
    for name,content,laugh,comment in zip(names,contents,laughs,comments):
        info = {
            'name':name,
            'content':comment,
            'laugh':laugh,
            'comment':comment,
        }
        infos.append(info)
        return  infos

if __name__ == "__main__":
    # re_scraper("https://www.qiushibaike.com/8hr/page/1/")
    urls = ["https://www.qiushibaike.com/8hr/page/{}/".format(str(i)) for i in range(1,35)]
    start_1 = time.time()
    for url in urls:
        re_scraper(url)
    end_1 = time.time()
    print('序列爬蟲時間:',end_1-start_1)

    start_2 = time.time()
    pool = Pool(processes=2)
    pool.map(re_scraper,urls)
    end_2 = time.time()
    print('2程序爬蟲耗時:',end_2-start_2)


    start_3 = time.time()
    pool = Pool(processes=4)
    pool.map(re_scraper,urls)
    end_3 = time.time()
    print('2程序爬蟲耗時:',end_3-start_3)