Beautiful Soup支援的解析器

通過以上對比可以看出，lxml解析器有解析HTML和XML的功能，而且速度快，容錯能力強，所以推薦使用它。

為何很多python爬蟲工程師都這樣說呢，下面我用例項來證明

例項：通過爬去糗事百科文字內容中的資訊來比較各解析器的效能，爬取的資訊有：使用者ID，發表的段子文字資訊，好笑數量和評論數量。如圖：

程式碼：

import re
from bs4 import BeautifulSoup
from lxml import etree
import lxml
import time
import requests

headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
                      'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'
}
def get_lxml(url):
    datas = []
    res = requests.get(url,headers = headers)
    html = etree.HTML(res.text)
    infos = html.xpath('//div[@id="content-left"]/div')
    for info in infos:
        ids = info.xpath('div[1]/a[2]/h2/text()')
        texts_x = info.xpath('a[1]/div[1]/span[1]/text()')
        num_laughs = info.xpath('div[2]/span[1]/i[1]/text()')
        num_comments = info.xpath('// */ i/text()')
        for id,text_x,num_luagh,num_comment in zip(ids,texts_x,num_laughs,num_comments):
            data = {
                'id':id.strip(),
                'text':text_x.strip(),
                'num_luagh':num_luagh,
                'num_comment':num_comment
            }
            datas.append(data)
    #print(datas)



def get_re(url):
    datas=[]
    res = requests.get(url)
    ids = re.findall('<h2>(.*?)</h2>',res.text,re.S)
    texts_x = re.findall('<div class="content">.*?<span>(.*?)</span>',res.text,re.S)
    num_laughs = re.findall('<span class="stats-vote"><i class="number">(.*?)</i> 好笑</span>',res.text,re.S)
    num_comments = re.findall('<i class="number">(\d+)</i> 評論',res.text,re.S)
    for id,text_x,num_luagh,num_comment in zip(ids,texts_x,num_laughs,num_comments):
        data = {
            'id':id.strip(),
            'text':text_x.strip(),
            'num_luagh':num_luagh.strip(),
            'num_comment':num_comment.strip()
        }
        datas.append(data)
    #print(datas)

def get_BeautifulSoup(url):
    datas = []
    res = requests.get(url)
    soup = BeautifulSoup(res.text,'lxml')
    ids = soup.select('div.author.clearfix > a > h2')
    #print(ids)
    texts_x = soup.select('a.contentHerf > div > span')
    #print(texts_x)
    num_laughs = soup.select('div.stats > span.stats-vote > i')
    #print(num_laughs)
    num_comments = soup.select('i')
    #print(num_comments)
    for id,text_x,num_luagh,num_comment in zip(ids,texts_x,num_laughs,num_comments):
        data = {
            'id':id.get_text().strip(),
            'text':text_x.get_text().strip(),
            'num_luagh':num_luagh.get_text().strip(),
            'num_comment':num_comment.get_text().strip()
        }
        datas.append(data)
    #print(datas)


if __name__=='__main__':
    urls = ['https://www.qiushibaike.com/text/page/{}/'.format(str(i)) for i in range(1,14)]
    for name,function in [('lxml',get_lxml),('re',get_re),('BeautifulSoup',get_BeautifulSoup)]:
        start = time.time()
        for url in urls:
            function(url)
            time.sleep(0.2)
        end = time.time()
        print(name,end-start)
 

執行結果：

資料證明，lxml效能遠遠大於正則表示式和BeautifulSoup。

謝謝您的閱讀！