執行平臺：windows

Python版本：Python 3.7.0

IDE:Sublime Text

瀏覽器：Chrome瀏覽器

思路：

            1.檢視網頁原始碼

            2.抓取單頁內容

            3.正則表示式提取資訊

            4.貓眼TOP100所有資訊寫入檔案

            5.多執行緒抓取

1.檢視網頁原始碼

按F12檢視網頁原始碼發現每一個電影的資訊都在“<dd></dd>”標籤之中。

點開之後，資訊如下：

二.抓取單頁內容

   在 瀏覽器中開啟貓眼電影網站，點選“榜單”，再點選“TOP100榜”如下圖：

接下來通過以下程式碼獲取網頁原始碼：

#-*-coding:utf-8-*-
import requests
from requests.exceptions import RequestException

#貓眼電影網站有反爬蟲措施，設定headers後可以爬取
headers = {
	'Content-Type': 'text/plain; charset=UTF-8',
	'Origin':'https://maoyan.com',
	'Referer':'https://maoyan.com/board/4',
	'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
	}

#爬取網頁原始碼
def get_one_page(url,headers):
	try:
		response =requests.get(url,headers =headers)
		if response.status_code == 200:
			return response.text
		return None
	except RequestsException:
		return None

def main():
	url = "https://maoyan.com/board/4"
	html = get_one_page(url,headers)
	print(html)

if __name__ == '__main__':
	main()
 

執行結果如下：

3.正則表示式提取資訊

上圖示示資訊即為要提取的資訊，程式碼實現如下：

#-*-coding:utf-8-*-
import requests
import re
from requests.exceptions import RequestException

#貓眼電影網站有反爬蟲措施，設定headers後可以爬取
headers = {
	'Content-Type': 'text/plain; charset=UTF-8',
	'Origin':'https://maoyan.com',
	'Referer':'https://maoyan.com/board/4',
	'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
	}

#爬取網頁原始碼
def get_one_page(url,headers):
	try:
		response =requests.get(url,headers =headers)
		if response.status_code == 200:
			return response.text
		return None
	except RequestsException:
		return None

#正則表示式提取資訊
def parse_one_page(html):
	pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
		+'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S)
	items = re.findall(pattern,html)
	for item in items:
		yield{
		'index':item[0],
		'image':item[1],
		'title':item[2],
		'actor':item[3].strip()[3:],
		'time':item[4].strip()[5:],
		'score':item[5]+item[6]
		}

def main():
	url = "https://maoyan.com/board/4"
	html = get_one_page(url,headers)
	for item in parse_one_page(html):
		print(item)

if __name__ == '__main__':
	main()
 

執行結果如下：

4.貓眼TOP100所有資訊寫入檔案

上邊程式碼實現單頁的資訊抓取，要想爬取100個電影的資訊，先觀察每一頁url的變化，點開每一頁我們會發現url進行變化，原url後面多了‘？offset=0’,且offset的值變化從0,10,20，變化如下：

程式碼實現如下：

#-*-coding:utf-8-*-
import requests
import re
import json
import os
from requests.exceptions import RequestException

#貓眼電影網站有反爬蟲措施，設定headers後可以爬取
headers = {
	'Content-Type': 'text/plain; charset=UTF-8',
	'Origin':'https://maoyan.com',
	'Referer':'https://maoyan.com/board/4',
	'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
	}

#爬取網頁原始碼
def get_one_page(url,headers):
	try:
		response =requests.get(url,headers =headers)
		if response.status_code == 200:
			return response.text
		return None
	except RequestsException:
		return None

#正則表示式提取資訊
def parse_one_page(html):
	pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
		+'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S)
	items = re.findall(pattern,html)
	for item in items:
		yield{
		'index':item[0],
		'image':item[1],
		'title':item[2],
		'actor':item[3].strip()[3:],
		'time':item[4].strip()[5:],
		'score':item[5]+item[6]
		}
#貓眼TOP100所有資訊寫入檔案
def write_to_file(content):
	#encoding ='utf-8',ensure_ascii =False,使寫入檔案的程式碼顯示為中文
	with open('result.txt','a',encoding ='utf-8') as f:
		f.write(json.dumps(content,ensure_ascii =False)+'\n')
		f.close()
#下載電影封面
def save_image_file(url,path):

	jd = requests.get(url)
	if jd.status_code == 200:
		with open(path,'wb') as f:
			f.write(jd.content)
			f.close()

def main(offset):
	url = "https://maoyan.com/board/4?offset="+str(offset)
	html = get_one_page(url,headers)
	if not os.path.exists('covers'):
		os.mkdir('covers')	
	for item in parse_one_page(html):
		print(item)
		write_to_file(item)
		save_image_file(item['image'],'covers/'+item['title']+'.jpg')

if __name__ == '__main__':
	#對每一頁資訊進行爬取
	for i in range(10):
		main(i*10)

爬取結果如下：

5.多執行緒抓取

進行比較，發現多執行緒爬取時間明顯較快：

多執行緒：

以下為完整程式碼：

#-*-coding:utf-8-*-
import requests
import re
import json
import os
from requests.exceptions import RequestException
from  multiprocessing import Pool
#貓眼電影網站有反爬蟲措施，設定headers後可以爬取
headers = {
	'Content-Type': 'text/plain; charset=UTF-8',
	'Origin':'https://maoyan.com',
	'Referer':'https://maoyan.com/board/4',
	'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
	}

#爬取網頁原始碼
def get_one_page(url,headers):
	try:
		response =requests.get(url,headers =headers)
		if response.status_code == 200:
			return response.text
		return None
	except RequestsException:
		return None

#正則表示式提取資訊
def parse_one_page(html):
	pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
		+'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S)
	items = re.findall(pattern,html)
	for item in items:
		yield{
		'index':item[0],
		'image':item[1],
		'title':item[2],
		'actor':item[3].strip()[3:],
		'time':item[4].strip()[5:],
		'score':item[5]+item[6]
		}
#貓眼TOP100所有資訊寫入檔案
def write_to_file(content):
	#encoding ='utf-8',ensure_ascii =False,使寫入檔案的程式碼顯示為中文
	with open('result.txt','a',encoding ='utf-8') as f:
		f.write(json.dumps(content,ensure_ascii =False)+'\n')
		f.close()
#下載電影封面
def save_image_file(url,path):

	jd = requests.get(url)
	if jd.status_code == 200:
		with open(path,'wb') as f:
			f.write(jd.content)
			f.close()

def main(offset):
	url = "https://maoyan.com/board/4?offset="+str(offset)
	html = get_one_page(url,headers)
	if not os.path.exists('covers'):
		os.mkdir('covers')	
	for item in parse_one_page(html):
		print(item)
		write_to_file(item)
		save_image_file(item['image'],'covers/'+item['title']+'.jpg')

if __name__ == '__main__':
	#對每一頁資訊進行爬取
	pool = Pool()
	pool.map(main,[i*10 for i in range(10)])
	pool.close()
	pool.join()