這裡通過pyspider框架

可以直接:pip3 install pyspider 下載框架

pyspider all 執行 可以看到 run 0.0.0.0:5000

直接在瀏覽器輸入localhost:5000 進入 建立新專案 

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-08-03 09:59:05
# Project: TripAdcisor

from pyspider.libs.base_handler import *
#import ssl
import pymongo

#ssl._create_default_https_context=ssl._create_unverified_context

class Handler(BaseHandler):
    crawl_config = {
    }
    conn = pymongo.MongoClient('localhost')
    db = conn.trip
    myset = db.london
    @every(minutes=24 * 60)
#on_start 啟動目標主網站,validate_cert = False跳過證書檢測,callback回撥函式
    def on_start(self):
        self.crawl('https://www.tripadvisor.cn/Attractions-g186338-Activities-c47-London_England.html', callback=self.index_page,validate_cert = False)

    @config(age=10 * 24 * 60 * 60)
#通過內建pyquery 獲取目標網頁的連線遍歷 分別訪問,並回調
    def index_page(self, response):
        for each in response.doc('.listing_title > a').items():
            self.crawl(each.attr.href, callback=self.detail_page,validate_cert = False)
#獲取下一頁連結地址訪問並回調自身
        next = response.doc('#FILTERED_LIST > div.al_border.deckTools.btm > div > div > a').attr.href
        print(next)
        self.crawl(next,callback= self.index_page,validate_cert=False)

    @config(priority=2)
#獲得每個連結的的詳細資訊返回
    def detail_page(self, response):
        url = response.url
        name = response.doc('.heading_title').text()
        rating = response.doc('div.rating > .more').text()
        location = response.doc('#taplc_attraction_detail_listing_0 > div.section.location > div.detail_section.address').text()[2:]
        phone = response.doc('.phone > div').text()
        durtion = response.doc('.hours > .duration').text()
        
        return {
            
            'name':name,
            'rating':rating,
            'location':location,
            'phone':phone,
            'durtion':durtion,
            'url':url
        }
#重寫on_result方法 儲存至資料庫
    def on_result(self,result):
        if result:
            self.save_to_mongo(result)
    def save_to_mongo(self,result):
        if self.myset.insert(result):
            print('save ok!',result)
        
 

執行時將status 調至running 或者debug    run即可