scrapy框架的另一種分頁處理以及mongodb的持久化儲存以及from_crawler類方法的使用
阿新 • • 發佈：2019-03-08
Coding pca rom utf-8 ngs ODB 持久 same req 
一.scrapy框架處理
1.分頁處理
以爬取亞馬遜為例
爬蟲文件.py
# -*- coding: utf-8 -*- import scrapy from Amazon.items import AmazonItem class AmazonSpider(scrapy.Spider): name = ‘amazon‘ allowed_domains = [‘www.amazon.cn‘] start_urls = [‘www.amazon.cn‘] def start_requests(self):# 重寫父類方法,拿到商品搜索頁 url = ‘https://www.amazon.cn/s/ref=nb_sb_noss?__mk_zh_CN=亞馬遜網站&url=search-alias%3Daps&field-keywords=iphone+-xs&rh=i%3Aaps%2Ck%3Aiphone+-xs&ajr=0‘ yield scrapy.Request(url=url, callback=self.parse) def parse(self, response): # 解析每一個商品的urllinks = response.xpath(‘//*[contains(@id,"result_")]/div/div[3]/div[1]/a/@href‘).extract() # 同時拿到下一頁的連接 next_page_url = response.xpath(‘//a[@id="pagnNextLink"]/@href‘).extract_first() print(‘>>>>>>>>>>>>>‘, next_page_url) # 再對這些每一個商品的url進行請求for link in links: yield scrapy.Request(url=link, callback=self.parse_detail) #分頁處理 # 把所有的商品詳情遍歷完了之後,再判斷是否有下一頁,有下一頁就繼續對下一頁發起請求 if next_page_url: scrapy.Request(url=next_page_url, callback=self.parse) def parse_detail(self, response): #每個商品的詳情頁解析出我們要的數據 title = response.xpath(‘//*[@id="productTitle"]/text()‘).extract_first().strip() price = (response.xpath("//*[@id=‘priceblock_ourprice‘]/text()") or response.xpath( "//*[@id=‘priceblock_saleprice‘]/text()")).extract_first().strip() deliver = response.xpath(‘//*[@id="ddmMerchantMessage"]/*[1]/text()‘).extract_first().strip() #把數據裝到容器裏面 item=AmazonItem() item[‘title‘]=title item[‘price‘]=price item[‘deliver‘]=deliver #記得返回,否則管道接不到 yield item
2.mongodb持久化儲存以及from_crawl的使用
pipelines.py
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don‘t forget to add your pipeline to the ITEM_PIPELINES setting # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html import pymongo class AmazonPipeline(object): @classmethod def from_crawler(cls, crawler): """ Scrapy會先通過getattr判斷我們是否自定義了from_crawler,有則調它來完 成實例化,早於__init__方法執行
自己要的參數要去settings.py文件配置 """ HOST = crawler.settings.get(‘HOST‘) PORT = crawler.settings.get(‘PORT‘) USER = crawler.settings.get(‘USER‘) PWD = crawler.settings.get(‘PWD‘) DB = crawler.settings.get(‘DB‘) TABLE = crawler.settings.get(‘TABLE‘) return cls(HOST, PORT, USER, PWD, DB, TABLE) def __init__(self,host,port,user,pwd,db,table): self.host=host self.port=port self.user=user self.pwd=pwd self.db=db self.table=table def open_spider(self,spider): #程序運行時執行一次 self.client=pymongo.MongoClient(host=self.host,port=self.port) def process_item(self, item, spider): dic_item=dict(item) if dic_item: self.client[self.db][self.table].save(dic_item) return item def close_spider(self,spider): #程序關閉時候執行一次 self.client.close()
settings.py
# -*- coding: utf-8 -*- # Scrapy settings for Amazon project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # https://doc.scrapy.org/en/latest/topics/settings.html # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html # https://doc.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = ‘Amazon‘ SPIDER_MODULES = [‘Amazon.spiders‘] NEWSPIDER_MODULE = ‘Amazon.spiders‘ # Crawl responsibly by identifying yourself (and your website) on the user-agent USER_AGENT = ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36‘ # Obey robots.txt rules ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8‘, # ‘Accept-Language‘: ‘en‘, #} # Enable or disable spider middlewares # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # ‘Amazon.middlewares.AmazonSpiderMiddleware‘: 543, #} # Enable or disable downloader middlewares # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { ‘Amazon.middlewares.AmazonDownloaderMiddleware‘: 543, } # Enable or disable extensions # See https://doc.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # ‘scrapy.extensions.telnet.TelnetConsole‘: None, #} # Configure item pipelines # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { ‘Amazon.pipelines.AmazonPipeline‘: 300, } # Enable and configure the AutoThrottle extension (disabled by default) # See https://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = ‘httpcache‘ #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = ‘scrapy.extensions.httpcache.FilesystemCacheStorage‘ ###MONGODB的配置 HOST=‘127.0.0.1‘ PORT=27017 USER=‘root‘ PWD=‘‘ DB=‘amazon‘ TABLE=‘goods‘View Code
二.補充一個小技巧
一直在命令行啟動爬蟲文件就很累了,可以這麽做
在爬蟲項目的根目錄直接寫一個.py文件,加入如下內容
#第一個,第二不變,第三個是爬蟲文件名稱,也可以加第四個,--nolog不達意你日誌
from scrapy.cmdline import execute execute([‘scrapy‘, ‘crawl‘, ‘amazon‘])
scrapy框架的另一種分頁處理以及mongodb的持久化儲存以及from_crawler類方法的使用