pyspider爬取豆瓣延禧攻略短評和迴應
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-08-30 15:08:10
# Project: yanxigonglve_douban
from pyspider.libs.base_handler import *
from lxml import etree
class Handler(BaseHandler):
crawl_config = {
}
@every(minutes=24 * 60)
def on_start(self):
self.crawl('https://movie.douban.com/subject/26999852/discussion/', callback=self.index_page,validate_cert=False)
@config(age=10 * 24 * 60 * 60)
def index_page(self, response):
# 利用xpath獲取短評的連結,主要目的是為了得到詳情頁面的迴應
s = etree.HTML(response.text)
href = s.xpath('//*[@id="posts-table"]/tr/td[1]/a/@href')
print(href)
for i in href:
self.crawl(i, callback=self.detail_page,validate_cert=False)
# 獲取下一頁的連結
next = response.doc('.next > a').attr.href
self.crawl(next, callback=self.index_page,validate_cert=False)
@config(priority=2)
def detail_page(self, response):
return {
"url": response.url,
"title": response.doc('title').text(),
'content': response.doc('.post .comment-item').text()
}