最近用Scrapy做爬蟲的時候碰到資料分佈在多個頁面，要發去多次請求才能收集到足夠的資訊，例如列表只有簡單的幾個資訊，更多的資訊在內頁。檢視官方文件沒找到相關的案例或說明，這個有點坑。

最後自己查了寫資料，研究後一下，終於整出來了。

yield scrapy.Request(item['url'], meta={'item': item}, callback=self.detail_parse)

多少級的請求的資料都可以收集。

程式碼演示如下：

spider.py

# -*- coding: utf-8 -*-
import scrapy
from Tencent.items import TencentItem


class TencentSpider(scrapy.Spider):
    # 爬蟲名稱
    name = 'tencent'
    # 允許爬取的域名
    allowed_domains = ['www.xxx.com']
    # 爬蟲基礎地址 用於爬蟲域名的拼接
    base_url = 'https://www.xxx.com/'
    # 爬蟲入口爬取地址
    start_urls = ['https://www.xxx.com/position.php']
    # 爬蟲爬取頁數控制初始值
    count = 1
    # 爬蟲爬取頁數 10為只爬取一頁
    page_end = 1

    def parse(self, response):


        nodeList = response.xpath("//table[@class='tablelist']/tr[@class='odd'] | //table[@class='tablelist']/tr[@class='even']")
        for node in nodeList:
            item = TencentItem()

            item['title'] = node.xpath("./td[1]/a/text()").extract()[0]
            if len(node.xpath("./td[2]/text()")):
                item['position'] = node.xpath("./td[2]/text()").extract()[0]
            else:
                item['position'] = ''
            item['num'] = node.xpath("./td[3]/text()").extract()[0]
            item['address'] = node.xpath("./td[4]/text()").extract()[0]
            item['time'] = node.xpath("./td[5]/text()").extract()[0]
            item['url'] = self.base_url + node.xpath("./td[1]/a/@href").extract()[0]
            # 根據內頁地址爬取
            yield scrapy.Request(item['url'], meta={'item': item}, callback=self.detail_parse)

            # 有下級頁面爬取 註釋掉資料返回
            # yield item

        # 迴圈爬取翻頁
        nextPage = response.xpath("//a[@id='next']/@href").extract()[0]
        # 爬取頁數控制及末頁控制
        if self.count < self.page_end and nextPage != 'javascript:;':
            if nextPage is not None:
                # 爬取頁數控制值自增
                self.count = self.count + 1
                # 翻頁請求
                yield scrapy.Request(self.base_url + nextPage, callback=self.parse)
        else:
            # 爬蟲結束
            return None

    def detail_parse(self, response):
        # 接收上級已爬取的資料
        item = response.meta['item']   

        #一級內頁資料提取 
        item['zhize'] = response.xpath("//*[@id='position_detail']/div/table/tr[3]/td/ul[1]").xpath('string(.)').extract()[0]
        item['yaoqiu'] = response.xpath("//*[@id='position_detail']/div/table/tr[4]/td/ul[1]").xpath('string(.)').extract()[0]

        # 二級內頁地址爬取
        yield scrapy.Request(item['url'] + "&123", meta={'item': item}, callback=self.detail_parse2)

        # 有下級頁面爬取 註釋掉資料返回
        # return item

    def detail_parse2(self, response):
        # 接收上級已爬取的資料
        item = response.meta['item']

        # 二級內頁資料提取 
        item['test'] = "111111111111111111"

        # 最終返回資料給爬蟲引擎
        return item
 

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TencentItem(scrapy.Item):
    # define the fields for your item here like:
    # 職位名稱
    title = scrapy.Field()
    # 職位類別
    position = scrapy.Field()
    # 招聘人數
    num = scrapy.Field()
    # 工作地點
    address = scrapy.Field()
    # 釋出時間
    time = scrapy.Field()
    # 詳情連結
    url = scrapy.Field()
    # 工作職責
    zhize = scrapy.Field()
    # 工作要求
    yaoqiu = scrapy.Field()
    # 測試
    test = scrapy.Field()