JiwuspiderSpider.py

# -*- coding: utf-8 -*-
from scrapy import Spider,Request
import re
from jiwu.items import JiwuItem
class JiwuspiderSpider(Spider):
 name = "jiwuspider"
 allowed_domains = ["wlmq.jiwu.com"]
 start_urls = ['http://wlmq.jiwu.com/loupan']
 def parse(self, response):
 """
 解析每一頁房屋的list
 :param response: 
 :return: 
 """
 for url in response.xpath('//a[@class="index_scale"]/@href').extract():
 yield Request(url,self.parse_html) # 取list集合中的url 呼叫詳情解析方法
 # 如果下一頁屬性還存在，則把下一頁的url獲取出來
 nextpage = response.xpath('//a[@class="tg-rownum-next index-icon"]/@href').extract_first()
 #判斷是否為空
 if nextpage:
 yield Request(nextpage,self.parse) #回撥自己繼續解析
 def parse_html(self,response):
 """
 解析每一個房產資訊的詳情頁面，生成item
 :param response: 
 :return: 
 """
 pattern = re.compile('<script type="text/javascript">.*?lng = '(.*?)';.*?lat = '(.*?)';.*?bname = '(.*?)';.*?'
 'address = '(.*?)';.*?price = '(.*?)';',re.S)
 item = JiwuItem()
 results = re.findall(pattern,response.text)
 for result in results:
 item['name'] = result[2]
 item['address'] = result[3]
 # 對價格判斷只取數字，如果為空就設定為0
 pricestr =result[4]
 pattern2 = re.compile('(d+)')
 s = re.findall(pattern2,pricestr)
 if len(s) == 0:
 item['price'] = 0
 else:item['price'] = s[0]
 item['lng'] = result[0]
 item['lat'] = result[1]
 yield item

私信菜鳥007獲取此案例原始碼！

item.py

# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class JiwuItem(scrapy.Item):
 # define the fields for your item here like:
 name = scrapy.Field()
 price =scrapy.Field()
 address =scrapy.Field()
 lng = scrapy.Field()
 lat = scrapy.Field()
 pass

pipelines.py 注意此處是吧mongodb的儲存方法註釋了，可以自選選擇儲存方式

# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo
from scrapy.conf import settings
from openpyxl import workbook
class JiwuPipeline(object):
 wb = workbook.Workbook()
 ws = wb.active
 ws.append(['小區名稱', '地址', '價格', '經度', '緯度'])
 def __init__(self):
 # 獲取資料庫連線資訊
 host = settings['MONGODB_URL']
 port = settings['MONGODB_PORT']
 dbname = settings['MONGODB_DBNAME']
 client = pymongo.MongoClient(host=host, port=port)
 # 定義資料庫
 db = client[dbname]
 self.table = db[settings['MONGODB_TABLE']]
 def process_item(self, item, spider):
 jiwu = dict(item)
 #self.table.insert(jiwu)
 line = [item['name'], item['address'], str(item['price']), item['lng'], item['lat']]
 self.ws.append(line)
 self.wb.save('jiwu.xlsx')
 return item

最後報表的資料

加群：960410445  即可回去數十套！

mongodb資料庫

地圖報表效果圖：BDP分享儀表盤，分享視覺化效果