lianjiaspider.py

import asyncio
import aiohttp
import pandas as pd
from lxml import etree


class LianjiaSpider(object):

    def __init__(self):
        self._headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36"}
        self._data = list()

    async def get(self, url):
        async with aiohttp.ClientSession() as session:
            try:
                async with session.get(url, headers=self._headers, timeout=3) as resp:
                    if resp.status == 200:
                        result = await resp.text()
                        return result
            except Exception as e:
                print(e.args)

    async def parse_html(self):
        for page in range(1, 77):
            url = "https://sjz.lianjia.com/zufang/pg{}/".format(page)
            print("正在爬取{}".format(url))
            html = await self.get(url)  # 獲取網頁內容
            html = etree.HTML(html)
            self.parse_page(html)
            print("正在儲存資料....")
            data = pd.DataFrame(self._data)
            data.to_csv("lianjia.csv", encoding='utf_8_sig')  # 寫入檔案

    def parse_page(self, html):
        info_panel = html.xpath("//div[@class='info-panel']")
        for info in info_panel:
            region = info.xpath(".//span[@class='region']/text()")
            zone = info.xpath(".//span[@class='zone']/span/text()")
            meters = info.xpath(".//span[@class='meters']/text()")
            where = info.xpath(".//div[@class='where']/span[4]/text()")

            con = info.xpath(".//div[@class='con']/text()")
            floor = con[0]  # 樓層
            type = con[1]  # 樣式

            agent = info.xpath(".//div[@class='con']/a/text()")[0]

            has = info.xpath(".//div[@class='left agency']//text()")

            price = info.xpath(".//div[@class='price']/span/text()")[0]
            price_pre = info.xpath(".//div[@class='price-pre']/text()")[0]
            look_num = info.xpath(".//div[@class='square']//span[@class='num']/text()")[0]

            one_data = {
                "region": region,
                "zone": zone,
                "meters": meters,
                "where": where,
                "louceng": floor,
                "type": type,
                "xiaoshou": agent,
                "has": has,
                "price": price,
                "price_pre": price_pre,
                "num": look_num
            }
            self._data.append(one_data)  # 新增資料

    def run(self):
        loop = asyncio.get_event_loop()
        tasks = [asyncio.ensure_future(self.parse_html())]
        loop.run_until_complete(asyncio.wait(tasks))


if __name__ == '__main__':
    Lian_jia = LianjiaSpider()
    Lian_jia.run()
 

lianjia.csv

  ,has,louceng,meters,num,price,price_pre,region,type,where,xiaoshou,zone
0,['集中供暖'],高樓層(共33層),['127.86平米\xa0\xa0'],0,2300,2018.11.08 更新,['鳳凰城梧桐苑\xa0\xa0'],板樓,['南'],南焦租房,['3室2廳\xa0\xa0']
1,['集中供暖'],中樓層(共6層),['55平米\xa0\xa0'],0,1200,2018.11.04 更新,['華興小區\xa0\xa0'],板樓,['南'],世紀公園租房,['1室1廳\xa0\xa0']
2,['集中供暖'],中樓層(共6層),['138平米\xa0\xa0'],0,2400,2018.11.04 更新,['河冶小區\xa0\xa0'],板樓,['南 北'],躍進租房,['3室2廳\xa0\xa0']
3,['集中供暖'],低樓層(共6層),['90平米\xa0\xa0'],1,1500,2018.11.06 更新,['瑞國花園\xa0\xa0'],板樓,['南'],躍進租房,['2室2廳\xa0\xa0']
4,['集中供暖'],低樓層(共14層),['180平米\xa0\xa0'],0,3500,2018.11.13 更新,['華脈新村\xa0\xa0'],板樓,['南 北'],四十中學租房,['4室2廳\xa0\xa0']
5,"['近地鐵', '隨時看房', '集中供暖']",中樓層(共40層),['57平米\xa0\xa0'],0,3000,2018.11.09 更新,['華潤大廈\xa0\xa0'],塔樓,['西'],南長租房,['1室1廳\xa0\xa0']
6,"['近地鐵', '隨時看房', '集中供暖']",中樓層(共40層),['42.56平米\xa0\xa0'],0,2200,2018.11.09 更新,['華潤大廈\xa0\xa0'],塔樓,['南'],南長租房,['1室1廳\xa0\xa0']
7,['集中供暖'],中樓層(共34層),['148平米\xa0\xa0'],0,2500,2018.11.08 更新,['北城國際B區\xa0\xa0'],板樓,['南 北'],沿東租房,['3室2廳\xa0\xa0']
8,"['近地鐵', '隨時看房', '集中供暖']",中樓層(共40層),['40.09平米\xa0\xa0'],0,2100,2018.11.09 更新,['華潤大廈\xa0\xa0'],塔樓,['南'],南長租房,['1室1廳\xa0\xa0']
9,"['近地鐵', '集中供暖']",低樓層(共33層),['185平米\xa0\xa0'],0,22000,2018.11.10 更新,['青鳥中山華府\xa0\xa0'],板樓,['北'],大經租房,['1室1廳\xa0\xa0']
10,"['近地鐵', '集中供暖']",低樓層(共33層),['242平米\xa0\xa0'],0,29000,2018.11.05 更新,['青鳥中山華府\xa0\xa0'],板樓,['北'],大經租房,['1室1廳\xa0\xa0']
 

去除無用字元

import re

f = open("lian_jia.csv", 'w', encoding='utf-8')
filename = 'lianjia.csv'
with open(filename, 'r', encoding='utf-8')as file:
    frd = file.readlines()
    for i in frd:
        pattern = re.compile(r'xa0')
        out = re.sub(pattern, '', i)
        s = "".join("".join("".join(out.split("\\\\")).split("']")).split("['"))
        d = "".join(s.split("平米"))
        f.write(d)
 

lian_jia.csv

 ,has,louceng,meters,num,price,price_pre,region,type,where,xiaoshou,zone
0,集中供暖,高樓層(共33層),127.86,0,2300,2018.11.08 更新,鳳凰城梧桐苑,板樓,南,南焦租房,3室2廳
1,集中供暖,中樓層(共6層),55,0,1200,2018.11.04 更新,華興小區,板樓,南,世紀公園租房,1室1廳
2,集中供暖,中樓層(共6層),138,0,2400,2018.11.04 更新,河冶小區,板樓,南 北,躍進租房,3室2廳
3,集中供暖,低樓層(共6層),90,1,1500,2018.11.06 更新,瑞國花園,板樓,南,躍進租房,2室2廳
4,集中供暖,低樓層(共14層),180,0,3500,2018.11.13 更新,華脈新村,板樓,南 北,四十中學租房,4室2廳
5,"近地鐵', '隨時看房', '集中供暖",中樓層(共40層),57,0,3000,2018.11.09 更新,華潤大廈,塔樓,西,南長租房,1室1廳
6,"近地鐵', '隨時看房', '集中供暖",中樓層(共40層),42.56,0,2200,2018.11.09 更新,華潤大廈,塔樓,南,南長租房,1室1廳
7,集中供暖,中樓層(共34層),148,0,2500,2018.11.08 更新,北城國際B區,板樓,南 北,沿東租房,3室2廳
8,"近地鐵', '隨時看房', '集中供暖",中樓層(共40層),40.09,0,2100,2018.11.09 更新,華潤大廈,塔樓,南,南長租房,1室1廳
9,"近地鐵', '集中供暖",低樓層(共33層),185,0,22000,2018.11.10 更新,青鳥中山華府,板樓,北,大經租房,1室1廳
10,"近地鐵', '集中供暖",低樓層(共33層),242,0,29000,2018.11.05 更新,青鳥中山華府,板樓,北,大經租房,1室1廳

視覺化戶型數量分佈

import pandas as pd
import matplotlib.pyplot as plt

house = pd.read_csv('lian_jia.csv', names=['', 'has', 'louceng', 'meters', 'num', 'price', 'price_pre', 'region', 'type', 'where','xiaoshou', 'zone'])
zone = house['zone'].value_counts()
plt.rcParams['font.sans-serif'] = ['FangSong']  # 指定預設字型
asd, sdf = plt.subplots(1, 1, dpi=200)  # 設定畫布
zone.head(10).plot(kind='bar', x='zone', y='size', title='戶型數量分佈', ax=sdf)  # 獲取前10條資料
plt.legend(['數量'])
plt.show()

柱狀圖