單單隻有地址資訊是沒法在echarts上畫出散點圖的,必須有經度緯度資訊,利用百度地圖API可以將地址資訊解析為經緯度資訊,該API每日可以調取6000次,在申請認證開發者資訊後每日可以調取三十萬次,足夠解析資料了,認證之後按如下操作建立應用


其中應用名稱隨便輸,應用型別選擇服務端,IP白名單選擇你的IP或者如圖中輸入預設所有IP都可以訪問

1.解析方法

通過訪問如下請求,獲得解析結果

http://api.map.baidu.com/geocoder/v2/?address=北京市海淀區上地十街10號&output=json&ak=您的ak&callback=showLocation
 

2.程式碼

使用非同步請求,大大加快解析速度,這裡用的是我比更熟悉的python3的urllib.request庫

#coding=utf-8
import pymongo
from gevent.pool import Pool
import urllib.request
import json
import urllib.parse
import socket
from gevent import monkey
from parseSettings import *
import gevent
import time
import datetime
monkey.patch_all()  #修改標準庫
 

socket.setdefaulttimeout(2) 
class LocationParser(object):
    #用於地址解析的類
    def __init__(self,ak,baseUrl):
        self.ak = ak
        self.baseUrl = baseUrl
    def parse(self,address,output,city=None):
        #通過api獲取json資料
        url = self.baseUrl+"?ak="+self.ak+"&address="+urllib.parse.quote(address)+"&output="
 
+output
        if city:
            url = url + "&city="+urllib.parse.quote(city)
        try:
            result = urllib.request.urlopen(url)
        except:
            print("連線錯誤")
            return None,url
        jsonresult = json.load(result)
        return jsonresult,url

class LocationDumper():
    def __init__(self,ip,port,locationDB,locationCol,sourceDB,sourceCol,requestNum = 10,city = None):
        #資料庫連線
        self.client = pymongo.MongoClient(ip,port)
        #地址解析器
        self.bParser = LocationParser("vxc9SN7XHCI1M6y8djyBD98Upx5CCprX",
                                                                  "http://api.map.baidu.com/geocoder/v2/")
        #併發數量控制
        self.requestNum = requestNum
        #帶解析資料集合
        self.sourceCol = self.client[sourceDB][sourceCol]
        #用於儲存結果的集合
        self.locationCol = self.client[locationDB][locationCol]
        self.oklength = 0
        self.errorlength = 0
    #用於解析每一條資料的方法
    def parseData(self,data):
        #資料集中沒有帶解析的資料,解析該條資料
        if self.locationCol.count({"_id":data["_id"]}) <= 0:
            if self.locationCol.count({"address":data["address"]}) > 0:
                print("同一地址不再解析"+data['address'])
                content = self.locationCol.find_one({"address":data["address"]})
                a = {'_id':data['_id'],
                "size":data['size'],
                "orient":data['orient'],
                "roomNum":data['roomNum'],
                "url":data['fromUrl'],
                "unitPrice":data['unitPrice'],
                "sumPrice":data['sumPrice'],
                "ln":content['ln'],
                "lat":content['lat'],
                "address":data['address'],
                "time":data['nowTime'],
                "city":data['city']}

                self.locationCol.insert(a)
                self.sourceCol.update({"_id":data["_id"]},{"$set":{"status":"OK"}})

                return
            result = self.bParser.parse(data["address"],"json",data["city"])
            #根據返回碼判斷解析結果是否正確,如果不正確,去掉城市引數重試
            if result[0] and result[0]['status'] != 0:
                result = self.bParser.parse(data["address"],"json")
            if not result[0]:
                return
            jsonResult = result[0]
            urlResult = result[1]
            ln = None
            lat = None
            if jsonResult['status'] == 0:
                ln = jsonResult['result']['location']['lng']
                lat = jsonResult['result']['location']['lat']
            else:
                try:
                    print("地址解析錯誤,status:"+str(jsonResult['status'])+",msg:"+jsonResult['msg']+",errorUrl:"+urlResult)
                    self.sourceCol.update({"_id":data["_id"]},{"$set":{"status":"ERROR"}})
                    self.errorlength += 1
                except:
                    print("沒有錯誤資訊,直接輸出資訊"+str(jsonResult))
                return
            try:
                    a = {'_id':data['_id'],
                                "size":data['size'],
                                "orient":data['orient'],
                                "roomNum":data['roomNum'],
                                "url":data['fromUrl'],
                                "unitPrice":data['unitPrice'],
                                "sumPrice":data['sumPrice'],
                                "ln":ln,
                                "lat":lat,
                                "address":data['address'],
                                "time":data['nowTime'],
                                "city":data['city']}    
            except:
                print("儲存資料異常,檢查該條資料sumprice:"+data['sumPrice']+data['address'])
                self.sourceCol.update({"_id":data["_id"]},{"$set":{"status":"ERROR    "}})
                return
            self.locationCol.insert(a)
            self.oklength += 1
            print("成功插入第%s條資料,解析的地址為:%s"%(self.oklength,data['address']))
            self.sourceCol.update({"_id":data['_id']},{"$set":{"status":"OK"}})    
        else:
            self.sourceCol.update({"_id":data['_id']},{"$set":{"status":"OK"}})    
            print("已經存在,不需要解析,狀態設定為以解析")
    #解析main方法
    def parse(self,Number = None):
        pool=gevent.pool.Pool(self.requestNum)
        datas = self.sourceCol.find({"status":"SUBSPENDING"})
        if Number:
            datas = datas.limit(Number)
        for data in datas:
            greenlet = gevent.spawn(self.parseData,data)
            pool.add(greenlet)
        pool.join()
        print("解析完畢,共成功%s條,失敗%s條"%(self.oklength,self.errorlength))
    def __del(self):
        self.client.close()
if __name__ == "__main__":

    starttime = datetime.datetime.now()
    d = LocationDumper(MONGO_IP,
                       MONGO_PORT,
                       LOCATION_DB,
                       LOCATION_COLL,
                       SOURCE_DB,
                       SOURCE_COLL,
                       requestNum = REQUEST_NUM)
    d.parse()
    endtime = datetime.datetime.now()
    print((endtime-starttime).seconds)

解析結果如下