代理池設定：

這裡代理ip從快代理那獲取，新使用者有4小時免費測試時間。從http://dps.kdlapi.com/api/getdps/中獲取我們的ip代理池，根據使用者名稱密碼最後生成proxy_auth代理池。

proxy_auth = []

username = "####"
password = "####"

ips = requests.get('http://dps.kdlapi.com/api/getdps/?orderid=#####&num=30&pt=1&format=json&sep=1')

ips_arr = json.loads(ips.text)

for ip in ips_arr['data']['proxy_list']:
    proxy_auth_temp = {"http":  "http://%(user)s:%(pwd)
 
[email protected]%(ip)s/" % {'user': username, 'pwd': password, 'ip': ip},
              "https": "http://%(user)s:%(pwd)[email protected]%(ip)s/" % {'user': username, 'pwd': password, 'ip': ip}}
    proxy_auth.append(proxy_auth_temp)

 美團網抓取鮮花商家資訊：

 在輸入鮮花資訊後，點選搜尋，開啟network，分析我們想要的資料

分析Query String Parameters構成，不難推測各個引數的意思，由於商傢俱體資訊需要點列表的超連結進去二級詳細頁面，所以我們需要獲取到列表頁的超連結資訊。 最後的python程式碼為

    data = {
        'uuid':'2a344b34b62b4666af73.1535625802.1.0.0',
        'userid':-1,
        'limit':32,
        'offset':32*page_num,
        'cateId':-1,
        'q':search,
    }
    headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
            }
    url = 'http://apimobile.meituan.com/group/v4/poi/pcsearch/50?' 
    
    reponse = requests.get(url,params=data,headers=headers)
  
    data = json.loads(reponse.text)
    if data and 'data' in data.keys():
        for item in data.get('data').get('searchResult'):
            url = 'http://www.meituan.com/shenghuo/'+ str(item.get('id'))+'/'
            print(url)
 

最後在商家詳細頁面抓取我們想要的商家資訊：

            headers = {
                'Host': 'www.meituan.com',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
            }
            
            prox = random.choice(proxy_auth)
          
            try:
                detail = requests.get(url, proxies=prox,headers=headers,timeout=10)
            except requests.RequestException as e:
                proxy_auth.remove(prox)
            else:
           
                soup = BeautifulSoup(detail.text,'html.parser')
                end = {
                    'seller_name' : soup.select('.seller-name')[0].get_text().strip(),
                    'address' : soup.select('.seller-info-body .item')[0].get_text().strip(),
                    'mobile' : soup.select('.seller-info-body .item')[1].get_text().strip(),
                    # 'time' : soup.select('.seller-info-body .item')[2].get_text().strip(),
                }
                print(end)
                result.append(end)
                # exit()
                
                urls.append(url)
                time.sleep(3)

 注意點：

     prox = random.choice(proxy_auth) 從我們的代理池中隨機挑選代理。

     在詳情頁使用try捕獲異常，如果產生異常時說明代理失效，從當前代理池中刪除

 完整示例：

from urllib.parse import urlencode 
import requests
import json
from bs4 import BeautifulSoup
import random
import time

urls = []
result = []
proxy_auth = []

username = "#########"
password = "#######"

ips = requests.get('http://dps.kdlapi.com/api/getdps/?orderid=#######&num=30&pt=1&format=json&sep=1')

ips_arr = json.loads(ips.text)

for ip in ips_arr['data']['proxy_list']:
    proxy_auth_temp = {"http":  "http://%(user)s:%(pwd)[email protected]%(ip)s/" % {'user': username, 'pwd': password, 'ip': ip},
              "https": "http://%(user)s:%(pwd)[email protected]%(ip)s/" % {'user': username, 'pwd': password, 'ip': ip}}
    proxy_auth.append(proxy_auth_temp)
    



def getAllUrls(page_num,search):
    
    data = {
        'uuid':'2a344b34b62b4666af73.1535625802.1.0.0',
        'userid':-1,
        'limit':32,
        'offset':32*page_num,
        'cateId':-1,
        'q':search,
    }
    headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
            }
    url = 'http://apimobile.meituan.com/group/v4/poi/pcsearch/50?' 
    
    reponse = requests.get(url,params=data,headers=headers)
  
    data = json.loads(reponse.text)
    if data and 'data' in data.keys():
        for item in data.get('data').get('searchResult'):
            url = 'http://www.meituan.com/shenghuo/'+ str(item.get('id'))+'/'
            print(url)
            headers = {
                'Host': 'www.meituan.com',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
            }
            
            prox = random.choice(proxy_auth)
          
            try:
                detail = requests.get(url, proxies=prox,headers=headers,timeout=10)
            except requests.RequestException as e:
                proxy_auth.remove(prox)
            else:
           
                soup = BeautifulSoup(detail.text,'html.parser')
                end = {
                    'seller_name' : soup.select('.seller-name')[0].get_text().strip(),
                    'address' : soup.select('.seller-info-body .item')[0].get_text().strip(),
                    'mobile' : soup.select('.seller-info-body .item')[1].get_text().strip(),
                    # 'time' : soup.select('.seller-info-body .item')[2].get_text().strip(),
                }
                print(end)
                result.append(end)
                # exit()
                
                urls.append(url)
                time.sleep(3)

def main():
    page_counts = 3 
    search = '鮮花'
    for i in range(0,page_counts):
        total = getAllUrls(i,search)


if __name__ == '__main__':
    
    main()
    # print(json.dumps(result))