1. 程式人生 > >python之爬蟲的入門04------爬蟲代理ip、儲存為CSV表格

python之爬蟲的入門04------爬蟲代理ip、儲存為CSV表格

一、爬蟲偽裝—使用代理ip

import urllib.request
import random

url = 'http://45.32.164.128/ip.php'                                      #URL地址

iplist = ['112.91.159.66:3128','119.31.210.170:7777','221.7.255.168:8080']    #多個代理IP地址

'''
細緻的設定代理,可以用opener的open方法開啟URL:
'''
proxy_support = urllib.request.ProxyHandler({'http':random.choice(iplist)})   # 引數是一個字典{'型別':'代理IP:埠號'}

opener = urllib.request.build_opener(proxy_support)                          # 訂製建立一個opener

urllib.request.install_opener(opener)                                    #安裝opener


response = urllib.request.urlopen(url)
html = response.read().decode('utf8')
print(html)

二、儲存為CSV表格

import csv


def write_msg(data):
    '''這個函式用來將爬取的資料寫入檔案'''
    with open("csvfile.csv", 'a+', newline='') as f:
        wf = csv.writer(f)
        wf.writerow(data)

if __name__ == '__main__':
    head = ['msg', 'area', 'price', 'address', 'user', 'phone']

    with open('./csvfile.csv', 'a+') as f:
        wf = csv.writer(f)
        wf.writerow(head)

    data = ['1', '2', '4', '6', 'g', 'uy']
    write_msg(data)

    data = ['3', '1', 'a', '6', 'g', 'uy']
    write_msg(data)

    data = ['w', 'wdveg546thfwer4335refveergrtbg5', '我的', 'we', 'fvr', 'df43ferrver3442ergfregbvrvebevevevfvervfvr']
    write_msg(data)