簡要介紹：

使用python3 環境，需要自己安裝的包有requests  （網址請求，獲取頁面資訊）和  Lxml（頁面解析，資訊提取） 。

首先確定從何處獲取 “IP”，本人此次爬取的是 西刺網的免費IP代理。

大概流程：

1.  請求有免費IP的網址（本次使用“http://www.xicidaili.com/nn/”）
2.  獲取網址的頁面資訊
3.  從已經獲得的頁面資訊中，提取有用的資料（關於代理IP的）。
4.  對爬取的IP資料進行篩選，驗證IP 的是否可用（**西刺網上的免費IP並不是全部一定可用**）
5.  將有用的IP資訊（IP地址及埠號）進行儲存，寫入文

import requests
from lxml import etree

#代理IP的資訊儲存
def write_proxy(proxies):
    print(proxies)
    for proxy in proxies:
        with open("ip_proxy.txt", 'a+') as f:
            print("正在寫入：", proxy)
            f.write(proxy + '\n')
    print("錄入完成！！！")



#解析網頁，並得到網頁中的代理IP
def get_proxy(html):
    #對獲取的頁面進行解析
    selector = etree.HTML(html)
    #print(selector.xpath("//title/text()"))
    proxies = []
    #資訊提取
    for each in selector.xpath("//tr[@class='odd']"):
        #ip.append(each[0])
        ip = each.xpath("./td[2]/text()")[0]
        port = each.xpath("./td[3]/text()")[0]
        proxy = ip + ":" + port

        proxies.append(proxy)
    print(len(proxies))
    test_proxies(proxies)



#驗證已得到IP的可用性，本段程式碼通過訪問百度網址，返回的response狀態碼判斷（是否可用）。
def test_proxies(proxies):
    proxies = proxies
    url= "http://www.baidu.com/"
    header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
    }
    normal_proxies=[]
    count = 1
    for proxy in proxies:
        print("第%s個。。" % count)
        count+=1
        try:
            response=requests.get(url,headers=header,proxies= {"http":proxy},timeout= 1)
            if response.status_code == 200:
                print("該代理IP可用：",proxy)
                normal_proxies.append(proxy)
            else:
                print("該代理IP不可用：",proxy)
        except Exception:
            print("該代理IP無效：", proxy)
            pass
    #print(normal_proxies)
    write_proxy(normal_proxies)


def get_html(url):
    header = {
        "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
    }
    response = requests.get(
        url,
        headers=header,
    )
    #print(response.text)
    get_proxy(response.text)


if __name__ == "__main__":

    url = "http://www.xicidaili.com/nn/"
    get_html(url)
 

多執行緒模式：

           因為驗證IP是否可用，程式是序列（故此只能驗證一個結束後，才進行驗證下一個，**效率極低**。）

流程還是同上。

import requests
import threading
from lxml import etree



#解析網頁，並得到網頁中的IP代理
def get_proxy(html):
    selector = etree.HTML(html)
    #print(selector.xpath("//title/text()"))
    proxies = []

    for each in selector.xpath("//tr[@class='odd']"):
        #ip.append(each[0])
        ip = each.xpath("./td[2]/text()")[0]
        port = each.xpath("./td[3]/text()")[0]
        #拼接IP地址，埠號
        proxy = ip + ":" + port
        proxies.append(proxy)
    print(len(proxies))
    test_proxies(proxies)


def thread_write_proxy(proxy):
    with open("./ip_proxy.txt", 'a+') as f:
        print("正在寫入：", proxy)
        f.write(proxy + '\n')
        print("錄入完成！！！")


#新增執行緒模式
def thread_test_proxy(proxy):
    url = "http://www.baidu.com/"
    header = {
        "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
    }
    try:
        response = requests.get(
            url, headers=header, proxies={"http": proxy}, timeout=1)
        if response.status_code == 200:
            print("該代理IP可用：", proxy)
            #normal_proxies.append(proxy)
            thread_write_proxy(proxy)
        else:
            print("該代理IP不可用：", proxy)
    except Exception:
        print("該代理IP無效：", proxy)
        pass



#驗證已得到IP的可用性
def test_proxies(proxies):
    proxies = proxies
    #print("test_proxies函式開始執行。。。\n", proxies)
    for proxy in proxies:
        test = threading.Thread(target=thread_test_proxy, args=(proxy, ))
        test.start()



def get_html(url):
    header = {
        "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
    }
    response = requests.get(
        url,
        headers=header,
    )
    #print(response.text)
    get_proxy(response.text)


if __name__ == "__main__":

    url = "http://www.xicidaili.com/nn/"
    get_html(url)