python爬蟲系列(2.2-requests庫的高階使用)
一、設定代理ip
1、直接在請求的時候加上proxies就可以,注意我們一般會寫上http和https的,這樣當遇到http請求就會走http字典對應的代理
2、具體程式碼
import requests
if __name__ == "__main__":
# 定義一個請求頭(模擬瀏覽器)
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'}
# 定義代理的字典(可以去http://www.xicidaili.com查詢最新的代理)
proxies = {
'http': 'http://115.223.223.29:9000',
'https': 'https://197.232.21.141:59075'
}
response = requests.get('http://httpbin.org/get', headers=headers, proxies=proxies)
response.encoding = 'utf-8'
print(response.text)
1、從網站上獲取cookie
import requests
if __name__ == "__main__":
# 定義一個請求頭(模擬瀏覽器)
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'}
response = requests.get('https://www.baidu.com', headers=headers)
# 迭代出全部的
for k, v in response.cookies.items():
print(k, '===', v)
print(response.cookies)
print(response.cookies.get_dict())
2、使用session會話儲存當前網上的cookies
import requests
if __name__ == "__main__":
url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',
'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
}
data = {
'first': 'true',
'pn': '1',
'kd': 'python',
}
# 使用session包裝下
session = requests.session()
response = session.post(url=url, headers=headers, data=data)
print(response.json())
1、只需要在請求的時候加上verify=False就可以
2、具體程式碼
import requests
if __name__ == "__main__":
# 定義一個請求頭(模擬瀏覽器)
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'}
response = requests.get('http://www.12306.cn/mormhweb/', headers=headers, verify=False)
response.encoding = 'utf-8'
print(response.text)