# 導入程序所需要的庫。
import requests
from bs4 import BeautifulSoup
import time

# 加入請求頭偽裝成瀏覽器
headers = {
    #通過Chrome瀏覽器復制User-Agent
    ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36‘
}


# 定義判斷用戶性別的函數
def judgment_sex(class_name):
    if class_name == [‘member_ico1‘]:
        return ‘女‘
    else:
        return ‘男‘


# 獲取詳細頁URL函數
def get_links(url):
    try:
        wb_date = requests.get(url, headers)
    except ConnectionAbortedError:
        print(‘拒絕連接‘)
    soup = BeautifulSoup(wb_date.text, ‘lxml‘)
    links = soup.select(‘#page_list > ul > li > a‘)
    for link in links:
        herf = link.get("href")
        get_info(herf)


# 獲取網頁信息函數
def get_info(url):
    wb_date = requests.get(url, headers)
    soup = BeautifulSoup(wb_date.text, ‘lxml‘)
    #通過瀏覽器copy selector
    tittles = soup.select(‘div.pho_info > h4‘)
    addresses = soup.select(‘span.pr5‘)
    prises = soup.select(‘#pricePart > div.day_l > span‘)
    images = soup.select(‘#floatRightBox > div.js_box.clearfix > div.member_pic > a > img‘)
    names = soup.select(‘#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a‘)
    sexs = soup.select(‘#floatRightBox > div.js_box.clearfix > div.member_pic > div‘)
    for tittle, address, prise, image, name, sex in zip(tittles, addresses, prises, images, names, sexs):
        date = {
            ‘tittle‘: tittle.get_text().strip(),
            ‘address‘: address.get_text().strip(),
            ‘price‘: prise.get_text(),
            ‘image‘: image.get("src"),
            ‘name‘: name.get_text(),
            ‘sex‘: judgment_sex(sex.get("class"))
        }
        print(date)


if __name__ == ‘__main__‘:

    urls = [‘http://bj.xiaozhu.com/search-duanzufang-p{}-0/‘.format(number) for number in range(1, 14)]
    for single_url in urls:
        get_links(single_url)
        #休眠十秒，防止被封IP
        time.sleep(10)
        
        #缺點：缺少IP管理，采用休眠方法，效率低

我的第一個爬蟲，爬取北京地區短租房信息