1. 獲取頁面信息，用XPath 做數據提取

2. 獲取每個blog裏的用標題、正文、閱讀次數信息


3. 保存到 json 文件內

# -*- coding:utf-8 -*-

import urllib
import urllib2
from lxml import etree

def loadPage(url):
    """
        作用：根據url發送請求，獲取服務器響應文件
        url: 需要爬取的url地址
    """
    #print url
    #headers = {"User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"}
 


    request = urllib2.Request(url)
    html = urllib2.urlopen(request).read()
    # 解析HTML文檔為HTML DOM模型
    content = etree.HTML(html)
    #print content
    # 返回所有匹配成功的列表集合
    link_list = content.xpath(‘//div[@class="postTitle"]/a/@href‘)
    for link in link_list:
        
        #print link
        loadpage(link)

 
# 取出每個文章的鏈接
def loadpage(link):
    headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"}
    request = urllib2.Request(link, headers = headers)
    html = urllib2.urlopen(request).read()
    # 解析
    content = etree.HTML(html)
    
 
# 取出每層發送的文章鏈接集合
    link_list = content.xpath(‘//div[@class="post"]//a‘)[0].text
    print link_list
    # 取出每個標題，正文，閱讀的連接
    #for link in link_list:
    #   print link
    #   write(link)

def blogSpider(url, beginPage, endPage):
    """
        作用：貼吧爬蟲調度器，負責組合處理每個頁面的url
        url : 貼吧url的前部分
        beginPage : 起始頁
        endPage : 結束頁
    """
    for page in range(beginPage, endPage + 1):
        pn = page
        fullurl = url + str(pn)
        #print fullurl
        loadPage(fullurl)
        #print html

        print "謝謝使用"

if __name__ == "__main__":
    beginPage = int(raw_input("請輸入起始頁："))
    endPage = int(raw_input("請輸入結束頁："))

    url = "http://www.cnblogs.com/wanglinjie/default.html?page="
    blogSpider(url, beginPage, endPage)

待完。。。

爬取【王琳傑-博客園】的博文