最近突然想學下爬蟲爬取一下豆瓣的圖書，按類別來爬取並分別儲存，然後就用正則寫了一份初級爬蟲，目前只是分類的頁面爬取，後面完善一下，希望能夠得到每本書的isbn編號，生成自己的資料庫。

# -*- coding:utf-8 -*-
import urllib2
import re
import sys

tags = [u'小說', u'散文', u'歷史', u'愛情', u'管理', u'程式設計', u'生活', u'心理']
haveBooked = set()


class BookSpider:
    def __init__(self):
        reload(sys)
        sys.setdefaultencoding('utf-8')
        self.start = 0
        self.tagIndex = 0
        self.param = '&filter=&type='
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64)'}
        self.filePath = 'DoubanTop250.txt'

    def GetPage(self):
        try:
            URL = 'https://book.douban.com/tag/' + tags[self.tagIndex]
            request = urllib2.Request(url=URL, headers=self.headers)
            response = urllib2.urlopen(request)
            page = response.read().decode('utf-8')
            self.start += 20
            pageNum = self.start / 20
            print '正在抓取 ' + tags[self.tagIndex] + ' 的第' + str(pageNum) + '頁資料'
            return page
        except urllib2.URLError, e:
            if hasattr(e, 'reason'):
                print '抓取失敗，具體原因：', e.reason

    def WriteBookTitle(self, titleInfo, fileBook):
        patternTitle = re.compile(u'(.*?)<span.*?>(.*?)</span>.*?', re.S)
        result = re.match(patternTitle, titleInfo)
        if result is None:
            fileBook.write('title: ' + titleInfo + '\r\n')
        else:
            titles = re.findall(patternTitle, titleInfo)
            fileBook.write('title: ' + titles[0][0].strip() + titles[0][1].strip() + '\r\n')

    def WriteBookPubInfo(self, pubInfo, fileBook):
        pubInfo += '!'
        patternPub1 = re.compile(u'(.*?)/(.*?)/(.*?)/(.*?)/(.*?)!', re.S)
        patternPub2 = re.compile(u'(.*?)/(.*?)/(.*?)/(.*?)!', re.S)
        result = re.match(patternPub1, pubInfo)
        if result is None:
            pubs = re.findall(patternPub2, pubInfo)
            fileBook.write('author: ' + pubs[0][0].strip() + '\r\n')
            fileBook.write('pubHouse: ' + pubs[0][1].strip() + '\r\n')
            fileBook.write('pubData: ' + pubs[0][2].strip() + '\r\n')
            fileBook.write('price: ' + pubs[0][3].strip() + '\r\n\r\n')
        else:
            pubs = re.findall(patternPub1, pubInfo)
            fileBook.write('author: ' + pubs[0][0].strip() + '\r\n')
            fileBook.write('translator: ' + pubs[0][1].strip() + '\r\n')
            fileBook.write('pubHouse: ' + pubs[0][2].strip() + '\r\n')
            fileBook.write('pubData: ' + pubs[0][3].strip() + '\r\n')
            fileBook.write('price: ' + pubs[0][4].strip() + '\r\n\r\n')

    def GetBook(self):
    	pattern = re.compile(u'<li.*?class="subject-item">.*?<div.*?class="info">.*?'
            + u'<h2.*?class="">.*?<a.*?subject_id:\'(.*?)\'.*?>(.*?)</a>.*?<div.*?class="pub">'
            + u'(.*?)</div>', re.S)
        try:
            while self.tagIndex < 2:
                fileName = "book" + tags[self.tagIndex] + ".txt"
                fileName.decode("utf-8").encode("gb2312")
                print fileName
                fileBook = open(fileName, 'w')
                while self.start < 1:
                    page = self.GetPage()
                    books = re.findall(pattern, page)
                    for book in books:
                        fileBook.write('subject_id: ' + book[0].strip() + '\r\n')
                        self.WriteBookTitle(book[1].strip(), fileBook)
                        self.WriteBookPubInfo(book[2].strip(), fileBook)
                    self.start += 1
                fileBook.close()
                self.start = 0
                self.tagIndex += 1
        except:
            print '抓取 ' + tags[self.tagIndex] + ' 第 ' + self.start / 20 + 1 + ' 頁失敗'

    def main(self):
        print '開始抓取圖書資料'
        self.GetBook()
        print '抓取完畢...'

DoubanSpoder = BookSpider()
DoubanSpoder.main()