1. 程式人生 > >schedule和CrawlerProcess定時執行多個爬蟲

schedule和CrawlerProcess定時執行多個爬蟲

import smtplib,schedule
# 通過CrawlerProcess同時執行幾個spider
from scrapy.crawler import CrawlerProcess
from spiders.liepin_spider import LiepinSpiderSpider as lie_1
from spiders.liepin_spider_2 import LiepinSpiderSpider as lie_2
from spiders.liepin_spider_3 import LiepinSpiderSpider as lie_3
# 匯入獲取專案配置的模組
from scrapy.utils.project import get_project_settings

def start_spider():
    try:
    #一定要先載入配置檔案
        process = CrawlerProcess(get_project_settings())
        process.crawl(lie_1)
        process.crawl(lie_2)
        process.crawl(lie_3)
        process.start()
        now_time = time.strftime('%Y-%m-%d', time.localtime(time.time()))
        print('---{}---執行完成'.format(now_time))

    except Exception as e:
        print('--出現錯誤--', e)
        #email_spider('liepin', e) #傳送郵件的方法,可以自己寫




def main():
    print('開始檢測,等待時間到達,開始執行')
    schedule.every().day.at("13:04").do(start_spider)
    schedule.every().day.at("17:00").do(start_spider)
    while True:
        schedule.run_pending()
        time.sleep(10)
        

if __name__ == '__main__':
    main()