設定 user -agent 和  開啟（注意路徑）

DOWNLOADER_MIDDLEWARES = {
   #'news.middlewares.NewsDownloaderMiddleware': 543,
   'news.middlewares.UserAgentMiddleware': 300,
}

#隨機換user-agent
class UserAgentMiddleware(object):
    """
        給每一個請求隨機切換一個User-Agent
    """
    def process_request(self, request, spider):
        user_agent = random.choice(us_list)
        request.headers['User-Agent'] = user_agent
注意這樣可以列印

request.headers['User-Agent']