Scrapy原始碼分析（四）：請求Request
阿新 • • 發佈：2019-02-13
本次我們要分析的Scrapy原始碼為Request模組，模組的位置：
from scrapytest.http import Request首先把Request的原始碼附上：
class Request(object_ref): def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, dont_filter=False, errback=None): self._encoding = encoding # this one has to be set first 設定了request使用的編碼 self.method = str(method).upper() #GET POST等等 self._set_url(url) #呼叫URL處理函式，設定url屬性 self._set_body(body) #設定Body屬性 assert isinstance(priority, int), "Request priority not an integer: %r" % priority #檢查priority是否int self.priority = priority assert callback or not errback, "Cannot use errback without a callback" self.callback = callback self.errback = errback self.cookies = cookies or {} self.headers = Headers(headers or {}, encoding=encoding) self.dont_filter = dont_filter self._meta = dict(meta) if meta else None @property #將meta作為一個屬性 def meta(self): if self._meta is None: self._meta = {} return self._meta def _get_url(self): return self._url def _set_url(self, url): #設定url，輸入要字元型的 if not isinstance(url, six.string_types): raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) s = safe_url_string(url, self.encoding) #Urlencode self._url = escape_ajax(s) #處理!# if ':' not in self._url: #要有http://或者https:// raise ValueError('Missing scheme in request url: %s' % self._url) url = property(_get_url, obsolete_setter(_set_url, 'url')) #將url設定為一個只讀屬性，修改url要通過.replace實現 def _get_body(self): return self._body def _set_body(self, body): if body is None: self._body = b'' #b''在py2.7中與普通字串型一致，為了相容py3 else: self._body = to_bytes(body, self.encoding) #將unicode型轉為utf-8等等byte型別 body = property(_get_body, obsolete_setter(_set_body, 'body')) #body也是隻讀屬性，修改要通過replace @property def encoding(self): return self._encoding def __str__(self): return "<%s %s>" % (self.method, self.url) __repr__ = __str__ def copy(self): """Return a copy of this Request""" return self.replace() def replace(self, *args, **kwargs): #建立一個新的Request物件，對於新給的kwargs，使用新的，沒給的使用原有物件的 """Create a new Request with the same attributes except for those given new values. """ for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'encoding', 'priority', 'dont_filter', 'callback', 'errback']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs)
下面我們來逐個函式分析：
主要做的工作是變數賦值，來看_set_url和_set_body兩個函式：def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, dont_filter=False, errback=None): self._encoding = encoding # this one has to be set first 設定了request使用的編碼 self.method = str(method).upper() #GET POST等等 self._set_url(url) #呼叫URL處理函式，設定url屬性 self._set_body(body) #設定Body屬性 assert isinstance(priority, int), "Request priority not an integer: %r" % priority #檢查priority是否int self.priority = priority assert callback or not errback, "Cannot use errback without a callback" self.callback = callback self.errback = errback self.cookies = cookies or {} self.headers = Headers(headers or {}, encoding=encoding) self.dont_filter = dont_filter self._meta = dict(meta) if meta else None
def _set_url(self, url): #設定url，輸入要字元型的 if not isinstance(url, six.string_types): raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) s = safe_url_string(url, self.encoding) #Urlencode self._url = escape_ajax(s) #處理!# if ':' not in self._url: #要有http://或者https:// raise ValueError('Missing scheme in request url: %s' % self._url) def _set_body(self, body): if body is None: self._body = b'' #b''在py2.7中與普通字串型一致，為了相容py3 else: self._body = to_bytes(body, self.encoding) #將unicode型轉為utf-8等等byte型別
six.string_types包含unicode型和str型（也就是py3的b''）
再來看一下用到的幾個工具類函式safe_url_string，to_bytes，escape_ajax
def safe_url_string(url, encoding='utf8'):
"""Convert the given url into a legal URL by escaping unsafe characters
according to RFC-3986.
If a unicode url is given, it is first converted to str using the given
encoding (which defaults to 'utf-8'). When passing a encoding, you should
use the encoding of the original page (the page from which the url was
extracted from).
Calling this function on an already "safe" url will return the url
unmodified.
Always returns a str.
"""
s = unicode_to_str(url, encoding)
return moves.urllib.parse.quote(s, _safe_chars)
def escape_ajax(url):
"""
Return the crawleable url according to:
http://code.google.com/web/ajaxcrawling/docs/getting-started.html
>>> escape_ajax("www.example.com/ajax.html#!key=value")
'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
>>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value")
'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key%3Dvalue'
>>> escape_ajax("www.example.com/ajax.html?#!key=value")
'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
>>> escape_ajax("www.example.com/ajax.html#!")
'www.example.com/ajax.html?_escaped_fragment_='
URLs that are not "AJAX crawlable" (according to Google) returned as-is:
>>> escape_ajax("www.example.com/ajax.html#key=value")
'www.example.com/ajax.html#key=value'
>>> escape_ajax("www.example.com/ajax.html#")
'www.example.com/ajax.html#'
>>> escape_ajax("www.example.com/ajax.html")
'www.example.com/ajax.html'
"""
defrag, frag = urldefrag(url)
if not frag.startswith('!'):
return url
return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
def to_bytes(text, encoding=None, errors='strict'):
"""Return the binary representation of `text`. If `text`
is already a bytes object, return it as-is."""
if isinstance(text, bytes):
return text
if not isinstance(text, six.string_types):
raise TypeError('to_bytes must receive a unicode, str or bytes '
'object, got %s' % type(text).__name__)
if encoding is None:
encoding = 'utf-8'
return text.encode(encoding, errors)safe_url_string是用來給含有非法字元的url做urlencode，
to_bytes將字串統一成byte型，
escape_ajax處理url中含有#!的情況。
再來看一下Header模組：
class Headers(CaselessDict):#
"""Case insensitive http headers dictionary"""
def __init__(self, seq=None, encoding='utf-8'):
self.encoding = encoding
super(Headers, self).__init__(seq)
def normkey(self, key):
"""Normalize key to bytes"""
return self._tobytes(key.title())
def normvalue(self, value):
"""Normalize values to bytes"""
if value is None:
value = []
elif isinstance(value, (six.text_type, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
return [self._tobytes(x) for x in value]
def _tobytes(self, x):
if isinstance(x, bytes):
return x
elif isinstance(x, six.text_type):
return x.encode(self.encoding)
elif isinstance(x, int):
return six.text_type(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
def __getitem__(self, key):
try:
return super(Headers, self).__getitem__(key)[-1]
except IndexError:
return None
def get(self, key, def_val=None):
try:
return super(Headers, self).get(key, def_val)[-1]
except IndexError:
return None
def getlist(self, key, def_val=None):
try:
return super(Headers, self).__getitem__(key)
except KeyError:
if def_val is not None:
return self.normvalue(def_val)
return []
def setlist(self, key, list_):
self[key] = list_
def setlistdefault(self, key, default_list=()):
return self.setdefault(key, default_list)
def appendlist(self, key, value):
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lst
def items(self):
return list(self.iteritems())
def iteritems(self):
return ((k, self.getlist(k)) for k in self.keys())
def values(self):
return [self[k] for k in self.keys()]
def to_string(self):
return headers_dict_to_raw(self)
def to_unicode_dict(self):
""" Return headers as a CaselessDict with unicode keys
and unicode values. Multiple values are joined with ','.
"""
return CaselessDict(
(to_unicode(key, encoding=self.encoding),
to_unicode(b','.join(value), encoding=self.encoding))
for key, value in self.items())
def __copy__(self):
return self.__class__(self)
copy = __copy__基類是CaselessDict，是一個不區分key大小寫的字典：
class CaselessDict(dict): #key大小寫不敏感的字典，key統一按照小寫儲存和查詢
__slots__ = ()
def __init__(self, seq=None):
super(CaselessDict, self).__init__()
if seq:
self.update(seq)
def __getitem__(self, key):
return dict.__getitem__(self, self.normkey(key))
def __setitem__(self, key, value):
dict.__setitem__(self, self.normkey(key), self.normvalue(value))
def __delitem__(self, key):
dict.__delitem__(self, self.normkey(key))
def __contains__(self, key):
return dict.__contains__(self, self.normkey(key))
has_key = __contains__
def __copy__(self):
return self.__class__(self)
copy = __copy__
def normkey(self, key):
"""Method to normalize dictionary key access"""
return key.lower()
def normvalue(self, value):
"""Method to normalize values prior to be setted"""
return value
def get(self, key, def_val=None):
return dict.get(self, self.normkey(key), self.normvalue(def_val))
def setdefault(self, key, def_val=None):
return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
def update(self, seq):
seq = seq.items() if isinstance(seq, dict) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super(CaselessDict, self).update(iseq)
@classmethod
def fromkeys(cls, keys, value=None):
return cls((k, value) for k in keys)
def pop(self, key, *args):
return dict.pop(self, self.normkey(key), *args)主要是定義了normkey，normvalue，使得對字典的訪問（get、set等等）要經過這兩個函式，從而處理了key，不區分大小寫。
再來看一下Header中的函式：
def normkey(self, key):
"""Normalize key to bytes"""
return self._tobytes(key.title())
def normvalue(self, value):
"""Normalize values to bytes"""
if value is None:
value = []
elif isinstance(value, (six.text_type, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
return [self._tobytes(x) for x in value]將key定義為byte型，首字母大寫。value使用list包裝，字串都是用byte型。value使用list是為了可以一個key對應多個值。
def __getitem__(self, key):
try:
return super(Headers, self).__getitem__(key)[-1]
except IndexError:
return None
def get(self, key, def_val=None):
try:
return super(Headers, self).get(key, def_val)[-1]
except IndexError:
return None
def getlist(self, key, def_val=None):
try:
return super(Headers, self).__getitem__(key)
except KeyError:
if def_val is not None:
return self.normvalue(def_val)
return []定義的get操作都是返回list的最後一個元素，getlist直接返回整個list。
def setlist(self, key, list_):
self[key] = list_
def setlistdefault(self, key, default_list=()):
return self.setdefault(key, default_list)
def appendlist(self, key, value):
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lstappendlist可以接受一個list型別的引數。
def to_string(self):
return headers_dict_to_raw(self)
def to_unicode_dict(self):
""" Return headers as a CaselessDict with unicode keys
and unicode values. Multiple values are joined with ','.
"""
return CaselessDict(
(to_unicode(key, encoding=self.encoding),
to_unicode(b','.join(value), encoding=self.encoding))
for key, value in self.items())
to_string函式將字典轉換為字串：
def headers_dict_to_raw(headers_dict):
r"""
Returns a raw HTTP headers representation of headers
For example:
>>> import w3lib.http
>>> w3lib.http.headers_dict_to_raw({b'Content-type': b'text/html', b'Accept': b'gzip'}) # doctest: +SKIP
'Content-type: text/html\\r\\nAccept: gzip'
>>>
Note that keys and values must be bytes.
Argument is ``None`` (returns ``None``):
>>> w3lib.http.headers_dict_to_raw(None)
>>>
"""
if headers_dict is None:
return None
raw_lines = []
for key, value in headers_dict.items():
if isinstance(value, bytes):
raw_lines.append(b": ".join([key, value]))
elif isinstance(value, (list, tuple)):
for v in value:
raw_lines.append(b": ".join([key, v]))
return b'\r\n'.join(raw_lines)list中的元素被拆開成多行顯示。
to_unicode_dict將header轉換為一個unicode型別的CaselessDict。
回到Request來：
url = property(_get_url, obsolete_setter(_set_url, 'url')) #將url設定為一個只讀屬性，修改url要通過.replace實現
body = property(_get_body, obsolete_setter(_set_body, 'body')) #body也是隻讀屬性，修改要通過replaceurl和body被設定為只讀屬性，修改會報錯。
def obsolete_setter(setter, attrname):
def newsetter(self, value):
c = self.__class__.__name__
msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c)
raise AttributeError(msg)
return newsetter def replace(self, *args, **kwargs): #建立一個新的Request物件，對於新給的kwargs，使用新的，沒給的使用原有物件的
"""Create a new Request with the same attributes except for those
given new values.
"""
for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta',
'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)