handlers.py
1.22 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
from gzip import GzipFile
from io import BytesIO
import urllib.request
import urllib.parse
import urllib.response
class UserAgentProcessor(urllib.request.BaseHandler):
"""A handler to add a custom UA string to urllib2 requests
"""
def __init__(self, uastring):
self.handler_order = 100
self.ua = uastring
def http_request(self, request):
request.add_header("User-Agent", self.ua)
return request
https_request = http_request
class GZipProcessor(urllib.request.BaseHandler):
"""
A handler to add gzip capabilities to urllib2 requests
http://techknack.net/python-urllib2-handlers/
"""
def http_request(self, req):
req.add_header("Accept-Encoding", "gzip")
return req
https_request = http_request
def http_response(self, req, resp):
if resp.headers.get("content-encoding") == "gzip":
body = resp.read()
gz = GzipFile(
fileobj=BytesIO(body),
mode="r"
)
old_resp = resp
resp = urllib.response.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
return resp
https_response = http_response