1
2
3
4 """
5 This module provides classes for issuing HTTP requests.
6
7 @author: Johannes Schwenk
8 @copyright: 2010, Johannes Schwenk
9 @version: 1.0
10 @date: 2010-09-15
11
12
13 """
14
15 import sys
16
17
18 reload(sys)
19 sys.setdefaultencoding('utf-8')
20
21 import urllib
22 import socket
23
24 from urllib2 import Request, urlopen, URLError, HTTPError
25 from httplib import BadStatusLine, InvalidURL, HTTPConnection
26
27 from log import BaseLogger
28 from config import USER_AGENT, REQUEST_TIMEOUT
29
30
31
32 socket.setdefaulttimeout(REQUEST_TIMEOUT)
33
34
36 """
37 Base class that adds the function L{_requestURL} for requesting a URL
38 over the HTTP protocol. It uses urllib2 and httplib to do so.
39
40 """
41
42
49
50
52 """
53 Queries the given URL with the given HTTP headers and optional data.
54
55 If C{data} is not C{None}, the request method will be C{PUT},
56 C{GET} otherwise.
57
58 If exceptions are thrown, they will be logged and the function returns
59 None. Otherwise the servers response is returned.
60
61 @param url: The URL to be requested from the server.
62 @param data: Optional data to be sent to the server.
63 @param headers: Optional headers to be sent with the request. Defaults
64 to C{User-Agent:} L{USER_AGENT} .
65
66 @type url: string
67 @type data: dict or sequence of two-element tuples
68 @type headers: dict
69
70 """
71 if data:
72 url = u'%s?%s' % (url, urllib.urlencode(data))
73 req = Request(url, None, headers)
74
75 """
76 This logs every HTTP request anywhere in the data acquisition
77 process, since every plugin has this class as a ancestor...
78 """
79 self.logger.debug(u'%s | %s | %s' % \
80 (req.get_method(), req.get_full_url(), req.headers))
81 try:
82 response = urlopen(req)
83 except HTTPError, e:
84 self.logger.exception(u'Server could not fulfill the request '
85 u'for "%s", reason: %s' % (url, e.code))
86 except URLError, e:
87 self.logger.exception(u'Failed to reach the server for '
88 u'"%s", reason: %s' % (url, e.reason))
89 except socket.error, e:
90 self.logger.exception(u'Socket error for "%s": %s' % (url, e))
91 except BadStatusLine, e:
92 self.logger.exception(u'Bad status line for "%s": %s' % (url, e))
93 except InvalidURL, e:
94 self.logger.exception(u'InvalidURL "%s": %s' % (url, e))
95 else:
96 return response
97 return None
98