Package buildxml :: Package xmlgetter :: Module request
[hide private]
[frames] | no frames]

Source Code for Module buildxml.xmlgetter.request

 1  #!/usr/bin/python 
 2  # -*- coding: utf-8 -*- 
 3   
 4  """ 
 5  This module provides classes for issuing HTTP requests. 
 6   
 7  @author: Johannes Schwenk 
 8  @copyright: 2010, Johannes Schwenk 
 9  @version: 1.0 
10  @date: 2010-09-15 
11   
12   
13  """ 
14   
15  import sys 
16   
17  # Imortant! 
18  reload(sys) 
19  sys.setdefaultencoding('utf-8') 
20   
21  import urllib 
22  import socket 
23   
24  from urllib2 import Request, urlopen, URLError, HTTPError 
25  from httplib import BadStatusLine, InvalidURL, HTTPConnection 
26   
27  from log import BaseLogger 
28  from config import USER_AGENT, REQUEST_TIMEOUT 
29   
30   
31  # timeout for fetching data 
32  socket.setdefaulttimeout(REQUEST_TIMEOUT) 
33   
34   
35 -class BaseRequester(BaseLogger):
36 """ 37 Base class that adds the function L{_requestURL} for requesting a URL 38 over the HTTP protocol. It uses urllib2 and httplib to do so. 39 40 """ 41 42
43 - def __init__(self, source_name):
44 """ 45 Initialize the instance. 46 47 """ 48 BaseLogger.__init__(self, source_name)
49 50
51 - def _requestURL(self, url, data=None, headers={u'User-Agent': USER_AGENT}):
52 """ 53 Queries the given URL with the given HTTP headers and optional data. 54 55 If C{data} is not C{None}, the request method will be C{PUT}, 56 C{GET} otherwise. 57 58 If exceptions are thrown, they will be logged and the function returns 59 None. Otherwise the servers response is returned. 60 61 @param url: The URL to be requested from the server. 62 @param data: Optional data to be sent to the server. 63 @param headers: Optional headers to be sent with the request. Defaults 64 to C{User-Agent:} L{USER_AGENT} . 65 66 @type url: string 67 @type data: dict or sequence of two-element tuples 68 @type headers: dict 69 70 """ 71 if data: 72 url = u'%s?%s' % (url, urllib.urlencode(data)) 73 req = Request(url, None, headers) 74 75 """ 76 This logs every HTTP request anywhere in the data acquisition 77 process, since every plugin has this class as a ancestor... 78 """ 79 self.logger.debug(u'%s | %s | %s' % \ 80 (req.get_method(), req.get_full_url(), req.headers)) 81 try: 82 response = urlopen(req) 83 except HTTPError, e: 84 self.logger.exception(u'Server could not fulfill the request ' 85 u'for "%s", reason: %s' % (url, e.code)) 86 except URLError, e: 87 self.logger.exception(u'Failed to reach the server for ' 88 u'"%s", reason: %s' % (url, e.reason)) 89 except socket.error, e: 90 self.logger.exception(u'Socket error for "%s": %s' % (url, e)) 91 except BadStatusLine, e: 92 self.logger.exception(u'Bad status line for "%s": %s' % (url, e)) 93 except InvalidURL, e: 94 self.logger.exception(u'InvalidURL "%s": %s' % (url, e)) 95 else: 96 return response 97 return None
98