upload_popcon.HTTP Class Reference

Inheritance diagram for upload_popcon.HTTP:

Public Member Functions
def	__init__ (self)
def	discardCookies (self)
def	getCookies (self)
def	getToken (self, username, password)
def	query (self, url, data=None, files=None, keepCookies=True)
def	setBaseUrl (self, baseUrl='')
def	setProxy (self, proxy='')
def	setRetries (self, retries=())
def	setTimeout (self, timeout=0)

Public Attributes
	baseUrl
	curl
	retries
	token

Static Public Attributes
	retryCodes = frozenset([502, 503])

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 64 of file upload_popcon.py.

Constructor & Destructor Documentation

__init__()

def upload_popcon.HTTP.__init__

(

self

)

Definition at line 70 of file upload_popcon.py.

References upload_popcon.HTTP.setBaseUrl(), and upload_popcon.HTTP.setRetries().

    def __init__(self):
        self.setBaseUrl()
        self.setRetries()

        self.curl = pycurl.Curl()
        self.curl.setopt(self.curl.COOKIEFILE, '')      # in memory

        #-toDo: make sure we have the right options set here to use ssl
        #-review(2015-09-25): check and see - action: AP
        # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
        self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
        self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)

        self.baseUrl = None

        self.token = None

Member Function Documentation

discardCookies()

def upload_popcon.HTTP.discardCookies

(

self

)

Discards cookies.

Definition at line 92 of file upload_popcon.py.

References upload_popcon.HTTP.curl.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

    def discardCookies(self):
        '''Discards cookies.
        '''
        self.curl.setopt(self.curl.COOKIELIST, 'ALL')

getCookies()

def upload_popcon.HTTP.getCookies

(

self

)

Returns the list of cookies.

Definition at line 87 of file upload_popcon.py.

References upload_popcon.HTTP.curl.

    def getCookies(self):
        '''Returns the list of cookies.
        '''
        return self.curl.getinfo(self.curl.INFO_COOKIELIST)

getToken()

def upload_popcon.HTTP.getToken	(		self,
			username,
			password
	)

Definition at line 132 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, upload_popcon.HTTP.curl, str, and upload_popcon.HTTP.token.

    def getToken(self, username, password):

        url = self.baseUrl + 'token'

        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.VERBOSE, 0)

        #-toDo: check if/why these are needed ...
        #-ap: hmm ...
        # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
        # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
        #-end hmmm ...
        #-review(2015-09-25): check and see - action: AP


        self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
        # self.curl.setopt( self.curl.POST, {})
        self.curl.setopt(self.curl.HTTPGET, 0)

        response = cStringIO.StringIO()
        self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
        self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )

        logging.debug('going to connect to server at: %s' % url )

        self.curl.perform()
        code = self.curl.getinfo(pycurl.RESPONSE_CODE)
        logging.debug('got: %s ', str(code))
        
        try:
            self.token = json.loads( response.getvalue() )['token']
        except Exception as e:
            logging.error('http::getToken> got error from server: %s ', str(e) )
            if 'No JSON object could be decoded' in str(e):
                return None
            logging.error("error getting token: %s", str(e))
            return None

        logging.debug('token: %s', self.token)
        logging.debug('returning: %s', response.getvalue())

        return response.getvalue()

query()

def upload_popcon.HTTP.query	(		self,
			url,
			data = None,
			files = None,
			keepCookies = True
	)

Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 175 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, upload_popcon.HTTP.curl, upload_popcon.HTTP.discardCookies(), upload_popcon.HTTP.retries, upload_popcon.HTTP.retryCodes, str, and upload_popcon.HTTP.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

    def query(self, url, data = None, files = None, keepCookies = True):
        '''Queries a URL, optionally with some data (dictionary).

        If no data is specified, a GET request will be used.
        If some data is specified, a POST request will be used.

        If files is specified, it must be a dictionary like data but
        the values are filenames.

        By default, cookies are kept in-between requests.

        A HTTPError exception is raised if the response's HTTP code is not 200.
        '''

        if not keepCookies:
            self.discardCookies()

        url = self.baseUrl + url

        # make sure the logs are safe ... at least somewhat :)
        data4log = copy.copy(data)
        if data4log:
            if 'password' in data4log.keys():
                data4log['password'] = '*'

        retries = [0] + list(self.retries)

        while True:
            logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])

            time.sleep(retries.pop(0))

            try:
                self.curl.setopt(self.curl.URL, url)
                self.curl.setopt(self.curl.HTTPGET, 1)

                # from now on we use the token we got from the login
                self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
                self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])

                if data is not None or files is not None:
                    # If there is data or files to send, use a POST request

                    finalData = {}

                    if data is not None:
                        finalData.update(data)

                    if files is not None:
                        for (key, fileName) in files.items():
                            finalData[key] = (self.curl.FORM_FILE, fileName)
                    self.curl.setopt( self.curl.HTTPPOST, finalData.items() )

                self.curl.setopt(pycurl.VERBOSE, 0)

                response = cStringIO.StringIO()
                self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
                self.curl.perform()

                code = self.curl.getinfo(self.curl.RESPONSE_CODE)

                if code in self.retryCodes and len(retries) > 0:
                    logging.debug('Retrying since we got the %s error code...', code)
                    continue

                if code != 200:
                    raise HTTPError(code, response.getvalue())

                return response.getvalue()

            except pycurl.error as e:
                if len(retries) == 0:
                    raise e
                logging.debug('Retrying since we got the %s pycurl exception...', str(e))

# common/http.py end

setBaseUrl()

def upload_popcon.HTTP.setBaseUrl	(		self,
			baseUrl = ''
	)

Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 98 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

    def setBaseUrl(self, baseUrl = ''):
        '''Allows to set a base URL which will be prefixed to all the URLs
        that will be queried later.
        '''
        self.baseUrl = baseUrl

setProxy()

def upload_popcon.HTTP.setProxy	(		self,
			proxy = ''
	)

Allows to set a proxy.

Definition at line 105 of file upload_popcon.py.

References upload_popcon.HTTP.curl.

    def setProxy(self, proxy = ''):
        '''Allows to set a proxy.
        '''
        self.curl.setopt(self.curl.PROXY, proxy)

setRetries()

def upload_popcon.HTTP.setRetries	(		self,
			retries = ()
	)

Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 117 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

    def setRetries(self, retries = ()):
        '''Allows to set retries.

        The retries are a sequence of the seconds to wait per retry.

        The retries are done on:
            * PyCurl errors (includes network problems, e.g. not being able
              to connect to the host).
            * 502 Bad Gateway (for the moment, to avoid temporary
              Apache-CherryPy issues).
            * 503 Service Temporarily Unavailable (for when we update
              the frontends).
        '''
        self.retries = retries

setTimeout()

def upload_popcon.HTTP.setTimeout	(		self,
			timeout = 0
	)

Allows to set a timeout.

Definition at line 111 of file upload_popcon.py.

References upload_popcon.HTTP.curl.

    def setTimeout(self, timeout = 0):
        '''Allows to set a timeout.
        '''
        self.curl.setopt(self.curl.TIMEOUT, timeout)

Member Data Documentation

baseUrl

upload_popcon.HTTP.baseUrl

Definition at line 83 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.HTTP.setBaseUrl(), and uploadConditions.HTTP.setBaseUrl().

curl

upload_popcon.HTTP.curl

Definition at line 74 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.discardCookies(), uploadConditions.HTTP.discardCookies(), upload_popcon.HTTP.getCookies(), uploadConditions.HTTP.getCookies(), upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.HTTP.setProxy(), uploadConditions.HTTP.setProxy(), upload_popcon.HTTP.setTimeout(), and uploadConditions.HTTP.setTimeout().

retries

upload_popcon.HTTP.retries

Definition at line 130 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

retryCodes

upload_popcon.HTTP.retryCodes = frozenset([502, 503])

static

Definition at line 68 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

token

upload_popcon.HTTP.token

Definition at line 85 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().