uploadConditions.HTTP Class Reference

Inheritance diagram for uploadConditions.HTTP:

Public Member Functions
def	__init__ (self)
def	discardCookies (self)
def	getCookies (self)
def	getToken (self, username, password)
def	query (self, url, data=None, files=None, keepCookies=True)
def	setBaseUrl (self, baseUrl='')
def	setProxy (self, proxy='')
def	setRetries (self, retries=())
def	setTimeout (self, timeout=0)

Public Attributes
	baseUrl
	curl
	retries
	token

Static Public Attributes
	retryCodes = frozenset([502, 503])

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 220 of file uploadConditions.py.

Constructor & Destructor Documentation

__init__()

def uploadConditions.HTTP.__init__

(

self

)

Definition at line 226 of file uploadConditions.py.

References upload_popcon.HTTP.setBaseUrl(), uploadConditions.HTTP.setBaseUrl(), upload_popcon.HTTP.setRetries(), and uploadConditions.HTTP.setRetries().

    def __init__(self):
        self.setBaseUrl()
        self.setRetries()

        self.curl = pycurl.Curl()
        self.curl.setopt(self.curl.COOKIEFILE, '')      # in memory

        #-toDo: make sure we have the right options set here to use ssl
        #-review(2015-09-25): check and see - action: AP
        # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
        self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
        self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)

        self.baseUrl = None

        self.token = None

Member Function Documentation

discardCookies()

def uploadConditions.HTTP.discardCookies

(

self

)

Discards cookies.

Definition at line 248 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

Referenced by uploadConditions.HTTP.query().

    def discardCookies(self):
        '''Discards cookies.
        '''
        self.curl.setopt(self.curl.COOKIELIST, 'ALL')

getCookies()

def uploadConditions.HTTP.getCookies

(

self

)

Returns the list of cookies.

Definition at line 243 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

    def getCookies(self):
        '''Returns the list of cookies.
        '''
        return self.curl.getinfo(self.curl.INFO_COOKIELIST)

getToken()

def uploadConditions.HTTP.getToken	(		self,
			username,
			password
	)

Definition at line 288 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, upload_popcon.HTTP.curl, uploadConditions.HTTP.curl, edm.decode(), str, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

    def getToken(self, username, password):

        url = self.baseUrl + 'token'

        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.VERBOSE, 0)

        #-toDo: check if/why these are needed ...
        #-ap: hmm ...
        # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
        # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
        #-end hmmm ...
        #-review(2015-09-25): check and see - action: AP

        self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
        # self.curl.setopt( self.curl.POST, {})
        self.curl.setopt(self.curl.HTTPGET, 0)

        response = io.BytesIO()
        self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
        self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
        logging.debug('going to connect to server at: %s' % url )

        self.curl.perform()
        code = self.curl.getinfo(pycurl.RESPONSE_CODE)
        logging.debug('got: %s ', str(code))
        if code in ( 502,503,504 ):
            logging.debug('Trying again after %d seconds...', waitForRetry)
            time.sleep( waitForRetry )
            response = io.StringIO()
            self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
            self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
            self.curl.perform()
            code = self.curl.getinfo(pycurl.RESPONSE_CODE)        
        resp = response.getvalue().decode('UTF-8')
        errorMsg = None
        if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
            logging.error("Invalid credentials provided.")
            return None
        if code==403 and not resp.find("Unauthorized access")==-1:
            logging.error("Unauthorized access. Please check the membership of group 'cms-cond-dropbox'")
            return None
        if code==200:
            try:
                self.token = json.loads( resp )['token']
            except Exception as e:
                errorMsg = 'Error while decoding returned json string'
                logging.debug('http::getToken> error while decoding json: %s ', str(resp) )
                logging.debug("error getting token: %s", str(e))
                resp = None
        else:
            errorMsg = 'HTTP Error code %s ' %code
            logging.debug('got: %s ', str(code))
            logging.debug('http::getToken> got error from server: %s ', str(resp) )
            resp = None
        if resp is None:
            raise Exception(errorMsg)
            
        logging.debug('token: %s', self.token)
        logging.debug('returning: %s', response.getvalue().decode('UTF-8'))

        return response.getvalue()

query()

def uploadConditions.HTTP.query	(		self,
			url,
			data = None,
			files = None,
			keepCookies = True
	)

Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 351 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, upload_popcon.HTTP.curl, uploadConditions.HTTP.curl, upload_popcon.HTTP.discardCookies(), uploadConditions.HTTP.discardCookies(), upload_popcon.HTTP.retries, uploadConditions.HTTP.retries, upload_popcon.HTTP.retryCodes, uploadConditions.HTTP.retryCodes, str, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

    def query(self, url, data = None, files = None, keepCookies = True):
        '''Queries a URL, optionally with some data (dictionary).

        If no data is specified, a GET request will be used.
        If some data is specified, a POST request will be used.

        If files is specified, it must be a dictionary like data but
        the values are filenames.

        By default, cookies are kept in-between requests.

        A HTTPError exception is raised if the response's HTTP code is not 200.
        '''

        if not keepCookies:
            self.discardCookies()

        url = self.baseUrl + url

        # make sure the logs are safe ... at least somewhat :)
        data4log = copy.copy(data)
        if data4log:
            if 'password' in data4log.keys():
                data4log['password'] = '*'

        retries = [0] + list(self.retries)

        while True:
            logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])

            time.sleep(retries.pop(0))

            try:
                self.curl.setopt(self.curl.URL, url)
                self.curl.setopt(self.curl.HTTPGET, 1)

                # from now on we use the token we got from the login
                self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
                self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])

                if data is not None or files is not None:
                    # If there is data or files to send, use a POST request

                    finalData = {}

                    if data is not None:
                        finalData.update(data)

                    if files is not None:
                        for (key, fileName) in files.items():
                            finalData[key] = (self.curl.FORM_FILE, fileName)
                    self.curl.setopt( self.curl.HTTPPOST, list(finalData.items()) )

                self.curl.setopt(pycurl.VERBOSE, 0)

                response = io.BytesIO()
                self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
                self.curl.perform()

                code = self.curl.getinfo(self.curl.RESPONSE_CODE)

                if code in self.retryCodes and len(retries) > 0:
                    logging.debug('Retrying since we got the %s error code...', code)
                    continue

                if code != 200:
                    raise HTTPError(code, response.getvalue())

                return response.getvalue()

            except pycurl.error as e:
                if len(retries) == 0:
                    raise e
                logging.debug('Retrying since we got the %s pycurl exception...', str(e))

# common/http.py end

setBaseUrl()

def uploadConditions.HTTP.setBaseUrl	(		self,
			baseUrl = ''
	)

Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 254 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, and uploadConditions.HTTP.baseUrl.

Referenced by uploadConditions.HTTP.__init__().

    def setBaseUrl(self, baseUrl = ''):
        '''Allows to set a base URL which will be prefixed to all the URLs
        that will be queried later.
        '''
        self.baseUrl = baseUrl

setProxy()

def uploadConditions.HTTP.setProxy	(		self,
			proxy = ''
	)

Allows to set a proxy.

Definition at line 261 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

    def setProxy(self, proxy = ''):
        '''Allows to set a proxy.
        '''
        self.curl.setopt(self.curl.PROXY, proxy)

setRetries()

def uploadConditions.HTTP.setRetries	(		self,
			retries = ()
	)

Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 273 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.__init__().

    def setRetries(self, retries = ()):
        '''Allows to set retries.

        The retries are a sequence of the seconds to wait per retry.

        The retries are done on:
            * PyCurl errors (includes network problems, e.g. not being able
              to connect to the host).
            * 502 Bad Gateway (for the moment, to avoid temporary
              Apache-CherryPy issues).
            * 503 Service Temporarily Unavailable (for when we update
              the frontends).
        '''
        self.retries = retries

setTimeout()

def uploadConditions.HTTP.setTimeout	(		self,
			timeout = 0
	)

Allows to set a timeout.

Definition at line 267 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

    def setTimeout(self, timeout = 0):
        '''Allows to set a timeout.
        '''
        self.curl.setopt(self.curl.TIMEOUT, timeout)

Member Data Documentation

baseUrl

uploadConditions.HTTP.baseUrl

Definition at line 239 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.getToken(), uploadConditions.HTTP.query(), and uploadConditions.HTTP.setBaseUrl().

curl

uploadConditions.HTTP.curl

Definition at line 230 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.discardCookies(), uploadConditions.HTTP.getCookies(), uploadConditions.HTTP.getToken(), uploadConditions.HTTP.query(), uploadConditions.HTTP.setProxy(), and uploadConditions.HTTP.setTimeout().

retries

uploadConditions.HTTP.retries

Definition at line 286 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

retryCodes

uploadConditions.HTTP.retryCodes = frozenset([502, 503])

static

Definition at line 224 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

token

uploadConditions.HTTP.token

Definition at line 241 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.getToken(), uploadConditions.HTTP.query(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), and uploadConditions.ConditionsUploader.signOut().