uploadConditions.HTTP Class Reference

Inheritance diagram for uploadConditions.HTTP:

Public Member Functions
def	__init__ (self)
def	discardCookies (self)
def	getCookies (self)
def	getToken (self, username, password)
def	query (self, url, data=None, files=None, keepCookies=True)
def	setBaseUrl (self, baseUrl='')
def	setProxy (self, proxy='')
def	setRetries (self, retries=())
def	setTimeout (self, timeout=0)

Public Attributes
	baseUrl
	curl
	retries
	token

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 230 of file uploadConditions.py.

Constructor & Destructor Documentation

def uploadConditions.HTTP.__init__

(

self

)

Definition at line 236 of file uploadConditions.py.

References upload_popcon.HTTP.setBaseUrl(), uploadConditions.HTTP.setBaseUrl(), upload_popcon.HTTP.setRetries(), and uploadConditions.HTTP.setRetries().

    def __init__(self):
        self.setBaseUrl()
        self.setRetries()

        self.curl = pycurl.Curl()
        self.curl.setopt(self.curl.COOKIEFILE, '')      # in memory

        #-toDo: make sure we have the right options set here to use ssl
        #-review(2015-09-25): check and see - action: AP
        # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
        self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
        self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)

        self.baseUrl = None

        self.token = None

Member Function Documentation

def uploadConditions.HTTP.discardCookies

(

self

)

Discards cookies.

Definition at line 258 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

    def discardCookies(self):
        '''Discards cookies.
        '''
        self.curl.setopt(self.curl.COOKIELIST, 'ALL')

def uploadConditions.HTTP.getCookies

(

self

)

Returns the list of cookies.

Definition at line 253 of file uploadConditions.py.

    def getCookies(self):
        '''Returns the list of cookies.
        '''
        return self.curl.getinfo(self.curl.INFO_COOKIELIST)

def uploadConditions.HTTP.getToken	(		self,
			username,
			password
	)

Definition at line 298 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, harvestTrackValidationPlots.str, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

    def getToken(self, username, password):

        url = self.baseUrl + 'token'

        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.VERBOSE, 0)

        #-toDo: check if/why these are needed ...
        #-ap: hmm ...
        # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
        # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
        #-end hmmm ...
        #-review(2015-09-25): check and see - action: AP

        self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
        # self.curl.setopt( self.curl.POST, {})
        self.curl.setopt(self.curl.HTTPGET, 0)

        response = cStringIO.StringIO()
        self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
        self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
        logging.debug('going to connect to server at: %s' % url )

        self.curl.perform()
        code = self.curl.getinfo(pycurl.RESPONSE_CODE)
        logging.debug('got: %s ', str(code))
        if code in ( 502,503,504 ):
            logging.debug('Trying again after %d seconds...', waitForRetry)
            time.sleep( waitForRetry )
            response = cStringIO.StringIO()
            self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
            self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
            self.curl.perform()
            code = self.curl.getinfo(pycurl.RESPONSE_CODE)
        resp = response.getvalue()
        errorMsg = None
        if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
            logging.error("Invalid credentials provided.")
            return None
        if code==403 and not resp.find("Unauthorized access")==-1:
            logging.error("Unauthorized access. Please check the membership of group 'cms-cond-dropbox'")
            return None
        if code==200:
            try:
                self.token = json.loads( resp )['token']
            except Exception as e:
                errorMsg = 'Error while decoding returned json string'
                logging.debug('http::getToken> error while decoding json: %s ', str(resp) )
                logging.debug("error getting token: %s", str(e))
                resp = None
        else:
            errorMsg = 'HTTP Error code %s ' %code
            logging.debug('got: %s ', str(code))
            logging.debug('http::getToken> got error from server: %s ', str(resp) )
            resp = None
        if resp is None:
            raise Exception(errorMsg)
            
        logging.debug('token: %s', self.token)
        logging.debug('returning: %s', response.getvalue())

        return resp

def uploadConditions.HTTP.query	(		self,
			url,
			data = None,
			files = None,
			keepCookies = True
	)

Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 361 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, upload_popcon.HTTP.discardCookies(), uploadConditions.HTTP.discardCookies(), list(), upload_popcon.HTTP.retries, uploadConditions.HTTP.retries, harvestTrackValidationPlots.str, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

    def query(self, url, data = None, files = None, keepCookies = True):
        '''Queries a URL, optionally with some data (dictionary).

        If no data is specified, a GET request will be used.
        If some data is specified, a POST request will be used.

        If files is specified, it must be a dictionary like data but
        the values are filenames.

        By default, cookies are kept in-between requests.

        A HTTPError exception is raised if the response's HTTP code is not 200.
        '''

        if not keepCookies:
            self.discardCookies()

        url = self.baseUrl + url

        # make sure the logs are safe ... at least somewhat :)
        data4log = copy.copy(data)
        if data4log:
            if 'password' in data4log.keys():
                data4log['password'] = '*'

        retries = [0] + list(self.retries)

        while True:
            logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])

            time.sleep(retries.pop(0))

            try:
                self.curl.setopt(self.curl.URL, url)
                self.curl.setopt(self.curl.HTTPGET, 1)

                # from now on we use the token we got from the login
                self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
                self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])

                if data is not None or files is not None:
                    # If there is data or files to send, use a POST request

                    finalData = {}

                    if data is not None:
                        finalData.update(data)

                    if files is not None:
                        for (key, fileName) in files.items():
                            finalData[key] = (self.curl.FORM_FILE, fileName)
                    self.curl.setopt( self.curl.HTTPPOST, finalData.items() )

                self.curl.setopt(pycurl.VERBOSE, 0)

                response = cStringIO.StringIO()
                self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
                self.curl.perform()

                code = self.curl.getinfo(self.curl.RESPONSE_CODE)

                if code in self.retryCodes and len(retries) > 0:
                    logging.debug('Retrying since we got the %s error code...', code)
                    continue

                if code != 200:
                    raise HTTPError(code, response.getvalue())

                return response.getvalue()

            except pycurl.error as e:
                if len(retries) == 0:
                    raise e
                logging.debug('Retrying since we got the %s pycurl exception...', str(e))

# common/http.py end

def uploadConditions.HTTP.setBaseUrl	(		self,
			baseUrl = ''
	)

Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 264 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, and uploadConditions.HTTP.baseUrl.

Referenced by uploadConditions.HTTP.__init__().

    def setBaseUrl(self, baseUrl = ''):
        '''Allows to set a base URL which will be prefixed to all the URLs
        that will be queried later.
        '''
        self.baseUrl = baseUrl

def uploadConditions.HTTP.setProxy	(		self,
			proxy = ''
	)

Allows to set a proxy.

Definition at line 271 of file uploadConditions.py.

    def setProxy(self, proxy = ''):
        '''Allows to set a proxy.
        '''
        self.curl.setopt(self.curl.PROXY, proxy)

def uploadConditions.HTTP.setRetries	(		self,
			retries = ()
	)

Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 283 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.__init__().

    def setRetries(self, retries = ()):
        '''Allows to set retries.

        The retries are a sequence of the seconds to wait per retry.

        The retries are done on:
            * PyCurl errors (includes network problems, e.g. not being able
              to connect to the host).
            * 502 Bad Gateway (for the moment, to avoid temporary
              Apache-CherryPy issues).
            * 503 Service Temporarily Unavailable (for when we update
              the frontends).
        '''
        self.retries = retries

def uploadConditions.HTTP.setTimeout	(		self,
			timeout = 0
	)

Allows to set a timeout.

Definition at line 277 of file uploadConditions.py.

    def setTimeout(self, timeout = 0):
        '''Allows to set a timeout.
        '''
        self.curl.setopt(self.curl.TIMEOUT, timeout)

Member Data Documentation

uploadConditions.HTTP.baseUrl

Definition at line 249 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.getToken(), uploadConditions.HTTP.query(), and uploadConditions.HTTP.setBaseUrl().

uploadConditions.HTTP.curl

Definition at line 240 of file uploadConditions.py.

uploadConditions.HTTP.retries

Definition at line 296 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

uploadConditions.HTTP.token

Definition at line 251 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.getToken(), uploadConditions.HTTP.query(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), and uploadConditions.ConditionsUploader.signOut().