Inheritance diagram for upload_popcon.HTTP:

Public Member Functions
def	__init__ (self)

def	discardCookies (self)

def	getCookies (self)

def	getToken (self, username, password)

def	query (self, url, data=None, files=None, keepCookies=True)

def	setBaseUrl (self, baseUrl='')

def	setProxy (self, proxy='')

def	setRetries (self, retries=())

def	setTimeout (self, timeout=0)

Public Attributes
	baseUrl

	curl

	retries

	token

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 64 of file upload_popcon.py.

Constructor & Destructor Documentation

◆ init()

def upload_popcon.HTTP.__init__ ( self )

Definition at line 70 of file upload_popcon.py.

     def __init__(self):
         self.setBaseUrl()
         self.setRetries()
  
         self.curl = pycurl.Curl()
         self.curl.setopt(self.curl.COOKIEFILE, '')      # in memory
  
         #-toDo: make sure we have the right options set here to use ssl
         #-review(2015-09-25): check and see - action: AP
         # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
         self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
         self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
  
         self.baseUrl = None
  
         self.token = None
  

References upload_popcon.HTTP.setBaseUrl(), and upload_popcon.HTTP.setRetries().

Member Function Documentation

◆ discardCookies()

def upload_popcon.HTTP.discardCookies ( self )

Discards cookies.

Definition at line 92 of file upload_popcon.py.

     def discardCookies(self):
         '''Discards cookies.
         '''
         self.curl.setopt(self.curl.COOKIELIST, 'ALL')
  
  

References upload_popcon.HTTP.curl.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

◆ getCookies()

def upload_popcon.HTTP.getCookies ( self )

Returns the list of cookies.

Definition at line 87 of file upload_popcon.py.

     def getCookies(self):
         '''Returns the list of cookies.
         '''
         return self.curl.getinfo(self.curl.INFO_COOKIELIST)
  

References upload_popcon.HTTP.curl.

◆ getToken()

def upload_popcon.HTTP.getToken	(	self,
		username,
		password
	)

Definition at line 132 of file upload_popcon.py.

     def getToken(self, username, password):
  
         url = self.baseUrl + 'token'
  
         self.curl.setopt(pycurl.URL, url)
         self.curl.setopt(pycurl.VERBOSE, 0)
  
         #-toDo: check if/why these are needed ...
         #-ap: hmm ...
         # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
         # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
         #-end hmmm ...
         #-review(2015-09-25): check and see - action: AP
  
  
         self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
         # self.curl.setopt( self.curl.POST, {})
         self.curl.setopt(self.curl.HTTPGET, 0)
  
         response = cStringIO.StringIO()
         self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
         self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
  
         logging.debug('going to connect to server at: %s' % url )
  
         self.curl.perform()
         code = self.curl.getinfo(pycurl.RESPONSE_CODE)
         logging.debug('got: %s ', str(code))
         
         try:
             self.token = json.loads( response.getvalue() )['token']
         except Exception as e:
             logging.error('http::getToken> got error from server: %s ', str(e) )
             if 'No JSON object could be decoded' in str(e):
                 return None
             logging.error("error getting token: %s", str(e))
             return None
  
         logging.debug('token: %s', self.token)
         logging.debug('returning: %s', response.getvalue())
  
         return response.getvalue()
  

References upload_popcon.HTTP.baseUrl, upload_popcon.HTTP.curl, str, and upload_popcon.HTTP.token.

◆ query()

def upload_popcon.HTTP.query	(	self,
		url,
		data = `None`,
		files = `None`,
		keepCookies = `True`
	)

Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 175 of file upload_popcon.py.

     def query(self, url, data = None, files = None, keepCookies = True):
         '''Queries a URL, optionally with some data (dictionary).
  
         If no data is specified, a GET request will be used.
         If some data is specified, a POST request will be used.
  
         If files is specified, it must be a dictionary like data but
         the values are filenames.
  
         By default, cookies are kept in-between requests.
  
         A HTTPError exception is raised if the response's HTTP code is not 200.
         '''
  
         if not keepCookies:
             self.discardCookies()
  
         url = self.baseUrl + url
  
         # make sure the logs are safe ... at least somewhat :)
         data4log = copy.copy(data)
         if data4log:
             if 'password' in data4log.keys():
                 data4log['password'] = '*'
  
         retries = [0] + list(self.retries)
  
         while True:
             logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
  
             time.sleep(retries.pop(0))
  
             try:
                 self.curl.setopt(self.curl.URL, url)
                 self.curl.setopt(self.curl.HTTPGET, 1)
  
                 # from now on we use the token we got from the login
                 self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
                 self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
  
                 if data is not None or files is not None:
                     # If there is data or files to send, use a POST request
  
                     finalData = {}
  
                     if data is not None:
                         finalData.update(data)
  
                     if files is not None:
                         for (key, fileName) in files.items():
                             finalData[key] = (self.curl.FORM_FILE, fileName)
                     self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
  
                 self.curl.setopt(pycurl.VERBOSE, 0)
  
                 response = cStringIO.StringIO()
                 self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
                 self.curl.perform()
  
                 code = self.curl.getinfo(self.curl.RESPONSE_CODE)
  
                 if code in self.retryCodes and len(retries) > 0:
                     logging.debug('Retrying since we got the %s error code...', code)
                     continue
  
                 if code != 200:
                     raise HTTPError(code, response.getvalue())
  
                 return response.getvalue()
  
             except pycurl.error as e:
                 if len(retries) == 0:
                     raise e
                 logging.debug('Retrying since we got the %s pycurl exception...', str(e))
  
 # common/http.py end
  

References upload_popcon.HTTP.baseUrl, upload_popcon.HTTP.curl, upload_popcon.HTTP.discardCookies(), upload_popcon.HTTP.retries, str, and upload_popcon.HTTP.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

◆ setBaseUrl()

def upload_popcon.HTTP.setBaseUrl	(	self,
		baseUrl = `''`
	)

Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 98 of file upload_popcon.py.

     def setBaseUrl(self, baseUrl = ''):
         '''Allows to set a base URL which will be prefixed to all the URLs
         that will be queried later.
         '''
         self.baseUrl = baseUrl
  
  

References upload_popcon.HTTP.baseUrl.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

◆ setProxy()

def upload_popcon.HTTP.setProxy	(	self,
		proxy = `''`
	)

Allows to set a proxy.

Definition at line 105 of file upload_popcon.py.

     def setProxy(self, proxy = ''):
         '''Allows to set a proxy.
         '''
         self.curl.setopt(self.curl.PROXY, proxy)
  
  

References upload_popcon.HTTP.curl.

◆ setRetries()

def upload_popcon.HTTP.setRetries	(	self,
		retries = `()`
	)

Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 117 of file upload_popcon.py.

     def setRetries(self, retries = ()):
         '''Allows to set retries.
  
         The retries are a sequence of the seconds to wait per retry.
  
         The retries are done on:
             * PyCurl errors (includes network problems, e.g. not being able
               to connect to the host).
             * 502 Bad Gateway (for the moment, to avoid temporary
               Apache-CherryPy issues).
             * 503 Service Temporarily Unavailable (for when we update
               the frontends).
         '''
         self.retries = retries
  

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

◆ setTimeout()

def upload_popcon.HTTP.setTimeout	(	self,
		timeout = `0`
	)

Allows to set a timeout.

Definition at line 111 of file upload_popcon.py.

     def setTimeout(self, timeout = 0):
         '''Allows to set a timeout.
         '''
         self.curl.setopt(self.curl.TIMEOUT, timeout)
  
  

References upload_popcon.HTTP.curl.

Member Data Documentation

◆ baseUrl

upload_popcon.HTTP.baseUrl

Definition at line 83 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.HTTP.setBaseUrl(), and uploadConditions.HTTP.setBaseUrl().

◆ curl

upload_popcon.HTTP.curl

Definition at line 74 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.discardCookies(), uploadConditions.HTTP.discardCookies(), upload_popcon.HTTP.getCookies(), uploadConditions.HTTP.getCookies(), upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.HTTP.setProxy(), uploadConditions.HTTP.setProxy(), upload_popcon.HTTP.setTimeout(), and uploadConditions.HTTP.setTimeout().

◆ retries

upload_popcon.HTTP.retries

Definition at line 130 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

◆ token

upload_popcon.HTTP.token

Definition at line 85 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

Public Member Functions

Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ discardCookies()

◆ getCookies()

◆ getToken()

◆ query()

◆ setBaseUrl()

◆ setProxy()

◆ setRetries()

◆ setTimeout()

Member Data Documentation

◆ baseUrl

◆ curl

◆ retries

◆ token

◆ init()