CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
List of all members | Public Member Functions | Public Attributes
uploadConditions.HTTP Class Reference
Inheritance diagram for uploadConditions.HTTP:

Public Member Functions

def __init__ (self)
 
def discardCookies (self)
 
def getCookies (self)
 
def getToken (self, username, password)
 
def query (self, url, data=None, files=None, keepCookies=True)
 
def setBaseUrl (self, baseUrl='')
 
def setProxy (self, proxy='')
 
def setRetries (self, retries=())
 
def setTimeout (self, timeout=0)
 

Public Attributes

 baseUrl
 
 curl
 
 retries
 
 token
 

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 220 of file uploadConditions.py.

Constructor & Destructor Documentation

◆ __init__()

def uploadConditions.HTTP.__init__ (   self)

Definition at line 226 of file uploadConditions.py.

References upload_popcon.HTTP.setBaseUrl(), uploadConditions.HTTP.setBaseUrl(), upload_popcon.HTTP.setRetries(), and uploadConditions.HTTP.setRetries().

226  def __init__(self):
227  self.setBaseUrl()
228  self.setRetries()
229 
230  self.curl = pycurl.Curl()
231  self.curl.setopt(self.curl.COOKIEFILE, '') # in memory
232 
233  #-toDo: make sure we have the right options set here to use ssl
234  #-review(2015-09-25): check and see - action: AP
235  # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
236  self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
237  self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
238 
239  self.baseUrl = None
240 
241  self.token = None
242 
def __init__(self, dataset, job_number, job_id, job_name, isDA, isMC, applyBOWS, applyEXTRACOND, extraconditions, runboundary, lumilist, intlumi, maxevents, gt, allFromGT, alignmentDB, alignmentTAG, apeDB, apeTAG, bowDB, bowTAG, vertextype, tracktype, refittertype, ttrhtype, applyruncontrol, ptcut, CMSSW_dir, the_dir)

Member Function Documentation

◆ discardCookies()

def uploadConditions.HTTP.discardCookies (   self)
Discards cookies.

Definition at line 248 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

Referenced by uploadConditions.HTTP.query().

248  def discardCookies(self):
249  '''Discards cookies.
250  '''
251  self.curl.setopt(self.curl.COOKIELIST, 'ALL')
252 
253 

◆ getCookies()

def uploadConditions.HTTP.getCookies (   self)
Returns the list of cookies.

Definition at line 243 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

243  def getCookies(self):
244  '''Returns the list of cookies.
245  '''
246  return self.curl.getinfo(self.curl.INFO_COOKIELIST)
247 

◆ getToken()

def uploadConditions.HTTP.getToken (   self,
  username,
  password 
)

Definition at line 288 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, upload_popcon.HTTP.curl, uploadConditions.HTTP.curl, edm.decode(), str, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

288  def getToken(self, username, password):
289 
290  url = self.baseUrl + 'token'
291 
292  self.curl.setopt(pycurl.URL, url)
293  self.curl.setopt(pycurl.VERBOSE, 0)
294 
295  #-toDo: check if/why these are needed ...
296  #-ap: hmm ...
297  # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
298  # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
299  #-end hmmm ...
300  #-review(2015-09-25): check and see - action: AP
301 
302  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
303  # self.curl.setopt( self.curl.POST, {})
304  self.curl.setopt(self.curl.HTTPGET, 0)
305 
306  response = io.BytesIO()
307  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
308  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
309  logging.debug('going to connect to server at: %s' % url )
310 
311  self.curl.perform()
312  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
313  logging.debug('got: %s ', str(code))
314  if code in ( 502,503,504 ):
315  logging.debug('Trying again after %d seconds...', waitForRetry)
316  time.sleep( waitForRetry )
317  response = io.StringIO()
318  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
319  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
320  self.curl.perform()
321  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
322  resp = response.getvalue().decode('UTF-8')
323  errorMsg = None
324  if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
325  logging.error("Invalid credentials provided.")
326  return None
327  if code==403 and not resp.find("Unauthorized access")==-1:
328  logging.error("Unauthorized access. Please check the membership of group 'cms-cond-dropbox'")
329  return None
330  if code==200:
331  try:
332  self.token = json.loads( resp )['token']
333  except Exception as e:
334  errorMsg = 'Error while decoding returned json string'
335  logging.debug('http::getToken> error while decoding json: %s ', str(resp) )
336  logging.debug("error getting token: %s", str(e))
337  resp = None
338  else:
339  errorMsg = 'HTTP Error code %s ' %code
340  logging.debug('got: %s ', str(code))
341  logging.debug('http::getToken> got error from server: %s ', str(resp) )
342  resp = None
343  if resp is None:
344  raise Exception(errorMsg)
345 
346  logging.debug('token: %s', self.token)
347  logging.debug('returning: %s', response.getvalue().decode('UTF-8'))
348 
349  return response.getvalue()
350 
bool decode(bool &, std::string_view)
Definition: types.cc:72
#define str(s)
def getToken(db, tag, since)

◆ query()

def uploadConditions.HTTP.query (   self,
  url,
  data = None,
  files = None,
  keepCookies = True 
)
Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 351 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, upload_popcon.HTTP.curl, uploadConditions.HTTP.curl, upload_popcon.HTTP.discardCookies(), uploadConditions.HTTP.discardCookies(), upload_popcon.HTTP.retries, uploadConditions.HTTP.retries, str, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

351  def query(self, url, data = None, files = None, keepCookies = True):
352  '''Queries a URL, optionally with some data (dictionary).
353 
354  If no data is specified, a GET request will be used.
355  If some data is specified, a POST request will be used.
356 
357  If files is specified, it must be a dictionary like data but
358  the values are filenames.
359 
360  By default, cookies are kept in-between requests.
361 
362  A HTTPError exception is raised if the response's HTTP code is not 200.
363  '''
364 
365  if not keepCookies:
366  self.discardCookies()
367 
368  url = self.baseUrl + url
369 
370  # make sure the logs are safe ... at least somewhat :)
371  data4log = copy.copy(data)
372  if data4log:
373  if 'password' in data4log.keys():
374  data4log['password'] = '*'
375 
376  retries = [0] + list(self.retries)
377 
378  while True:
379  logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
380 
381  time.sleep(retries.pop(0))
382 
383  try:
384  self.curl.setopt(self.curl.URL, url)
385  self.curl.setopt(self.curl.HTTPGET, 1)
386 
387  # from now on we use the token we got from the login
388  self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
389  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
390 
391  if data is not None or files is not None:
392  # If there is data or files to send, use a POST request
393 
394  finalData = {}
395 
396  if data is not None:
397  finalData.update(data)
398 
399  if files is not None:
400  for (key, fileName) in files.items():
401  finalData[key] = (self.curl.FORM_FILE, fileName)
402  self.curl.setopt( self.curl.HTTPPOST, list(finalData.items()) )
403 
404  self.curl.setopt(pycurl.VERBOSE, 0)
405 
406  response = io.BytesIO()
407  self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
408  self.curl.perform()
409 
410  code = self.curl.getinfo(self.curl.RESPONSE_CODE)
411 
412  if code in self.retryCodes and len(retries) > 0:
413  logging.debug('Retrying since we got the %s error code...', code)
414  continue
415 
416  if code != 200:
417  raise HTTPError(code, response.getvalue())
418 
419  return response.getvalue()
420 
421  except pycurl.error as e:
422  if len(retries) == 0:
423  raise e
424  logging.debug('Retrying since we got the %s pycurl exception...', str(e))
425 
426 # common/http.py end
427 
#define str(s)

◆ setBaseUrl()

def uploadConditions.HTTP.setBaseUrl (   self,
  baseUrl = '' 
)
Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 254 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, and uploadConditions.HTTP.baseUrl.

Referenced by uploadConditions.HTTP.__init__().

254  def setBaseUrl(self, baseUrl = ''):
255  '''Allows to set a base URL which will be prefixed to all the URLs
256  that will be queried later.
257  '''
258  self.baseUrl = baseUrl
259 
260 

◆ setProxy()

def uploadConditions.HTTP.setProxy (   self,
  proxy = '' 
)
Allows to set a proxy.

Definition at line 261 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

261  def setProxy(self, proxy = ''):
262  '''Allows to set a proxy.
263  '''
264  self.curl.setopt(self.curl.PROXY, proxy)
265 
266 

◆ setRetries()

def uploadConditions.HTTP.setRetries (   self,
  retries = () 
)
Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 273 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.__init__().

273  def setRetries(self, retries = ()):
274  '''Allows to set retries.
275 
276  The retries are a sequence of the seconds to wait per retry.
277 
278  The retries are done on:
279  * PyCurl errors (includes network problems, e.g. not being able
280  to connect to the host).
281  * 502 Bad Gateway (for the moment, to avoid temporary
282  Apache-CherryPy issues).
283  * 503 Service Temporarily Unavailable (for when we update
284  the frontends).
285  '''
286  self.retries = retries
287 

◆ setTimeout()

def uploadConditions.HTTP.setTimeout (   self,
  timeout = 0 
)
Allows to set a timeout.

Definition at line 267 of file uploadConditions.py.

References upload_popcon.HTTP.curl, and uploadConditions.HTTP.curl.

267  def setTimeout(self, timeout = 0):
268  '''Allows to set a timeout.
269  '''
270  self.curl.setopt(self.curl.TIMEOUT, timeout)
271 
272 

Member Data Documentation

◆ baseUrl

uploadConditions.HTTP.baseUrl

◆ curl

uploadConditions.HTTP.curl

◆ retries

uploadConditions.HTTP.retries

Definition at line 286 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

◆ token

uploadConditions.HTTP.token