CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
uploadConditions.HTTP Class Reference
Inheritance diagram for uploadConditions.HTTP:

Public Member Functions

def __init__
 
def discardCookies
 
def getCookies
 
def getToken
 
def query
 
def setBaseUrl
 
def setProxy
 
def setRetries
 
def setTimeout
 

Public Attributes

 baseUrl
 
 curl
 
 retries
 
 token
 

Static Public Attributes

tuple retryCodes = frozenset([502, 503])
 

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 230 of file uploadConditions.py.

Constructor & Destructor Documentation

def uploadConditions.HTTP.__init__ (   self)

Definition at line 236 of file uploadConditions.py.

References upload_popcon.HTTP.setBaseUrl(), uploadConditions.HTTP.setBaseUrl(), upload_popcon.HTTP.setRetries(), and uploadConditions.HTTP.setRetries().

236 
237  def __init__(self):
238  self.setBaseUrl()
239  self.setRetries()
240 
241  self.curl = pycurl.Curl()
242  self.curl.setopt(self.curl.COOKIEFILE, '') # in memory
243 
244  #-toDo: make sure we have the right options set here to use ssl
245  #-review(2015-09-25): check and see - action: AP
246  # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
247  self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
248  self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
249 
250  self.baseUrl = None
251 
252  self.token = None

Member Function Documentation

def uploadConditions.HTTP.discardCookies (   self) 
Discards cookies.

Definition at line 258 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

258 
259  def discardCookies(self):
260  '''Discards cookies.
261  '''
262  self.curl.setopt(self.curl.COOKIELIST, 'ALL')
263 
def uploadConditions.HTTP.getCookies (   self) 
Returns the list of cookies.

Definition at line 253 of file uploadConditions.py.

253 
254  def getCookies(self):
255  '''Returns the list of cookies.
256  '''
257  return self.curl.getinfo(self.curl.INFO_COOKIELIST)
def uploadConditions.HTTP.getToken (   self,
  username,
  password 
)

Definition at line 298 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

298 
299  def getToken(self, username, password):
300 
301  url = self.baseUrl + 'token'
302 
303  self.curl.setopt(pycurl.URL, url)
304  self.curl.setopt(pycurl.VERBOSE, 0)
305 
306  #-toDo: check if/why these are needed ...
307  #-ap: hmm ...
308  # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
309  # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
310  #-end hmmm ...
311  #-review(2015-09-25): check and see - action: AP
312 
313  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
314  # self.curl.setopt( self.curl.POST, {})
315  self.curl.setopt(self.curl.HTTPGET, 0)
316 
317  response = cStringIO.StringIO()
318  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
319  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
320  logging.debug('going to connect to server at: %s' % url )
321 
322  self.curl.perform()
323  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
324  logging.debug('got: %s ', str(code))
325  if code in ( 502,503,504 ):
326  logging.debug('Trying again after %d seconds...', waitForRetry)
327  time.sleep( waitForRetry )
328  response = cStringIO.StringIO()
329  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
330  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
331  self.curl.perform()
332  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
333  resp = response.getvalue()
334  errorMsg = None
335  if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
336  logging.error("Invalid credentials provided.")
337  return None
338  if code==403 and not resp.find("Unauthorized access")==-1:
339  logging.error("Unauthorized access. Please check the membership of group 'cms-cond-dropbox'")
340  return None
341  if code==200:
342  try:
343  self.token = json.loads( resp )['token']
344  except Exception as e:
345  errorMsg = 'Error while decoding returned json string'
346  logging.debug('http::getToken> error while decoding json: %s ', str(resp) )
347  logging.debug("error getting token: %s", str(e))
348  resp = None
349  else:
350  errorMsg = 'HTTP Error code %s ' %code
351  logging.debug('got: %s ', str(code))
352  logging.debug('http::getToken> got error from server: %s ', str(resp) )
353  resp = None
354  if resp is None:
355  raise Exception(errorMsg)
356 
357  logging.debug('token: %s', self.token)
358  logging.debug('returning: %s', response.getvalue())
359 
360  return resp
def uploadConditions.HTTP.query (   self,
  url,
  data = None,
  files = None,
  keepCookies = True 
) 
Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 361 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, uploadConditions.HTTP.baseUrl, upload_popcon.HTTP.discardCookies(), uploadConditions.HTTP.discardCookies(), list(), upload_popcon.HTTP.retries, uploadConditions.HTTP.retries, upload_popcon.HTTP.retryCodes, uploadConditions.HTTP.retryCodes, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, and upload_popcon.ConditionsUploader.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

361 
362  def query(self, url, data = None, files = None, keepCookies = True):
363  '''Queries a URL, optionally with some data (dictionary).
364 
365  If no data is specified, a GET request will be used.
366  If some data is specified, a POST request will be used.
367 
368  If files is specified, it must be a dictionary like data but
369  the values are filenames.
370 
371  By default, cookies are kept in-between requests.
372 
373  A HTTPError exception is raised if the response's HTTP code is not 200.
374  '''
375 
376  if not keepCookies:
377  self.discardCookies()
378 
379  url = self.baseUrl + url
380 
381  # make sure the logs are safe ... at least somewhat :)
382  data4log = copy.copy(data)
383  if data4log:
384  if 'password' in data4log.keys():
385  data4log['password'] = '*'
386 
387  retries = [0] + list(self.retries)
388 
389  while True:
390  logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
391 
392  time.sleep(retries.pop(0))
393 
394  try:
395  self.curl.setopt(self.curl.URL, url)
396  self.curl.setopt(self.curl.HTTPGET, 1)
397 
398  # from now on we use the token we got from the login
399  self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
400  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
401 
402  if data is not None or files is not None:
403  # If there is data or files to send, use a POST request
404 
405  finalData = {}
406 
407  if data is not None:
408  finalData.update(data)
409 
410  if files is not None:
411  for (key, fileName) in files.items():
412  finalData[key] = (self.curl.FORM_FILE, fileName)
413  self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
414 
415  self.curl.setopt(pycurl.VERBOSE, 0)
416 
417  response = cStringIO.StringIO()
418  self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
419  self.curl.perform()
420 
421  code = self.curl.getinfo(self.curl.RESPONSE_CODE)
422 
423  if code in self.retryCodes and len(retries) > 0:
424  logging.debug('Retrying since we got the %s error code...', code)
425  continue
426 
427  if code != 200:
428  raise HTTPError(code, response.getvalue())
429 
430  return response.getvalue()
431 
432  except pycurl.error as e:
433  if len(retries) == 0:
434  raise e
435  logging.debug('Retrying since we got the %s pycurl exception...', str(e))
436 
437 # common/http.py end
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def uploadConditions.HTTP.setBaseUrl (   self,
  baseUrl = '' 
) 
Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 264 of file uploadConditions.py.

References upload_popcon.HTTP.baseUrl, and uploadConditions.HTTP.baseUrl.

Referenced by uploadConditions.HTTP.__init__().

264 
265  def setBaseUrl(self, baseUrl = ''):
266  '''Allows to set a base URL which will be prefixed to all the URLs
267  that will be queried later.
268  '''
269  self.baseUrl = baseUrl
270 
def uploadConditions.HTTP.setProxy (   self,
  proxy = '' 
) 
Allows to set a proxy.

Definition at line 271 of file uploadConditions.py.

271 
272  def setProxy(self, proxy = ''):
273  '''Allows to set a proxy.
274  '''
275  self.curl.setopt(self.curl.PROXY, proxy)
276 
def uploadConditions.HTTP.setRetries (   self,
  retries = () 
) 
Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 283 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.__init__().

283 
284  def setRetries(self, retries = ()):
285  '''Allows to set retries.
286 
287  The retries are a sequence of the seconds to wait per retry.
288 
289  The retries are done on:
290  * PyCurl errors (includes network problems, e.g. not being able
291  to connect to the host).
292  * 502 Bad Gateway (for the moment, to avoid temporary
293  Apache-CherryPy issues).
294  * 503 Service Temporarily Unavailable (for when we update
295  the frontends).
296  '''
297  self.retries = retries
def uploadConditions.HTTP.setTimeout (   self,
  timeout = 0 
) 
Allows to set a timeout.

Definition at line 277 of file uploadConditions.py.

277 
278  def setTimeout(self, timeout = 0):
279  '''Allows to set a timeout.
280  '''
281  self.curl.setopt(self.curl.TIMEOUT, timeout)
282 

Member Data Documentation

uploadConditions.HTTP.baseUrl

Definition at line 249 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.getToken(), uploadConditions.HTTP.query(), and uploadConditions.HTTP.setBaseUrl().

uploadConditions.HTTP.curl

Definition at line 240 of file uploadConditions.py.

uploadConditions.HTTP.retries

Definition at line 296 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

tuple uploadConditions.HTTP.retryCodes = frozenset([502, 503])
static

Definition at line 234 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.query().

uploadConditions.HTTP.token

Definition at line 251 of file uploadConditions.py.

Referenced by uploadConditions.HTTP.getToken(), uploadConditions.HTTP.query(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), and uploadConditions.ConditionsUploader.signOut().

Generated for CMSSW Reference Manual by doxygen 1.8.5