test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Private Member Functions
uploadConditions.ConditionsUploader Class Reference
Inheritance diagram for uploadConditions.ConditionsUploader:

Public Member Functions

def __init__
 
def setHost
 
def signIn
 
def signInAgain
 
def signOut
 
def uploadFile
 

Public Attributes

 hostname
 
 http
 
 password
 
 token
 
 urlTemplate
 
 userName
 

Private Member Functions

def _checkForUpdates
 

Detailed Description

Upload conditions to the CMS conditions uploader service.

Definition at line 418 of file uploadConditions.py.

Constructor & Destructor Documentation

def uploadConditions.ConditionsUploader.__init__ (   self,
  hostname = defaultHostname,
  urlTemplate = defaultUrlTemplate 
)

Definition at line 422 of file uploadConditions.py.

422 
423  def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
424  self.hostname = hostname
425  self.urlTemplate = urlTemplate
426  self.userName = None
427  self.http = None
428  self.password = None

Member Function Documentation

def uploadConditions.ConditionsUploader._checkForUpdates (   self)
private 
Updates this script, if a new version is found.

Definition at line 470 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

470 
471  def _checkForUpdates(self):
472  '''Updates this script, if a new version is found.
473  '''
474 
475  logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
476  version = int(self.http.query('getUploadScriptVersion'))
477 
478  if version <= __version__:
479  logging.debug('%s: Script is up-to-date.', self.hostname)
480  return
481 
482  logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
483 
484  uploadScript = self.http.query('getUploadScript')
485 
486  self.signOut()
487 
488  logging.info('%s: ... saving the new version ...', self.hostname)
489  with open(sys.argv[0], 'wb') as f:
490  f.write(uploadScript)
491 
492  logging.info('%s: ... executing the new version...', self.hostname)
493  os.execl(sys.executable, *([sys.executable] + sys.argv))
494 
def uploadConditions.ConditionsUploader.setHost (   self,
  hostname 
)

Definition at line 429 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, and uploadConditions.ConditionsUploader.hostname.

429 
430  def setHost( self, hostname ):
431  self.hostname = hostname
def uploadConditions.ConditionsUploader.signIn (   self,
  username,
  password 
) 
init the server.

Definition at line 432 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, uploadConditions.ConditionsUploader.http, upload_popcon.ConditionsUploader.urlTemplate, and uploadConditions.ConditionsUploader.urlTemplate.

Referenced by uploadConditions.ConditionsUploader.signInAgain(), and cmstc.TagCollector.signInInteractive().

432 
433  def signIn(self, username, password):
434  ''' init the server.
435  '''
436  self.http = HTTP()
437  if socket.getfqdn().strip().endswith('.cms'):
438  self.http.setProxy('https://cmsproxy.cms:3128/')
439  self.http.setBaseUrl(self.urlTemplate % self.hostname)
440  '''Signs in the server.
441  '''
442 
443  logging.info('%s: Signing in user %s ...', self.hostname, username)
444  try:
445  self.token = self.http.getToken(username, password)
446  except Exception as e:
447  logging.error("Caught exception when trying to get token for user %s from %s: %s" % (username, self.hostname, str(e)) )
448  return False
449 
450  if not self.token:
451  logging.error("could not get token for user %s from %s" % (username, self.hostname) )
452  return False
453 
454  logging.debug( "got: '%s'", str(self.token) )
455  self.userName = username
456  self.password = password
457  return True
def uploadConditions.ConditionsUploader.signInAgain (   self)

Definition at line 458 of file uploadConditions.py.

References cond::auth::ServiceCredentials.password, cond::CredentialData.password, upload_popcon.ConditionsUploader.password, uploadConditions.ConditionsUploader.password, upload_popcon.ConditionsUploader.signIn(), uploadConditions.ConditionsUploader.signIn(), cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

458 
459  def signInAgain(self):
460  return self.signIn( self.userName, self.password )
def uploadConditions.ConditionsUploader.signOut (   self) 
Signs out the server.

Definition at line 461 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates().

461 
462  def signOut(self):
463  '''Signs out the server.
464  '''
465 
466  logging.info('%s: Signing out...', self.hostname)
467  # self.http.query('logout')
468  self.token = None
469 
def uploadConditions.ConditionsUploader.uploadFile (   self,
  filename,
  backend = defaultBackend,
  temporaryFile = defaultTemporaryFile 
) 
Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 495 of file uploadConditions.py.

References uploadConditions.addToTarFile(), upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, mps_monitormerge.items, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

495 
496  def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
497  '''Uploads a file to the dropBox.
498 
499  The filename can be without extension, with .db or with .txt extension.
500  It will be stripped and then both .db and .txt files are used.
501  '''
502 
503  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
504  basename = os.path.basename(basepath)
505 
506  logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
507 
508  try:
509  tarFile = tarfile.open(temporaryFile, 'w:bz2')
510 
511  with open('%s.db' % basepath, 'rb') as data:
512  addToTarFile(tarFile, data, 'data.db')
513  except Exception as e:
514  msg = 'Error when creating tar file. \n'
515  msg += 'Please check that you have write access to the directory you are running,\n'
516  msg += 'and that you have enough space on this disk (df -h .)\n'
517  logging.error(msg)
518  raise Exception(msg)
519 
520  with tempfile.NamedTemporaryFile() as metadata:
521  with open('%s.txt' % basepath, 'rb') as originalMetadata:
522  json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
523 
524  metadata.seek(0)
525  addToTarFile(tarFile, metadata, 'metadata.txt')
526 
527  tarFile.close()
528 
529  logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
530 
531  fileHash = hashlib.sha1()
532  with open(temporaryFile, 'rb') as f:
533  while True:
534  data = f.read(4 * 1024 * 1024)
535  if not data:
536  break
537  fileHash.update(data)
538 
539  fileHash = fileHash.hexdigest()
540  fileInfo = os.stat(temporaryFile)
541  fileSize = fileInfo.st_size
542 
543  logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
544 
545  logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
546  os.rename(temporaryFile, fileHash)
547  try:
548  ret = self.http.query('uploadFile',
549  {
550  'backend': backend,
551  'fileName': basename,
552  'userName': self.userName,
553  },
554  files = {
555  'uploadedFile': fileHash,
556  }
557  )
558  except Exception as e:
559  logging.error('Error from uploading: %s' % str(e))
560  ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
561 
562  os.unlink(fileHash)
563 
564  statusInfo = json.loads(ret)['upload']
565  logging.debug( 'upload returned: %s', statusInfo )
566 
567  okTags = []
568  skippedTags = []
569  failedTags = []
570  for tag, info in statusInfo['itemStatus'].items():
571  logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
572  if 'ok' in info['status'].lower() :
573  okTags.append( tag )
574  logging.info('tag %s successfully uploaded', tag)
575  if 'skip' in info['status'].lower() :
576  skippedTags.append( tag )
577  logging.warning('found tag %s to be skipped. reason: \n ... \t%s ', tag, info['info'])
578  if 'fail' in info['status'].lower() :
579  failedTags.append( tag )
580  logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
581 
582  if len(okTags) > 0: logging.info ("tags sucessfully uploaded: %s ", str(okTags) )
583  if len(skippedTags) > 0: logging.warning("tags SKIPped to upload : %s ", str(skippedTags) )
584  if len(failedTags) > 0: logging.error ("tags FAILed to upload : %s ", str(failedTags) )
585 
586  fileLogURL = 'https://%s/logs/dropBox/getFileLog?fileHash=%s'
587  logging.info('file log at: %s', fileLogURL % (self.hostname,fileHash))
588 
589  return len(okTags)>0
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:27

Member Data Documentation

uploadConditions.ConditionsUploader.hostname

Definition at line 423 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), uploadConditions.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.uploadFile().

uploadConditions.ConditionsUploader.http

Definition at line 426 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.password

Definition at line 427 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signInAgain().

uploadConditions.ConditionsUploader.token

Definition at line 444 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signOut().

uploadConditions.ConditionsUploader.urlTemplate

Definition at line 424 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.userName

Definition at line 425 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signInAgain(), and uploadConditions.ConditionsUploader.uploadFile().

Generated for CMSSW Reference Manual by doxygen 1.8.5