test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Private Member Functions
uploadConditions.ConditionsUploader Class Reference
Inheritance diagram for uploadConditions.ConditionsUploader:

Public Member Functions

def __init__
 
def setHost
 
def signIn
 
def signInAgain
 
def signOut
 
def uploadFile
 

Public Attributes

 hostname
 
 http
 
 password
 
 token
 
 urlTemplate
 
 userName
 

Private Member Functions

def _checkForUpdates
 

Detailed Description

Upload conditions to the CMS conditions uploader service.

Definition at line 423 of file uploadConditions.py.

Constructor & Destructor Documentation

def uploadConditions.ConditionsUploader.__init__ (   self,
  hostname = defaultHostname,
  urlTemplate = defaultUrlTemplate 
)

Definition at line 427 of file uploadConditions.py.

427 
428  def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
429  self.hostname = hostname
430  self.urlTemplate = urlTemplate
431  self.userName = None
432  self.http = None
433  self.password = None

Member Function Documentation

def uploadConditions.ConditionsUploader._checkForUpdates (   self)
private 
Updates this script, if a new version is found.

Definition at line 475 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

475 
476  def _checkForUpdates(self):
477  '''Updates this script, if a new version is found.
478  '''
479 
480  logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
481  version = int(self.http.query('getUploadScriptVersion'))
482 
483  if version <= __version__:
484  logging.debug('%s: Script is up-to-date.', self.hostname)
485  return
486 
487  logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
488 
489  uploadScript = self.http.query('getUploadScript')
490 
491  self.signOut()
492 
493  logging.info('%s: ... saving the new version ...', self.hostname)
494  with open(sys.argv[0], 'wb') as f:
495  f.write(uploadScript)
496 
497  logging.info('%s: ... executing the new version...', self.hostname)
498  os.execl(sys.executable, *([sys.executable] + sys.argv))
499 
def uploadConditions.ConditionsUploader.setHost (   self,
  hostname 
)

Definition at line 434 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, and uploadConditions.ConditionsUploader.hostname.

434 
435  def setHost( self, hostname ):
436  self.hostname = hostname
def uploadConditions.ConditionsUploader.signIn (   self,
  username,
  password 
) 
init the server.

Definition at line 437 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, uploadConditions.ConditionsUploader.http, upload_popcon.ConditionsUploader.urlTemplate, and uploadConditions.ConditionsUploader.urlTemplate.

Referenced by uploadConditions.ConditionsUploader.signInAgain(), and cmstc.TagCollector.signInInteractive().

437 
438  def signIn(self, username, password):
439  ''' init the server.
440  '''
441  self.http = HTTP()
442  if socket.getfqdn().strip().endswith('.cms'):
443  self.http.setProxy('https://cmsproxy.cms:3128/')
444  self.http.setBaseUrl(self.urlTemplate % self.hostname)
445  '''Signs in the server.
446  '''
447 
448  logging.info('%s: Signing in user %s ...', self.hostname, username)
449  try:
450  self.token = self.http.getToken(username, password)
451  except Exception as e:
452  logging.error("Caught exception when trying to get token for user %s from %s: %s" % (username, self.hostname, str(e)) )
453  return False
454 
455  if not self.token:
456  logging.error("could not get token for user %s from %s" % (username, self.hostname) )
457  return False
458 
459  logging.debug( "got: '%s'", str(self.token) )
460  self.userName = username
461  self.password = password
462  return True
def uploadConditions.ConditionsUploader.signInAgain (   self)

Definition at line 463 of file uploadConditions.py.

References cond::auth::ServiceCredentials.password, cond::CredentialData.password, upload_popcon.ConditionsUploader.password, uploadConditions.ConditionsUploader.password, upload_popcon.ConditionsUploader.signIn(), uploadConditions.ConditionsUploader.signIn(), cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

463 
464  def signInAgain(self):
465  return self.signIn( self.userName, self.password )
def uploadConditions.ConditionsUploader.signOut (   self) 
Signs out the server.

Definition at line 466 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates().

466 
467  def signOut(self):
468  '''Signs out the server.
469  '''
470 
471  logging.info('%s: Signing out...', self.hostname)
472  # self.http.query('logout')
473  self.token = None
474 
def uploadConditions.ConditionsUploader.uploadFile (   self,
  filename,
  backend = defaultBackend,
  temporaryFile = defaultTemporaryFile 
) 
Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 500 of file uploadConditions.py.

References uploadConditions.addToTarFile(), upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, mps_monitormerge.items, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

500 
501  def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
502  '''Uploads a file to the dropBox.
503 
504  The filename can be without extension, with .db or with .txt extension.
505  It will be stripped and then both .db and .txt files are used.
506  '''
507 
508  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
509  basename = os.path.basename(basepath)
510 
511  logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
512 
513  try:
514  tarFile = tarfile.open(temporaryFile, 'w:bz2')
515 
516  with open('%s.db' % basepath, 'rb') as data:
517  addToTarFile(tarFile, data, 'data.db')
518  except Exception as e:
519  msg = 'Error when creating tar file. \n'
520  msg += 'Please check that you have write access to the directory you are running,\n'
521  msg += 'and that you have enough space on this disk (df -h .)\n'
522  logging.error(msg)
523  raise Exception(msg)
524 
525  with tempfile.NamedTemporaryFile() as metadata:
526  with open('%s.txt' % basepath, 'rb') as originalMetadata:
527  json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
528 
529  metadata.seek(0)
530  addToTarFile(tarFile, metadata, 'metadata.txt')
531 
532  tarFile.close()
533 
534  logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
535 
536  fileHash = hashlib.sha1()
537  with open(temporaryFile, 'rb') as f:
538  while True:
539  data = f.read(4 * 1024 * 1024)
540  if not data:
541  break
542  fileHash.update(data)
543 
544  fileHash = fileHash.hexdigest()
545  fileInfo = os.stat(temporaryFile)
546  fileSize = fileInfo.st_size
547 
548  logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
549 
550  logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
551  os.rename(temporaryFile, fileHash)
552  try:
553  ret = self.http.query('uploadFile',
554  {
555  'backend': backend,
556  'fileName': basename,
557  'userName': self.userName,
558  },
559  files = {
560  'uploadedFile': fileHash,
561  }
562  )
563  except Exception as e:
564  logging.error('Error from uploading: %s' % str(e))
565  ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
566 
567  os.unlink(fileHash)
568 
569  statusInfo = json.loads(ret)['upload']
570  logging.debug( 'upload returned: %s', statusInfo )
571 
572  okTags = []
573  skippedTags = []
574  failedTags = []
575  for tag, info in statusInfo['itemStatus'].items():
576  logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
577  if 'ok' in info['status'].lower() :
578  okTags.append( tag )
579  logging.info('tag %s successfully uploaded', tag)
580  if 'skip' in info['status'].lower() :
581  skippedTags.append( tag )
582  logging.warning('found tag %s to be skipped. reason: \n ... \t%s ', tag, info['info'])
583  if 'fail' in info['status'].lower() :
584  failedTags.append( tag )
585  logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
586 
587  if len(okTags) > 0: logging.info ("tags sucessfully uploaded: %s ", str(okTags) )
588  if len(skippedTags) > 0: logging.warning("tags SKIPped to upload : %s ", str(skippedTags) )
589  if len(failedTags) > 0: logging.error ("tags FAILed to upload : %s ", str(failedTags) )
590 
591  fileLogURL = 'https://%s/logs/dropBox/getFileLog?fileHash=%s'
592  logging.info('file log at: %s', fileLogURL % (self.hostname,fileHash))
593 
594  return len(okTags)>0
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:27

Member Data Documentation

uploadConditions.ConditionsUploader.hostname

Definition at line 428 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), uploadConditions.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.uploadFile().

uploadConditions.ConditionsUploader.http

Definition at line 431 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.password

Definition at line 432 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signInAgain().

uploadConditions.ConditionsUploader.token

Definition at line 449 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signOut().

uploadConditions.ConditionsUploader.urlTemplate

Definition at line 429 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.userName

Definition at line 430 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signInAgain(), and uploadConditions.ConditionsUploader.uploadFile().

Generated for CMSSW Reference Manual by doxygen 1.8.5