test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Private Member Functions
uploadConditions.ConditionsUploader Class Reference
Inheritance diagram for uploadConditions.ConditionsUploader:

Public Member Functions

def __init__
 
def setHost
 
def signIn
 
def signOut
 
def uploadFile
 

Public Attributes

 hostname
 
 http
 
 password
 
 token
 
 urlTemplate
 
 userName
 

Private Member Functions

def _checkForUpdates
 

Detailed Description

Upload conditions to the CMS conditions uploader service.

Definition at line 445 of file uploadConditions.py.

Constructor & Destructor Documentation

def uploadConditions.ConditionsUploader.__init__ (   self,
  hostname = defaultHostname,
  urlTemplate = defaultUrlTemplate 
)

Definition at line 449 of file uploadConditions.py.

449 
450  def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
451  self.hostname = hostname
452  self.urlTemplate = urlTemplate
453  self.userName = None
454  self.http = None
455  self.password = None
456  self.token = None

Member Function Documentation

def uploadConditions.ConditionsUploader._checkForUpdates (   self)
private 
Updates this script, if a new version is found.

Definition at line 506 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

506 
507  def _checkForUpdates(self):
508  '''Updates this script, if a new version is found.
509  '''
510 
511  logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
512  version = int(self.http.query('getUploadScriptVersion'))
513 
514  if version <= __version__:
515  logging.debug('%s: Script is up-to-date.', self.hostname)
516  return
517 
518  logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
519 
520  uploadScript = self.http.query('getUploadScript')
521 
522  self.signOut()
523 
524  logging.info('%s: ... saving the new version ...', self.hostname)
525  with open(sys.argv[0], 'wb') as f:
526  f.write(uploadScript)
527 
528  logging.info('%s: ... executing the new version...', self.hostname)
529  os.execl(sys.executable, *([sys.executable] + sys.argv))
530 
def uploadConditions.ConditionsUploader.setHost (   self,
  hostname 
)

Definition at line 457 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

457 
458  def setHost( self, hostname ):
459  if not hostname==self.hostname:
460  self.token = None
461  self.hostname = hostname
def uploadConditions.ConditionsUploader.signIn (   self,
  username,
  password 
)

Definition at line 462 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, uploadConditions.ConditionsUploader.http, cond::auth::ServiceCredentials.password, cond::CredentialData.password, upload_popcon.ConditionsUploader.password, uploadConditions.ConditionsUploader.password, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, uploadConditions.ConditionsUploader.token, upload_popcon.ConditionsUploader.urlTemplate, uploadConditions.ConditionsUploader.urlTemplate, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

Referenced by cmstc.TagCollector.signInInteractive().

462 
463  def signIn(self, username, password ):
464  if self.token is None:
465  logging.debug("Initializing connection with server %s",self.hostname)
466  ''' init the server.
467  '''
468  self.http = HTTP()
469  if socket.getfqdn().strip().endswith('.cms'):
470  self.http.setProxy('https://cmsproxy.cms:3128/')
471  self.http.setBaseUrl(self.urlTemplate % self.hostname)
472  '''Signs in the server.
473  '''
474 
475  logging.info('%s: Signing in user %s ...', self.hostname, username)
476  try:
477  self.token = self.http.getToken(username, password)
478  except Exception as e:
479  ret = -1
480  # optionally, we may want to have a different return for network related errors:
481  #code = self.http.curl.getinfo(pycurl.RESPONSE_CODE)
482  #if code in ( 502,503,504 ):
483  # ret = -10
484  logging.error("Caught exception when trying to connect to %s: %s" % (self.hostname, str(e)) )
485  return ret
486 
487  if not self.token:
488  logging.error("could not get token for user %s from %s" % (username, self.hostname) )
489  return -2
490 
491  logging.debug( "got: '%s'", str(self.token) )
492  self.userName = username
493  self.password = password
494  else:
495  logging.debug("User %s has been already authenticated." %username)
496  return 0
def uploadConditions.ConditionsUploader.signOut (   self) 
Signs out the server.

Definition at line 497 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates().

497 
498  def signOut(self):
499  '''Signs out the server.
500  '''
501 
502  logging.info('%s: Signing out...', self.hostname)
503  # self.http.query('logout')
504  self.token = None
505 
def uploadConditions.ConditionsUploader.uploadFile (   self,
  filename,
  backend = defaultBackend,
  temporaryFile = defaultTemporaryFile 
) 
Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 531 of file uploadConditions.py.

References uploadConditions.addToTarFile(), upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, mps_monitormerge.items, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

531 
532  def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
533  '''Uploads a file to the dropBox.
534 
535  The filename can be without extension, with .db or with .txt extension.
536  It will be stripped and then both .db and .txt files are used.
537  '''
538 
539  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
540  basename = os.path.basename(basepath)
541 
542  logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
543 
544  try:
545  tarFile = tarfile.open(temporaryFile, 'w:bz2')
546 
547  with open('%s.db' % basepath, 'rb') as data:
548  addToTarFile(tarFile, data, 'data.db')
549  except Exception as e:
550  msg = 'Error when creating tar file. \n'
551  msg += 'Please check that you have write access to the directory you are running,\n'
552  msg += 'and that you have enough space on this disk (df -h .)\n'
553  logging.error(msg)
554  raise Exception(msg)
555 
556  with tempfile.NamedTemporaryFile() as metadata:
557  with open('%s.txt' % basepath, 'rb') as originalMetadata:
558  json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
559 
560  metadata.seek(0)
561  addToTarFile(tarFile, metadata, 'metadata.txt')
562 
563  tarFile.close()
564 
565  logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
566 
567  fileHash = hashlib.sha1()
568  with open(temporaryFile, 'rb') as f:
569  while True:
570  data = f.read(4 * 1024 * 1024)
571  if not data:
572  break
573  fileHash.update(data)
574 
575  fileHash = fileHash.hexdigest()
576  fileInfo = os.stat(temporaryFile)
577  fileSize = fileInfo.st_size
578 
579  logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
580 
581  logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
582  os.rename(temporaryFile, fileHash)
583  try:
584  ret = self.http.query('uploadFile',
585  {
586  'backend': backend,
587  'fileName': basename,
588  'userName': self.userName,
589  },
590  files = {
591  'uploadedFile': fileHash,
592  }
593  )
594  except Exception as e:
595  logging.error('Error from uploading: %s' % str(e))
596  ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
597 
598  os.unlink(fileHash)
599 
600  statusInfo = json.loads(ret)['upload']
601  logging.debug( 'upload returned: %s', statusInfo )
602 
603  okTags = []
604  skippedTags = []
605  failedTags = []
606  for tag, info in statusInfo['itemStatus'].items():
607  logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
608  if 'ok' in info['status'].lower() :
609  okTags.append( tag )
610  logging.info('tag %s successfully uploaded', tag)
611  if 'skip' in info['status'].lower() :
612  skippedTags.append( tag )
613  logging.warning('found tag %s to be skipped. reason: \n ... \t%s ', tag, info['info'])
614  if 'fail' in info['status'].lower() :
615  failedTags.append( tag )
616  logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
617 
618  if len(okTags) > 0: logging.info ("tags sucessfully uploaded: %s ", str(okTags) )
619  if len(skippedTags) > 0: logging.warning("tags SKIPped to upload : %s ", str(skippedTags) )
620  if len(failedTags) > 0: logging.error ("tags FAILed to upload : %s ", str(failedTags) )
621 
622  fileLogURL = 'https://%s/logs/dropBox/getFileLog?fileHash=%s'
623  logging.info('file log at: %s', fileLogURL % (self.hostname,fileHash))
624 
625  return len(okTags)>0
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:27

Member Data Documentation

uploadConditions.ConditionsUploader.hostname

Definition at line 450 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), uploadConditions.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.uploadFile().

uploadConditions.ConditionsUploader.http

Definition at line 453 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.password

Definition at line 454 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.token

Definition at line 455 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), and uploadConditions.ConditionsUploader.signOut().

uploadConditions.ConditionsUploader.urlTemplate

Definition at line 451 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.userName

Definition at line 452 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn(), and uploadConditions.ConditionsUploader.uploadFile().

Generated for CMSSW Reference Manual by doxygen 1.8.5