CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes | Private Member Functions
uploadConditions.ConditionsUploader Class Reference
Inheritance diagram for uploadConditions.ConditionsUploader:

Public Member Functions

def __init__
 
def setHost
 
def signIn
 
def signOut
 
def uploadFile
 

Public Attributes

 hostname
 
 http
 
 password
 
 token
 
 urlTemplate
 
 userName
 

Private Member Functions

def _checkForUpdates
 

Detailed Description

Upload conditions to the CMS conditions uploader service.

Definition at line 435 of file uploadConditions.py.

Constructor & Destructor Documentation

def uploadConditions.ConditionsUploader.__init__ (   self,
  hostname = defaultHostname,
  urlTemplate = defaultUrlTemplate 
)

Definition at line 439 of file uploadConditions.py.

439 
440  def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
441  self.hostname = hostname
442  self.urlTemplate = urlTemplate
443  self.userName = None
444  self.http = None
445  self.password = None
446  self.token = None

Member Function Documentation

def uploadConditions.ConditionsUploader._checkForUpdates (   self)
private 
Updates this script, if a new version is found.

Definition at line 494 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

494 
495  def _checkForUpdates(self):
496  '''Updates this script, if a new version is found.
497  '''
498 
499  logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
500  version = int(self.http.query('getUploadScriptVersion'))
501 
502  if version <= __version__:
503  logging.debug('%s: Script is up-to-date.', self.hostname)
504  return
505 
506  logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
507 
508  uploadScript = self.http.query('getUploadScript')
509 
510  self.signOut()
511 
512  logging.info('%s: ... saving the new version ...', self.hostname)
513  with open(sys.argv[0], 'wb') as f:
514  f.write(uploadScript)
515 
516  logging.info('%s: ... executing the new version...', self.hostname)
517  os.execl(sys.executable, *([sys.executable] + sys.argv))
518 
def uploadConditions.ConditionsUploader.setHost (   self,
  hostname 
)

Definition at line 447 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

447 
448  def setHost( self, hostname ):
449  if not hostname==self.hostname:
450  self.token = None
451  self.hostname = hostname
def uploadConditions.ConditionsUploader.signIn (   self,
  username,
  password 
)

Definition at line 452 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, uploadConditions.ConditionsUploader.http, cond::auth::ServiceCredentials.password, cond::CredentialData.password, upload_popcon.ConditionsUploader.password, uploadConditions.ConditionsUploader.password, str, digitizers_cfi.strip, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, uploadConditions.ConditionsUploader.token, upload_popcon.ConditionsUploader.urlTemplate, uploadConditions.ConditionsUploader.urlTemplate, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

452 
453  def signIn(self, username, password ):
454  if self.token is None:
455  logging.debug("Initializing connection with server %s",self.hostname)
456  ''' init the server.
457  '''
458  self.http = HTTP()
459  if socket.getfqdn().strip().endswith('.cms'):
460  self.http.setProxy('https://cmsproxy.cms:3128/')
461  self.http.setBaseUrl(self.urlTemplate % self.hostname)
462  '''Signs in the server.
463  '''
464  logging.info('%s: Signing in user %s ...', self.hostname, username)
465  try:
466  self.token = self.http.getToken(username, password)
467  except Exception as e:
468  ret = -1
469  # optionally, we may want to have a different return for network related errors:
470  #code = self.http.curl.getinfo(pycurl.RESPONSE_CODE)
471  #if code in ( 502,503,504 ):
472  # ret = -10
473  logging.error("Caught exception when trying to connect to %s: %s" % (self.hostname, str(e)) )
474  return ret
475 
476  if not self.token:
477  logging.error("could not get token for user %s from %s" % (username, self.hostname) )
478  return -2
479  logging.debug( "got: '%s'", str(self.token) )
480  self.userName = username
481  self.password = password
482  else:
483  logging.debug("User %s has been already authenticated." %username)
484  return 0
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:45
str
#define str(s)
Definition: TestProcessor.cc:53
def uploadConditions.ConditionsUploader.signOut (   self) 
Signs out the server.

Definition at line 485 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates().

485 
486  def signOut(self):
487  '''Signs out the server.
488  '''
489 
490  logging.info('%s: Signing out...', self.hostname)
491  # self.http.query('logout')
492  self.token = None
493 
def uploadConditions.ConditionsUploader.uploadFile (   self,
  filename,
  backend = defaultBackend,
  temporaryFile = defaultTemporaryFile 
) 
Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 519 of file uploadConditions.py.

References uploadConditions.addToTarFile(), alcaDQMUpload.encode(), upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, mps_monitormerge.items, str, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

519 
520  def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
521  '''Uploads a file to the dropBox.
522 
523  The filename can be without extension, with .db or with .txt extension.
524  It will be stripped and then both .db and .txt files are used.
525  '''
526 
527  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
528  basename = os.path.basename(basepath)
529 
530  logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
531 
532  try:
533  tarFile = tarfile.open(temporaryFile, 'w:bz2')
534 
535  with open('%s.db' % basepath, 'rb') as data:
536  addToTarFile(tarFile, data, 'data.db')
537  except Exception as e:
538  msg = 'Error when creating tar file. \n'
539  msg += 'Please check that you have write access to the directory you are running,\n'
540  msg += 'and that you have enough space on this disk (df -h .)\n'
541  logging.error(msg)
542  raise Exception(msg)
543 
544  with tempfile.NamedTemporaryFile(mode='rb+') as metadata:
545  with open('%s.txt' % basepath, 'r') as originalMetadata:
546  metadata.write(json.dumps(json.load(originalMetadata), sort_keys = True, indent = 4).encode())
547 
548  metadata.seek(0)
549  addToTarFile(tarFile, metadata, 'metadata.txt')
550 
551  tarFile.close()
552 
553  logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
554 
555  fileHash = hashlib.sha1()
556  with open(temporaryFile, 'rb') as f:
557  while True:
558  data = f.read(4 * 1024 * 1024)
559  if not data:
560  break
561  fileHash.update(data)
562 
563  fileHash = fileHash.hexdigest()
564  fileInfo = os.stat(temporaryFile)
565  fileSize = fileInfo.st_size
566 
567  logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
568 
569  logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
570  os.rename(temporaryFile, fileHash)
571  try:
572  ret = self.http.query('uploadFile',
573  {
574  'backend': backend,
575  'fileName': basename,
576  'userName': self.userName,
577  },
578  files = {
579  'uploadedFile': fileHash,
580  }
581  )
582  except Exception as e:
583  logging.error('Error from uploading: %s' % str(e))
584  ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
585 
586  os.unlink(fileHash)
587 
588  statusInfo = json.loads(ret)['upload']
589  logging.debug( 'upload returned: %s', statusInfo )
590 
591  okTags = []
592  skippedTags = []
593  failedTags = []
594  for tag, info in statusInfo['itemStatus'].items():
595  logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
596  if 'ok' in info['status'].lower() :
597  okTags.append( tag )
598  logging.info('tag %s successfully uploaded', tag)
599  if 'skip' in info['status'].lower() :
600  skippedTags.append( tag )
601  logging.warning('found tag %s to be skipped. reason: \n ... \t%s ', tag, info['info'])
602  if 'fail' in info['status'].lower() :
603  failedTags.append( tag )
604  logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
605 
606  if len(okTags) > 0: logging.info ("tags sucessfully uploaded: %s ", str(okTags) )
607  if len(skippedTags) > 0: logging.warning("tags SKIPped to upload : %s ", str(skippedTags) )
608  if len(failedTags) > 0: logging.error ("tags FAILed to upload : %s ", str(failedTags) )
609 
610  fileLogURL = 'https://cms-conddb.cern.ch/cmsDbBrowser/logs/show_cond_uploader_log/%s/%s'
611  backend = 'Prod'
612  if self.hostname=='cms-conddb-dev.cern.ch':
613  backend = 'Prep'
614  logging.info('file log at: %s', fileLogURL % (backend,fileHash))
615 
616  return len(okTags)>0
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:29
str
#define str(s)
Definition: TestProcessor.cc:53

Member Data Documentation

uploadConditions.ConditionsUploader.hostname

Definition at line 440 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), uploadConditions.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.uploadFile().

uploadConditions.ConditionsUploader.http

Definition at line 443 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.password

Definition at line 444 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.token

Definition at line 445 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), and uploadConditions.ConditionsUploader.signOut().

uploadConditions.ConditionsUploader.urlTemplate

Definition at line 441 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn().

uploadConditions.ConditionsUploader.userName

Definition at line 442 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.signIn(), and uploadConditions.ConditionsUploader.uploadFile().

Generated for CMSSW Reference Manual by doxygen 1.8.5