Inheritance diagram for uploadConditions.ConditionsUploader:

Public Member Functions
def	__init__ (self, hostname=defaultHostname, urlTemplate=defaultUrlTemplate)

def	setHost (self, hostname)

def	signIn (self, username, password)

def	signOut (self)

def	uploadFile (self, filename, backend=defaultBackend, temporaryFile=defaultTemporaryFile)

Public Attributes
	hostname

	http

	password

	token

	urlTemplate

	userName

Private Member Functions
def	_checkForUpdates (self)

Detailed Description

Upload conditions to the CMS conditions uploader service.

Definition at line 435 of file uploadConditions.py.

Constructor & Destructor Documentation

◆ init()

def uploadConditions.ConditionsUploader.__init__	(	self,
		hostname = `defaultHostname`,
		urlTemplate = `defaultUrlTemplate`
	)

Definition at line 439 of file uploadConditions.py.

     def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
         self.hostname = hostname
         self.urlTemplate = urlTemplate 
         self.userName = None
         self.http = None
         self.password = None
         self.token = None
 

Member Function Documentation

◆ _checkForUpdates()

def uploadConditions.ConditionsUploader._checkForUpdates ( self )

private

Updates this script, if a new version is found.

Definition at line 494 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, uploadConditions.ConditionsUploader.http, createfilelist.int, upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

     def _checkForUpdates(self):
         '''Updates this script, if a new version is found.
         '''
 
         logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
         version = int(self.http.query('getUploadScriptVersion'))
 
         if version <= __version__:
             logging.debug('%s: Script is up-to-date.', self.hostname)
             return
 
         logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
 
         uploadScript = self.http.query('getUploadScript')
 
         self.signOut()
 
         logging.info('%s: ... saving the new version ...', self.hostname)
         with open(sys.argv[0], 'wb') as f:
             f.write(uploadScript)
 
         logging.info('%s: ... executing the new version...', self.hostname)
         os.execl(sys.executable, *([sys.executable] + sys.argv))
 
 

◆ setHost()

def uploadConditions.ConditionsUploader.setHost	(	self,
		hostname
	)

Definition at line 447 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

     def setHost( self, hostname ):
         if not hostname==self.hostname:
             self.token = None
             self.hostname = hostname
 

◆ signIn()

def uploadConditions.ConditionsUploader.signIn	(	self,
		username,
		password
	)

Definition at line 452 of file uploadConditions.py.

     def signIn(self, username, password ):
         if self.token is None:
             logging.debug("Initializing connection with server %s",self.hostname)
             ''' init the server.
             '''
             self.http = HTTP()
             if socket.getfqdn().strip().endswith('.cms'):
                 self.http.setProxy('https://cmsproxy.cms:3128/')
             self.http.setBaseUrl(self.urlTemplate % self.hostname)
             '''Signs in the server.
             '''
             logging.info('%s: Signing in user %s ...', self.hostname, username)
             try:
                 self.token = self.http.getToken(username, password)
             except Exception as e:
                 ret = -1
                 # optionally, we may want to have a different return for network related errors:
                 #code = self.http.curl.getinfo(pycurl.RESPONSE_CODE)
                 #if code in ( 502,503,504 ):
                 #    ret = -10
                 logging.error("Caught exception when trying to connect to %s: %s" % (self.hostname, str(e)) )
                 return ret
 
             if not self.token:
                 logging.error("could not get token for user %s from %s" % (username, self.hostname) )
                 return -2
             logging.debug( "got: '%s'", str(self.token) )
             self.userName = username
             self.password = password
         else:
             logging.debug("User %s has been already authenticated." %username)
         return 0
 

◆ signOut()

def uploadConditions.ConditionsUploader.signOut ( self )

Signs out the server.

Definition at line 485 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates().

     def signOut(self):
         '''Signs out the server.
         '''
 
         logging.info('%s: Signing out...', self.hostname)
         # self.http.query('logout')
         self.token = None
 
 

◆ uploadFile()

def uploadConditions.ConditionsUploader.uploadFile	(	self,
		filename,
		backend = `defaultBackend`,
		temporaryFile = `defaultTemporaryFile`
	)

Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 519 of file uploadConditions.py.

References uploadConditions.addToTarFile(), alcaDQMUpload.encode(), upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, uploadConditions.ConditionsUploader.http, mps_monitormerge.items, str, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

     def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
         '''Uploads a file to the dropBox.
 
         The filename can be without extension, with .db or with .txt extension.
         It will be stripped and then both .db and .txt files are used.
         '''
 
         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
         basename = os.path.basename(basepath)
 
         logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
 
         try:
             tarFile = tarfile.open(temporaryFile, 'w:bz2')
 
             with open('%s.db' % basepath, 'rb') as data:
                 addToTarFile(tarFile, data, 'data.db')
         except Exception as e:
             msg = 'Error when creating tar file. \n'
             msg += 'Please check that you have write access to the directory you are running,\n'
             msg += 'and that you have enough space on this disk (df -h .)\n'
             logging.error(msg)
             raise Exception(msg)
 
         with tempfile.NamedTemporaryFile(mode='rb+') as metadata:
             with open('%s.txt' % basepath, 'r') as originalMetadata:
                 metadata.write(json.dumps(json.load(originalMetadata), sort_keys = True, indent = 4).encode())
 
             metadata.seek(0)
             addToTarFile(tarFile, metadata, 'metadata.txt')
 
         tarFile.close()
 
         logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
 
         fileHash = hashlib.sha1()
         with open(temporaryFile, 'rb') as f:
             while True:
                 data = f.read(4 * 1024 * 1024)
                 if not data:
                     break
                 fileHash.update(data)
 
         fileHash = fileHash.hexdigest()
         fileInfo = os.stat(temporaryFile)
         fileSize = fileInfo.st_size
 
         logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
 
         logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
         os.rename(temporaryFile, fileHash)
         try:
             ret = self.http.query('uploadFile',
                               {
                                 'backend': backend,
                                 'fileName': basename,
                                 'userName': self.userName,
                               },
                               files = {
                                         'uploadedFile': fileHash,
                                       }
                               )
         except Exception as e:
             logging.error('Error from uploading: %s' % str(e))
             ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
 
         os.unlink(fileHash)
 
         statusInfo = json.loads(ret)['upload']
         logging.debug( 'upload returned: %s', statusInfo )
 
         okTags      = []
         skippedTags = []
         failedTags  = []
         for tag, info in statusInfo['itemStatus'].items():
             logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
             if 'ok'   in info['status'].lower() :
                 okTags.append( tag )
                 logging.info('tag %s successfully uploaded', tag)
             if 'skip' in info['status'].lower() :
                 skippedTags.append( tag )
                 logging.warning('found tag %s to be skipped. reason:  \n ... \t%s ', tag, info['info'])
             if 'fail' in info['status'].lower() :
                 failedTags.append( tag )
                 logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
 
         if len(okTags)      > 0: logging.info   ("tags sucessfully uploaded: %s ", str(okTags) )
         if len(skippedTags) > 0: logging.warning("tags SKIPped to upload   : %s ", str(skippedTags) )
         if len(failedTags)  > 0: logging.error  ("tags FAILed  to upload   : %s ", str(failedTags) )
 
         fileLogURL = 'https://cms-conddb.cern.ch/cmsDbBrowser/logs/show_cond_uploader_log/%s/%s' 
         backend = 'Prod'
         if self.hostname=='cms-conddb-dev.cern.ch':
             backend = 'Prep'
         logging.info('file log at: %s', fileLogURL % (backend,fileHash))
 
         return len(okTags)>0
 