Inheritance diagram for upload_popcon.ConditionsUploader:

Public Member Functions
def	__init__ (self, hostname=defaultHostname, urlTemplate=defaultUrlTemplate)

def	setHost (self, hostname)

def	signIn (self, username, password)

def	signInAgain (self)

def	signOut (self)

def	uploadFile (self, filename, backend=defaultBackend, temporaryFile=defaultTemporaryFile)

Public Attributes
	hostname

	http

	password

	token

	urlTemplate

	userName

Private Member Functions
def	_uploadFile (self, filename, backend=defaultBackend, temporaryFile=defaultTemporaryFile)

Detailed Description

Upload conditions to the CMS conditions uploader service.

Definition at line 259 of file upload_popcon.py.

Constructor & Destructor Documentation

def upload_popcon.ConditionsUploader.__init__	(	self,
		hostname = `defaultHostname`,
		urlTemplate = `defaultUrlTemplate`
	)

Definition at line 263 of file upload_popcon.py.

     def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
         self.hostname = hostname
         self.urlTemplate = urlTemplate 
         self.userName = None
         self.http = None
         self.password = None
 

Member Function Documentation

def upload_popcon.ConditionsUploader._uploadFile	(	self,
		filename,
		backend = `defaultBackend`,
		temporaryFile = `defaultTemporaryFile`
	)

private

Definition at line 340 of file upload_popcon.py.

References upload_popcon.addToTarFile(), upload_popcon.ConditionsUploader.hostname, mps_monitormerge.items, harvestTrackValidationPlots.str, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, and upload_popcon.ConditionsUploader.userName.

Referenced by upload_popcon.ConditionsUploader.uploadFile().

     def _uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
 
         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
         basename = os.path.basename(basepath)
 
         logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
 
         try:
             tarFile = tarfile.open(temporaryFile, 'w:bz2')
 
             with open('%s.db' % basepath, 'rb') as data:
                 addToTarFile(tarFile, data, 'data.db')
         except Exception as e:
             msg = 'Error when creating tar file. \n'
             msg += 'Please check that you have write access to the directory you are running,\n'
             msg += 'and that you have enough space on this disk (df -h .)\n'
             logging.error(msg)
             raise Exception(msg)
 
         with tempfile.NamedTemporaryFile() as metadata:
             with open('%s.txt' % basepath, 'rb') as originalMetadata:
                 json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
 
             metadata.seek(0)
             addToTarFile(tarFile, metadata, 'metadata.txt')
 
         tarFile.close()
 
         logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
 
         fileHash = hashlib.sha1()
         with open(temporaryFile, 'rb') as f:
             while True:
                 data = f.read(4 * 1024 * 1024)
                 if not data:
                     break
                 fileHash.update(data)
 
         fileHash = fileHash.hexdigest()
         fileInfo = os.stat(temporaryFile)
         fileSize = fileInfo.st_size
 
         logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
 
         logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
         os.rename(temporaryFile, fileHash)
         try:
             ret = self.http.query('uploadFile',
                               {
                                 'backend': backend,
                                 'fileName': basename,
                                 'userName': self.userName,
                               },
                               files = {
                                         'uploadedFile': fileHash,
                                       }
                               )
         except Exception as e:
             logging.error('Error from uploading: %s' % str(e))
             ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
 
         os.unlink(fileHash)
 
         statusInfo = json.loads(ret)['upload']
         logging.debug( 'upload returned: %s', statusInfo )
 
         okTags      = []
         skippedTags = []
         failedTags  = []
         for tag, info in statusInfo['itemStatus'].items():
             logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
             if 'ok'   in info['status'].lower() :
                 okTags.append( tag )
                 logging.info('tag %s successfully uploaded', tag)
             if 'skip' in info['status'].lower() :
                 skippedTags.append( tag )
                 logging.warning('found tag %s to be skipped. reason:  \n ... \t%s ', tag, info['info'])
             if 'fail' in info['status'].lower() :
                 failedTags.append( tag )
                 logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
 
         if len(okTags)      > 0: logging.info   ("tags sucessfully uploaded: %s ", str(okTags) )
         if len(skippedTags) > 0: logging.warning("tags SKIPped to upload   : %s ", str(skippedTags) )
         if len(failedTags)  > 0: logging.error  ("tags FAILed  to upload   : %s ", str(failedTags) )
 
         fileLogURL = 'https://%s/logs/dropBox/getFileLog?fileHash=%s' 
         logging.info('file log at: %s', fileLogURL % (self.hostname,fileHash))
 
         return len(okTags)>0

def upload_popcon.ConditionsUploader.setHost	(	self,
		hostname
	)

Definition at line 270 of file upload_popcon.py.

References upload_popcon.ConditionsUploader.hostname.

Referenced by upload_popcon.ConditionsUploader.uploadFile().

     def setHost( self, hostname ):
         self.hostname = hostname
 

def upload_popcon.ConditionsUploader.signIn	(	self,
		username,
		password
	)

init the server.

Definition at line 273 of file upload_popcon.py.

References upload_popcon.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, digi_MixPreMix_cfi.strip, and upload_popcon.ConditionsUploader.urlTemplate.

Referenced by upload_popcon.ConditionsUploader.signInAgain(), and cmstc.TagCollector.signInInteractive().

     def signIn(self, username, password):
         ''' init the server.
         '''
         self.http = HTTP()
         if socket.getfqdn().strip().endswith('.cms'):
             self.http.setProxy('https://cmsproxy.cms:3128/')
         self.http.setBaseUrl(self.urlTemplate % self.hostname)
         '''Signs in the server.
         '''
 
         logging.info('%s: Signing in user %s ...', self.hostname, username)
         try:
             self.token = self.http.getToken(username, password)
         except Exception as e:
             logging.error("Caught exception when trying to get token for user %s from %s: %s" % (username, self.hostname, str(e)) )
             return False
 
         if not self.token:
             logging.error("could not get token for user %s from %s" % (username, self.hostname) )
             return False
 
         logging.debug( "got: '%s'", str(self.token) )
         self.userName = username
         self.password = password
         return True
 

def upload_popcon.ConditionsUploader.signInAgain ( self )

Definition at line 299 of file upload_popcon.py.

References cond::auth::ServiceCredentials.password, cond::CredentialData.password, upload_popcon.ConditionsUploader.password, upload_popcon.ConditionsUploader.signIn(), cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, and upload_popcon.ConditionsUploader.userName.

Referenced by upload_popcon.ConditionsUploader.uploadFile().

     def signInAgain(self):
         return self.signIn( self.userName, self.password )
 

def upload_popcon.ConditionsUploader.signOut ( self )

Signs out the server.

Definition at line 302 of file upload_popcon.py.

References upload_popcon.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, and upload_popcon.ConditionsUploader.token.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates().

     def signOut(self):
         '''Signs out the server.
         '''
 
         logging.info('%s: Signing out...', self.hostname)
         # self.http.query('logout')
         self.token = None
 
 

def upload_popcon.ConditionsUploader.uploadFile	(	self,
		filename,
		backend = `defaultBackend`,
		temporaryFile = `defaultTemporaryFile`
	)

Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 311 of file upload_popcon.py.

References upload_popcon.ConditionsUploader._uploadFile(), upload_popcon.ConditionsUploader.setHost(), and upload_popcon.ConditionsUploader.signInAgain().

     def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
         '''Uploads a file to the dropBox.
 
         The filename can be without extension, with .db or with .txt extension.
         It will be stripped and then both .db and .txt files are used.
         '''
 
         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
         metadataFilename = '%s.txt' % basepath
         with open(metadataFilename, 'rb') as metadataFile:
             metadata = json.load( metadataFile )
         # When dest db = prep the hostname has to be set to dev.
         forceHost = False
         destDb = metadata['destinationDatabase']
         ret = False
         if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
             if destDb.startswith('oracle://cms_orcoff_prep'):
                     self.setHost( defaultDevHostname )
                     self.signInAgain()
                     forceHost = True
             ret = self._uploadFile(filename, backend, temporaryFile)
             if forceHost:
                 # set back the hostname to the original global setting
                 self.setHost( defaultHostname )
                 self.signInAgain()
         else:
             logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
         return ret
 