Reference Manual

def upload_popcon.ConditionsUploader._uploadFile	(	self,
		filename,
		backend = `defaultBackend`,
		temporaryFile = `defaultTemporaryFile`
	)

private

Definition at line 340 of file upload_popcon.py.

References upload_popcon.addToTarFile(), upload_popcon.ConditionsUploader.hostname, mps_monitormerge.items, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, and upload_popcon.ConditionsUploader.userName.

Referenced by upload_popcon.ConditionsUploader.uploadFile().

 
     def _uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
 
         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
         basename = os.path.basename(basepath)
 
         logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
 
         try:
             tarFile = tarfile.open(temporaryFile, 'w:bz2')
 
             with open('%s.db' % basepath, 'rb') as data:
                 addToTarFile(tarFile, data, 'data.db')
         except Exception as e:
             msg = 'Error when creating tar file. \n'
             msg += 'Please check that you have write access to the directory you are running,\n'
             msg += 'and that you have enough space on this disk (df -h .)\n'
             logging.error(msg)
             raise Exception(msg)
 
         with tempfile.NamedTemporaryFile() as metadata:
             with open('%s.txt' % basepath, 'rb') as originalMetadata:
                 json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
 
             metadata.seek(0)
             addToTarFile(tarFile, metadata, 'metadata.txt')
 
         tarFile.close()
 
         logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
 
         fileHash = hashlib.sha1()
         with open(temporaryFile, 'rb') as f:
             while True:
                 data = f.read(4 * 1024 * 1024)
                 if not data:
                     break
                 fileHash.update(data)
 
         fileHash = fileHash.hexdigest()
         fileInfo = os.stat(temporaryFile)
         fileSize = fileInfo.st_size
 
         logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
 
         logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
         os.rename(temporaryFile, fileHash)
         try:
             ret = self.http.query('uploadFile',
                               {
                                 'backend': backend,
                                 'fileName': basename,
                                 'userName': self.userName,
                               },
                               files = {
                                         'uploadedFile': fileHash,
                                       }
                               )
         except Exception as e:
             logging.error('Error from uploading: %s' % str(e))
             ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
 
         os.unlink(fileHash)
 
         statusInfo = json.loads(ret)['upload']
         logging.debug( 'upload returned: %s', statusInfo )
 
         okTags      = []
         skippedTags = []
         failedTags  = []
         for tag, info in statusInfo['itemStatus'].items():
             logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
             if 'ok'   in info['status'].lower() :
                 okTags.append( tag )
                 logging.info('tag %s successfully uploaded', tag)
             if 'skip' in info['status'].lower() :
                 skippedTags.append( tag )
                 logging.warning('found tag %s to be skipped. reason:  \n ... \t%s ', tag, info['info'])
             if 'fail' in info['status'].lower() :
                 failedTags.append( tag )
                 logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
 
         if len(okTags)      > 0: logging.info   ("tags sucessfully uploaded: %s ", str(okTags) )
         if len(skippedTags) > 0: logging.warning("tags SKIPped to upload   : %s ", str(skippedTags) )
         if len(failedTags)  > 0: logging.error  ("tags FAILed  to upload   : %s ", str(failedTags) )
 
         fileLogURL = 'https://%s/logs/dropBox/getFileLog?fileHash=%s' 
         logging.info('file log at: %s', fileLogURL % (self.hostname,fileHash))
 
         return len(okTags)>0

def upload_popcon.ConditionsUploader.setHost	(	self,
		hostname
	)

Definition at line 270 of file upload_popcon.py.

References upload_popcon.ConditionsUploader.hostname.

Referenced by upload_popcon.ConditionsUploader.uploadFile().

 
     def setHost( self, hostname ):
         self.hostname = hostname

upload_popcon.ConditionsUploader.setHost

def setHost

Definition: upload_popcon.py:270

upload_popcon.ConditionsUploader.hostname

hostname

Definition: upload_popcon.py:264

def upload_popcon.ConditionsUploader.signIn	(	self,
		username,
		password
	)

init the server.

Definition at line 273 of file upload_popcon.py.

References upload_popcon.ConditionsUploader.hostname, upload_popcon.ConditionsUploader.http, and upload_popcon.ConditionsUploader.urlTemplate.

Referenced by upload_popcon.ConditionsUploader.signInAgain(), and cmstc.TagCollector.signInInteractive().

 
     def signIn(self, username, password):
         ''' init the server.
         '''
         self.http = HTTP()
         if socket.getfqdn().strip().endswith('.cms'):
             self.http.setProxy('https://cmsproxy.cms:3128/')
         self.http.setBaseUrl(self.urlTemplate % self.hostname)
         '''Signs in the server.
         '''
 
         logging.info('%s: Signing in user %s ...', self.hostname, username)
         try:
             self.token = self.http.getToken(username, password)
         except Exception as e:
             logging.error("Caught exception when trying to get token for user %s from %s: %s" % (username, self.hostname, str(e)) )
             return False
 
         if not self.token:
             logging.error("could not get token for user %s from %s" % (username, self.hostname) )
             return False
 
         logging.debug( "got: '%s'", str(self.token) )
         self.userName = username
         self.password = password
         return True

upload_popcon.ConditionsUploader.http

http

Definition: upload_popcon.py:267

upload_popcon.ConditionsUploader.urlTemplate

urlTemplate

Definition: upload_popcon.py:265

upload_popcon.ConditionsUploader.password

password

Definition: upload_popcon.py:268

upload_popcon.ConditionsUploader.token

token

Definition: upload_popcon.py:285

upload_popcon.ConditionsUploader.userName

userName

Definition: upload_popcon.py:266

upload_popcon.ConditionsUploader.hostname

hostname

Definition: upload_popcon.py:264

upload_popcon.HTTP

Definition: upload_popcon.py:64

upload_popcon.ConditionsUploader.signIn

def signIn

Definition: upload_popcon.py:273

def upload_popcon.ConditionsUploader.signInAgain ( self )

Definition at line 299 of file upload_popcon.py.

References cond::auth::ServiceCredentials.password, cond::CredentialData.password, upload_popcon.ConditionsUploader.password, upload_popcon.ConditionsUploader.signIn(), cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, and upload_popcon.ConditionsUploader.userName.

Referenced by upload_popcon.ConditionsUploader.uploadFile().

 
     def signInAgain(self):
         return self.signIn( self.userName, self.password )

upload_popcon.ConditionsUploader.password

password

Definition: upload_popcon.py:268

upload_popcon.ConditionsUploader.signInAgain

def signInAgain

Definition: upload_popcon.py:299

upload_popcon.ConditionsUploader.userName

userName

Definition: upload_popcon.py:266

upload_popcon.ConditionsUploader.signIn

def signIn

Definition: upload_popcon.py:273

def upload_popcon.ConditionsUploader.signOut ( self )

Signs out the server.

Definition at line 302 of file upload_popcon.py.

References upload_popcon.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, and upload_popcon.ConditionsUploader.token.

Referenced by uploadConditions.ConditionsUploader._checkForUpdates().

 
     def signOut(self):
         '''Signs out the server.
         '''
 
         logging.info('%s: Signing out...', self.hostname)
         # self.http.query('logout')
         self.token = None
 

upload_popcon.ConditionsUploader.signOut

def signOut

Definition: upload_popcon.py:302

upload_popcon.ConditionsUploader.token

token

Definition: upload_popcon.py:285

upload_popcon.ConditionsUploader.hostname

hostname

Definition: upload_popcon.py:264

def upload_popcon.ConditionsUploader.uploadFile	(	self,
		filename,
		backend = `defaultBackend`,
		temporaryFile = `defaultTemporaryFile`
	)

Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 311 of file upload_popcon.py.

References upload_popcon.ConditionsUploader._uploadFile(), upload_popcon.ConditionsUploader.setHost(), and upload_popcon.ConditionsUploader.signInAgain().

 
     def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
         '''Uploads a file to the dropBox.
 
         The filename can be without extension, with .db or with .txt extension.
         It will be stripped and then both .db and .txt files are used.
         '''
 
         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
         metadataFilename = '%s.txt' % basepath
         with open(metadataFilename, 'rb') as metadataFile:
             metadata = json.load( metadataFile )
         # When dest db = prep the hostname has to be set to dev.
         forceHost = False
         destDb = metadata['destinationDatabase']
         ret = False
         if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
             if destDb.startswith('oracle://cms_orcoff_prep'):
                     self.setHost( defaultDevHostname )
                     self.signInAgain()
                     forceHost = True
             ret = self._uploadFile(filename, backend, temporaryFile)
             if forceHost:
                 # set back the hostname to the original global setting
                 self.setHost( defaultHostname )
                 self.signInAgain()
         else:
             logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
         return ret