Inheritance diagram for uploadConditions.ConditionsUploader:

Public Member Functions
def	__init__ (self, hostname=defaultHostname, urlTemplate=defaultUrlTemplate)

def	setHost (self, hostname)

def	signIn (self, username, password)

def	signOut (self)

def	uploadFile (self, filename, backend=defaultBackend, temporaryFile=defaultTemporaryFile)

Public Attributes
	hostname

	http

	password

	token

	urlTemplate

	userName

Private Member Functions
def	_checkForUpdates (self)

Detailed Description

Upload conditions to the CMS conditions uploader service.

Definition at line 446 of file uploadConditions.py.

Constructor & Destructor Documentation

def uploadConditions.ConditionsUploader.__init__	(	self,
		hostname = `defaultHostname`,
		urlTemplate = `defaultUrlTemplate`
	)

Definition at line 450 of file uploadConditions.py.

     def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
         self.hostname = hostname
         self.urlTemplate = urlTemplate 
         self.userName = None
         self.http = None
         self.password = None
         self.token = None
 

Member Function Documentation

def uploadConditions.ConditionsUploader._checkForUpdates ( self )

private

Updates this script, if a new version is found.

Definition at line 507 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, createfilelist.int, upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

     def _checkForUpdates(self):
         '''Updates this script, if a new version is found.
         '''
 
         logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
         version = int(self.http.query('getUploadScriptVersion'))
 
         if version <= __version__:
             logging.debug('%s: Script is up-to-date.', self.hostname)
             return
 
         logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
 
         uploadScript = self.http.query('getUploadScript')
 
         self.signOut()
 
         logging.info('%s: ... saving the new version ...', self.hostname)
         with open(sys.argv[0], 'wb') as f:
             f.write(uploadScript)
 
         logging.info('%s: ... executing the new version...', self.hostname)
         os.execl(sys.executable, *([sys.executable] + sys.argv))
 
 

def uploadConditions.ConditionsUploader.setHost	(	self,
		hostname
	)

Definition at line 458 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

     def setHost( self, hostname ):
         if not hostname==self.hostname:
             self.token = None
             self.hostname = hostname
 

def uploadConditions.ConditionsUploader.signIn	(	self,
		username,
		password
	)

Definition at line 463 of file uploadConditions.py.

Referenced by cmstc.TagCollector.signInInteractive().

     def signIn(self, username, password ):
         if self.token is None:
             logging.debug("Initializing connection with server %s",self.hostname)
             ''' init the server.
             '''
             self.http = HTTP()
             if socket.getfqdn().strip().endswith('.cms'):
                 self.http.setProxy('https://cmsproxy.cms:3128/')
             self.http.setBaseUrl(self.urlTemplate % self.hostname)
             '''Signs in the server.
             '''
 
             logging.info('%s: Signing in user %s ...', self.hostname, username)
             try:
                 self.token = self.http.getToken(username, password)
             except Exception as e:
                 ret = -1
                 # optionally, we may want to have a different return for network related errors:
                 #code = self.http.curl.getinfo(pycurl.RESPONSE_CODE)
                 #if code in ( 502,503,504 ):
                 #    ret = -10
                 logging.error("Caught exception when trying to connect to %s: %s" % (self.hostname, str(e)) )
                 return ret
 
             if not self.token:
                 logging.error("could not get token for user %s from %s" % (username, self.hostname) )
                 return -2
 
             logging.debug( "got: '%s'", str(self.token) )
             self.userName = username
             self.password = password
         else:
             logging.debug("User %s has been already authenticated." %username)
         return 0
 

def uploadConditions.ConditionsUploader.signOut ( self )

Signs out the server.

Definition at line 498 of file uploadConditions.py.

References upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, cond::TagInfo_t.token, upload_popcon.HTTP.token, uploadConditions.HTTP.token, upload_popcon.ConditionsUploader.token, and uploadConditions.ConditionsUploader.token.

Referenced by cmstc.TagCollector.__del__(), and uploadConditions.ConditionsUploader._checkForUpdates().

     def signOut(self):
         '''Signs out the server.
         '''
 
         logging.info('%s: Signing out...', self.hostname)
         # self.http.query('logout')
         self.token = None
 
 

def uploadConditions.ConditionsUploader.uploadFile	(	self,
		filename,
		backend = `defaultBackend`,
		temporaryFile = `defaultTemporaryFile`
	)

Uploads a file to the dropBox.

The filename can be without extension, with .db or with .txt extension.
It will be stripped and then both .db and .txt files are used.

Definition at line 532 of file uploadConditions.py.

References uploadConditions.addToTarFile(), upload_popcon.ConditionsUploader.hostname, uploadConditions.ConditionsUploader.hostname, mps_monitormerge.items, str, cond::auth::ServiceCredentials.userName, cond::CredentialData.userName, upload_popcon.ConditionsUploader.userName, and uploadConditions.ConditionsUploader.userName.

     def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
         '''Uploads a file to the dropBox.
 
         The filename can be without extension, with .db or with .txt extension.
         It will be stripped and then both .db and .txt files are used.
         '''
 
         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
         basename = os.path.basename(basepath)
 
         logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
 
         try:
             tarFile = tarfile.open(temporaryFile, 'w:bz2')
 
             with open('%s.db' % basepath, 'rb') as data:
                 addToTarFile(tarFile, data, 'data.db')
         except Exception as e:
             msg = 'Error when creating tar file. \n'
             msg += 'Please check that you have write access to the directory you are running,\n'
             msg += 'and that you have enough space on this disk (df -h .)\n'
             logging.error(msg)
             raise Exception(msg)
 
         with tempfile.NamedTemporaryFile() as metadata:
             with open('%s.txt' % basepath, 'rb') as originalMetadata:
                 json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
 
             metadata.seek(0)
             addToTarFile(tarFile, metadata, 'metadata.txt')
 
         tarFile.close()
 
         logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
 
         fileHash = hashlib.sha1()
         with open(temporaryFile, 'rb') as f:
             while True:
                 data = f.read(4 * 1024 * 1024)
                 if not data:
                     break
                 fileHash.update(data)
 
         fileHash = fileHash.hexdigest()
         fileInfo = os.stat(temporaryFile)
         fileSize = fileInfo.st_size
 
         logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
 
         logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
         os.rename(temporaryFile, fileHash)
         try:
             ret = self.http.query('uploadFile',
                               {
                                 'backend': backend,
                                 'fileName': basename,
                                 'userName': self.userName,
                               },
                               files = {
                                         'uploadedFile': fileHash,
                                       }
                               )
         except Exception as e:
             logging.error('Error from uploading: %s' % str(e))
             ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
 
         os.unlink(fileHash)
 
         statusInfo = json.loads(ret)['upload']
         logging.debug( 'upload returned: %s', statusInfo )
 
         okTags      = []
         skippedTags = []
         failedTags  = []
         for tag, info in statusInfo['itemStatus'].items():
             logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
             if 'ok'   in info['status'].lower() :
                 okTags.append( tag )
                 logging.info('tag %s successfully uploaded', tag)
             if 'skip' in info['status'].lower() :
                 skippedTags.append( tag )
                 logging.warning('found tag %s to be skipped. reason:  \n ... \t%s ', tag, info['info'])
             if 'fail' in info['status'].lower() :
                 failedTags.append( tag )
                 logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
 
         if len(okTags)      > 0: logging.info   ("tags sucessfully uploaded: %s ", str(okTags) )
         if len(skippedTags) > 0: logging.warning("tags SKIPped to upload   : %s ", str(skippedTags) )
         if len(failedTags)  > 0: logging.error  ("tags FAILed  to upload   : %s ", str(failedTags) )
 
         fileLogURL = 'https://%s/logs/dropBox/getFileLog?fileHash=%s' 
         logging.info('file log at: %s', fileLogURL % (self.hostname,fileHash))
 
         return len(okTags)>0
 