popcon2dropbox Namespace Reference

Functions
def	checkFile (dbName)
def	copy (args, dbName)
def	run (args)
def	saveFileForImportErrors (datef, dbName, withMetadata=False)
def	upload (args, dbName)

Variables
string	auth_path_key = 'COND_AUTH_PATH'
	consoleHandler = logging.StreamHandler(sys.stdout)
string	dateformatForFolder = "%Y-%m-%d-%H-%M-%S"
string	dateformatForLabel = "%Y-%m-%d %H:%M:%S"
string	errorInImportFileFolder = 'import_errors'
string	fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"
	levStr = os.environ[messageLevelEnvVar]
	logFormatter = logging.Formatter(fmt_str)
	logger = logging.getLogger()
	logLevel = logging.INFO
string	messageLevelEnvVar = 'POPCON_LOG_LEVEL'

Function Documentation

checkFile()

def popcon2dropbox.checkFile

(

dbName

)

Definition at line 32 of file popcon2dropbox.py.

References str.

Referenced by run().

def checkFile( dbName ):
    dbFileName = '%s.db' %dbName
    # check if the expected input file is there... 
    # exit code < 0 => error
    # exit code = 0 => skip
    # exit code = 1 => import                                                                                                                             
    if not os.path.exists( dbFileName ):
       logger.error('The file generated by PopCon with the data to be imported %s has not been found.'%dbFileName )
       return -1

    empty = True
    try:
       dbcon = sqlite3.connect( dbFileName )
       dbcur = dbcon.cursor()
       dbcur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='IOV'");
       if dbcur.fetchone() is None:
           logger.error('The condition database with the data to be imported has not been found in the file generated by PopCon: %s'%dbFileName )
           return -1
       dbcur.execute('SELECT * FROM IOV')
       rows = dbcur.fetchall()
       for r in rows:
          empty = False
       dbcon.close()
       if empty:
           logger.warning('The data set generated by PopCon contains no data to be imported. The import will be skipped.')
           return 0
       return 1
    except Exception as e:
       logger.error('Check on input data failed: %s' %str(e))
       return -2

copy()

def popcon2dropbox.copy	(		args,
			dbName
	)

Definition at line 126 of file popcon2dropbox.py.

References print(), saveFileForImportErrors(), and str.

Referenced by run().

def copy( args, dbName ):
    dbFileName = '%s.db' %dbName
    destDb = args.destDb
    destTag = args.destTag
    comment = args.comment

    datef = datetime.now()
    destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro"  }
    if destDb.lower() in destMap.keys():
        destDb = destMap[destDb.lower()]
    else:
        if destDb.startswith('sqlite'):
            destDb = destDb.split(':')[1]
        else:
            logger.error( 'Destination connection %s is not supported.' %destDb )
            return 
    # run the copy
    note = '"Importing data with O2O execution"'
    commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
    copyCommand = 'conddb %s' %commandOptions
    logger.info( 'Executing command: %s' %copyCommand )
    try:
        pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
        stdout = pipe.communicate()[0]
        print(stdout)
        retCode = pipe.returncode
        if retCode != 0:
            saveFileForImportErrors( datef, dbName )
        ret = retCode
    except Exception as e:
        ret = 1
        logger.error( str(e) )
    return ret

run()

def popcon2dropbox.run

(

args

)

Definition at line 160 of file popcon2dropbox.py.

References checkFile(), copy(), print(), and upload().

def run( args ):
    
    dbName = datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
    dbFileName = '%s.db' %dbName

    if args.auth is not None and not args.auth=='':
        if auth_path_key in os.environ:
            logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
        else:
            logger.info("Setting the authentication path to %s in the environment." %args.auth)
            os.environ[auth_path_key]=args.auth
    if os.path.exists( '%s.db' %dbName ):
       logger.info("Removing files with name %s" %dbName )
       os.remove( '%s.db' %dbName )
    if os.path.exists( '%s.txt' %dbName ):
       os.remove( '%s.txt' %dbName )
    command = 'cmsRun %s ' %args.job_file
    command += ' targetFile=%s' %dbFileName
    command += ' destinationDatabase=%s' %args.destDb
    command += ' destinationTag=%s' %args.destTag
    command += ' 2>&1'
    logger.info( 'Executing command: %s' %command )
    pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
    stdout = pipe.communicate()[0]
    retCode = pipe.returncode
    print(stdout)
    logger.info('PopCon Analyzer return code is: %s' %retCode )
    if retCode!=0:
       logger.error( 'O2O job failed. Skipping upload.' )
       return retCode

    ret = checkFile( dbName )
    if ret > 0:
        if args.copy:
            ret = copy( args, dbName )
        else:
            ret = upload( args, dbName )
    if ret >=0:
        logger.info('Deleting local file %s.db' %dbName ) 
        os.remove( '%s.db' %dbName )
    return ret

saveFileForImportErrors()

def popcon2dropbox.saveFileForImportErrors	(		datef,
			dbName,
			withMetadata = False
	)

Definition at line 63 of file popcon2dropbox.py.

Referenced by copy(), and upload().

def saveFileForImportErrors( datef, dbName, withMetadata=False ):
    # save a copy of the files in case of upload failure...
    leafFolderName = datef.strftime(dateformatForFolder)
    fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
    if not os.path.exists(fileFolder):
        os.makedirs(fileFolder)
    df= '%s.db' %dbName
    dataDestFile = os.path.join( fileFolder, df)
    if not os.path.exists(dataDestFile):
        shutil.copy2(df, dataDestFile)
    if withMetadata:
        mf= '%s.txt' %dbName
        metadataDestFile = os.path.join( fileFolder, mf )
        if not os.path.exists(metadataDestFile):
            shutil.copy2(df, metadataDestFile)
    logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
    

upload()

def popcon2dropbox.upload	(		args,
			dbName
	)

Definition at line 80 of file popcon2dropbox.py.

References print(), saveFileForImportErrors(), and str.

Referenced by run().

def upload( args, dbName ):
    destDb = args.destDb
    destTag = args.destTag
    comment = args.comment

    datef = datetime.now()

    # first remove any existing metadata file...                                                                                                                                
    if os.path.exists( '%s.txt' %dbName ):
       logger.debug('Removing already existing file %s' %dbName)
       os.remove( '%s.txt' %dbName )
    
    # dump Metadata for the Upload
    uploadMd = {}
    uploadMd['destinationDatabase'] = destDb
    tags = {}
    tagInfo = {}
    tags[ destTag ] = tagInfo
    uploadMd['destinationTags'] = tags
    uploadMd['inputTag'] = destTag
    uploadMd['since'] = None
    datelabel = datef.strftime(dateformatForLabel)
    commentStr = ''
    if not comment is None:
       commentStr = comment
    uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
    with open( '%s.txt' %dbName, 'wb') as jf:
       jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
       jf.write('\n')

    # run the upload
    uploadCommand = 'uploadConditions.py %s' %dbName
    ret = 0
    try:
       pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
       stdout = pipe.communicate()[0]
       print(stdout)
       retCode = pipe.returncode
       if retCode != 0:
           saveFileForImportErrors( datef, dbName, True )
       ret |= retCode
    except Exception as e:
       ret |= 1
       logger.error(str(e))
    return ret

Variable Documentation

auth_path_key

string popcon2dropbox.auth_path_key = 'COND_AUTH_PATH'

Definition at line 16 of file popcon2dropbox.py.

consoleHandler

popcon2dropbox.consoleHandler = logging.StreamHandler(sys.stdout)

Definition at line 28 of file popcon2dropbox.py.

dateformatForFolder

string popcon2dropbox.dateformatForFolder = "%Y-%m-%d-%H-%M-%S"

Definition at line 13 of file popcon2dropbox.py.

dateformatForLabel

string popcon2dropbox.dateformatForLabel = "%Y-%m-%d %H:%M:%S"

Definition at line 14 of file popcon2dropbox.py.

errorInImportFileFolder

string popcon2dropbox.errorInImportFileFolder = 'import_errors'

Definition at line 12 of file popcon2dropbox.py.

fmt_str

string popcon2dropbox.fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"

Definition at line 19 of file popcon2dropbox.py.

levStr

popcon2dropbox.levStr = os.environ[messageLevelEnvVar]

Definition at line 22 of file popcon2dropbox.py.

logFormatter

popcon2dropbox.logFormatter = logging.Formatter(fmt_str)

Definition at line 25 of file popcon2dropbox.py.

logger

popcon2dropbox.logger = logging.getLogger()

Definition at line 26 of file popcon2dropbox.py.

logLevel

popcon2dropbox.logLevel = logging.INFO

Definition at line 20 of file popcon2dropbox.py.

Referenced by edm::storage::DavixFile.configureDavixLogLevel().

messageLevelEnvVar

string popcon2dropbox.messageLevelEnvVar = 'POPCON_LOG_LEVEL'

Definition at line 18 of file popcon2dropbox.py.

---

Generated for CMSSW Reference Manual by  1.8.14