o2o_helper Namespace Reference

Functions
def	configLogger (logfile, loglevel=logging.INFO)
def	copy_payload (dbFile, inputTag, destTags, destDb, since, userText)
def	create_metadata (metadataFilename, inputTag, destTags, destDb, since, userText)
def	exists_iov (dbFile, tag)
def	insert_to_file (template, target, replace_dict)
def	kill_subproc_noexcept (p)
def	send_mail (subject, message, send_to, send_from, text_attachments=[])
def	upload_payload (dbFile, inputTag, destTags, destDb, since, userText)

Detailed Description

Helper Script for StripO2O
@author: Huilin Qu

Function Documentation

configLogger()

def o2o_helper.configLogger	(		logfile,
			loglevel = logging.INFO
	)

Setting up logging to both file and console.
@see: https://docs.python.org/2/howto/logging-cookbook.html

Definition at line 25 of file o2o_helper.py.

def configLogger(logfile,loglevel=logging.INFO):
    '''Setting up logging to both file and console.
    @see: https://docs.python.org/2/howto/logging-cookbook.html
    '''
    # set up logging to file
    logging.basicConfig(level=loglevel,
                        format='[%(asctime)s] %(levelname)s: %(message)s',
                        filename=logfile,
                        filemode='w')
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(loglevel)
    # set a format which is simpler for console use
    formatter = logging.Formatter('[%(levelname)s] %(message)s')
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)
 

copy_payload()

def o2o_helper.copy_payload	(		dbFile,
			inputTag,
			destTags,
			destDb,
			since,
			userText
	)

Upload payload using conddb copy.

Definition at line 104 of file o2o_helper.py.

def copy_payload(dbFile, inputTag, destTags, destDb, since, userText):
    '''Upload payload using conddb copy.'''
    if isinstance(destTags, str):
        destTags = [destTags]
    if destDb.lower() == 'oracle://cms_orcon_prod/cms_conditions':
        copyDestDb = 'onlineorapro'
    elif destDb.lower() == 'oracle://cms_orcoff_prep/cms_conditions':
        copyDestDb = 'oradev'
    else:
        copyDestDb = destDb
    success = 0
    def copy(dest):
        command = 'conddb --force --yes --db {db} copy {inputTag} {destTag} --destdb {destDb} --synchronize --note "{note}"'.format(
            db=dbFile, inputTag=inputTag, destTag=dest, destDb=copyDestDb, note=userText)
        logging.info('Copy tag [%s] from %s to [%s] in %s:' % (inputTag, dbFile, dest, destDb))
        logging.debug(command)
        pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        out = pipe.communicate()[0]
        logging.info(out)
        return pipe.returncode
    for dest in destTags:
        returncode = copy(dest)
        if returncode == 0: success += 1
    logging.info('@@@Upload return code = %d@@@' % (success - len(destTags)))
    if success != len(destTags):
        raise RuntimeError('Upload FAILED!')
 
 

References filterCSVwithJSON.copy.

create_metadata()

def o2o_helper.create_metadata	(		metadataFilename,
			inputTag,
			destTags,
			destDb,
			since,
			userText
	)

Create metadata file for the conditionsUpload service.
@see: uploadConditions.runWizard()
@see: https://twiki.cern.ch/twiki/bin/view/CMS/DropBox

Keyword arguments:
metadataFilename -- output metadata filename
inputTag -- input tag name
destTags -- a list of destination tags
destDb -- [destinationDatabase] in metadata
since -- [since] in metadata
userText -- [userText] in metadata

Definition at line 56 of file o2o_helper.py.

def create_metadata(metadataFilename, inputTag, destTags, destDb, since, userText):
    '''Create metadata file for the conditionsUpload service.
    @see: uploadConditions.runWizard()
    @see: https://twiki.cern.ch/twiki/bin/view/CMS/DropBox
 
    Keyword arguments:
    metadataFilename -- output metadata filename
    inputTag -- input tag name
    destTags -- a list of destination tags
    destDb -- [destinationDatabase] in metadata
    since -- [since] in metadata
    userText -- [userText] in metadata
    '''
    if isinstance(destTags, str):
        destTags = [destTags]
    if since:
        since = int(since)  # convert to int if since is not None (input since can be a str)
    destinationTags = {}
    for destinationTag in destTags:
        destinationTags[destinationTag] = {}
    metadata = {
        'destinationDatabase': destDb,
        'destinationTags': destinationTags,
        'inputTag': inputTag,
        'since': since,
        'userText': userText,
    }
    logging.info('Writing metadata in %s', metadataFilename)
    logging.debug(metadata)
    with open(metadataFilename, 'wb') as metadataFile:
        metadataFile.write(json.dumps(metadata, sort_keys=True, indent=4))
 

References createfilelist.int.

Referenced by upload_payload().

exists_iov()

def o2o_helper.exists_iov	(		dbFile,
			tag
	)

Check if there exists any IOV for a specific tag in the given sqlite file.

Definition at line 150 of file o2o_helper.py.

def exists_iov(dbFile, tag):
    '''Check if there exists any IOV for a specific tag in the given sqlite file.'''
    dataConnection = sqlite3.connect(dbFile)
    dataCursor = dataConnection.cursor()
    dataCursor.execute('select SINCE from IOV where TAG_NAME=:tag_name', {'tag_name' : tag})
    return len(dataCursor.fetchall()) > 0

insert_to_file()

def o2o_helper.insert_to_file	(		template,
			target,
			replace_dict
	)

Update the template file based on the replace_dict, and write to the target.

Definition at line 44 of file o2o_helper.py.

def insert_to_file(template, target, replace_dict):
    '''Update the template file based on the replace_dict, and write to the target.'''
    logging.debug('Creating "%s" from template "%s" using dictionary:'%(target, template))
    logging.debug(replace_dict)
    with open(template, 'r') as input_file:
        config=input_file.read()
    with open(target, 'w') as output_file:
        for key, value in six.iteritems(replace_dict):
            config = config.replace(key, value)
        output_file.write(config)
    return config
 

kill_subproc_noexcept()

def o2o_helper.kill_subproc_noexcept

(

p

)

Kill a subprocess without throwing OSError.
Used for cleaning up subprocesses when the main script crashes.

Definition at line 16 of file o2o_helper.py.

def kill_subproc_noexcept(p):
    '''Kill a subprocess without throwing OSError.
    Used for cleaning up subprocesses when the main script crashes.'''
    try:
        p.terminate()
    except OSError:
        pass
 
 

send_mail()

def o2o_helper.send_mail	(		subject,
			message,
			send_to,
			send_from,
			text_attachments = []
	)

Send an email. [send_to] needs to be a list.

Definition at line 132 of file o2o_helper.py.

def send_mail(subject, message, send_to, send_from, text_attachments=[]):
    '''Send an email. [send_to] needs to be a list.'''
    msg = MIMEMultipart()
    msg['Subject'] = subject
    msg['From'] = send_from
    msg['To'] = ','.join(send_to)
    msg.attach(MIMEText(message))
 
    for fn in text_attachments:
        with open(fn, 'rb') as txtfile:
            attachment = MIMEText(txtfile.read())
            attachment.add_header('Content-Disposition', 'attachment', filename=os.path.basename(fn))
            msg.attach(attachment)
 
    s = smtplib.SMTP('localhost')
    s.sendmail(send_from, send_to, msg.as_string())
    s.quit()
 

References join().

upload_payload()

def o2o_helper.upload_payload	(		dbFile,
			inputTag,
			destTags,
			destDb,
			since,
			userText
	)

Upload payload using conditionUploader. 

Definition at line 88 of file o2o_helper.py.

def upload_payload(dbFile, inputTag, destTags, destDb, since, userText):
    '''Upload payload using conditionUploader. '''
    if isinstance(destTags, str):
        destTags = [destTags]
    metadataFilename = dbFile.replace('.db', '.txt')
    create_metadata(metadataFilename, inputTag, destTags, destDb, since, userText)
    logging.info('Uploading tag [%s] from %s to [%s] in %s:' % (inputTag, dbFile, ','.join(destTags), destDb))
    command = "uploadConditions.py %s" % dbFile
    pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out = pipe.communicate()[0]
    logging.info(out)
    logging.info('@@@Upload return code = %d@@@' % pipe.returncode)
    if pipe.returncode != 0:
        raise RuntimeError('Upload FAILED!')
 
 

References create_metadata(), and join().