conddb_migrate Namespace Reference

Functions
def	check_and_run (args)
def	fetch_gts (connection)
def	main ()
def	make_gt_connection (args)
def	migrate_account (args)
def	migrate_accounts (args)
def	migrate_gt (args)
def	migrate_gts (args)
def	run_command (command, output_file)
def	tags_in_gts (args)

Detailed Description

CMS Conditions DB migration script.

Function Documentation

def conddb_migrate.check_and_run

(

args

)

Definition at line 179 of file conddb_migrate.py.

Referenced by main().

def check_and_run(args):
    if 'SCRAM_ARCH' not in os.environ:
        raise Exception('SCRAM_ARCH needs to be set: run cmsenv within a newish release.')

    if 'CMSSW_BASE' not in os.environ:
        raise Exception('CMSSW_BASE needs to be set: run cmsenv within a newish release.')

    if 'jobs' in args and args.jobs <= 0:
        raise Exception('If set, --jobs needs to be >= 1.')

    aliases = {
        'root': 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS',
        'boost': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
    }

    if args.db in aliases:
        args.db = aliases[args.db]

    # Check that the release and database match to prevent mistakes...
    if args.db == 'oracle://cms_orcoff_prep/CMS_CONDITIONS' and \
        not 'BOOST' in os.environ['CMSSW_VERSION']:
        raise Exception('Boost database without a Boost release -- mistake?')

    if args.db == 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS' and \
        'BOOST' in os.environ['CMSSW_VERSION']:
        raise Exception('ROOT database with a Boost release -- mistake?')

    # Create output log folder
    os.makedirs(args.output)

    args.func(args)

def conddb_migrate.fetch_gts

(

connection

)

Definition at line 122 of file conddb_migrate.py.

References ComparisonHelper.zip().

Referenced by migrate_gts(), and tags_in_gts().

def fetch_gts(connection):
    logging.info('Fetching global tag list...')
    cursor = connection.cursor()
    cursor.execute('''
        select substr(table_name, length('tagtree_table_') + 1) gt
        from all_tables
        where owner = 'CMS_COND_31X_GLOBALTAG'
            and table_name like 'TAGTREE_TABLE_%'
        order by gt
    ''')
    gts = zip(*cursor.fetchall())[0]
    logging.info('Fetching global tag list... Done: %s global tags found.', len(gts))
    return gts

def conddb_migrate.main

(

)

Entry point.

Definition at line 212 of file conddb_migrate.py.

References check_and_run().

def main():
    '''Entry point.
    '''

    parser = argparse.ArgumentParser(description='conddb_migrate - the CMS Conditions DB migration script')
    parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v prints debugging information of this tool, like tracebacks in case of errors.')
    parser.add_argument('--output', '-o', default=time.strftime('%Y-%m-%d-%H-%M-%S'), help='Output folder. Default: {current_timestamp}, i.e. %(default)s')
    parser.add_argument('db', help='Destination database. Aliases: "root" (CMS_CONDITIONS), "boost" (CMS_TEST_CONDITIONS), both in prep. *Make sure the database kind matches the code, i.e. use a BOOST IB when uploading to a Boost database; and a normal release when uploading to the ROOT database -- this script checks the CMSSW_VERSION when using the two official aliases in prep to prevent mistakes, but not for other databases.*')
    parser_subparsers = parser.add_subparsers(title='Available subcommands')

    parser_account = parser_subparsers.add_parser('account', description='Migrates all (non-migrated) tags, IOVs and payloads, from an account.')
    parser_account.add_argument('account', help='The account to migrate.')
    parser_account.set_defaults(func=migrate_account)

    parser_accounts = parser_subparsers.add_parser('accounts', description='Migrates all accounts (see "account" command).')
    parser_accounts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
    parser_accounts.set_defaults(func=migrate_accounts)

    parser_gt = parser_subparsers.add_parser('gt', description='Migrates a single global tag.')
    parser_gt.add_argument('gt', help='The global tag to migrate.')
    parser_gt.set_defaults(func=migrate_gt)

    parser_gts = parser_subparsers.add_parser('gts', description='Migrates all global tags (see "gt" command).')
    parser_gts.add_argument('authpath', help='Authentication path.')
    parser_gts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
    parser_gts.set_defaults(func=migrate_gts)

    parser_tags_in_gts = parser_subparsers.add_parser('tags_in_gts', description='Dumps the set of tags (including account name) which are in each global tag.')
    parser_tags_in_gts.add_argument('authpath', help='Authentication path.')
    parser_tags_in_gts.set_defaults(func=tags_in_gts)

    args = parser.parse_args()

    logging.basicConfig(
        format = '[%(asctime)s] %(levelname)s: %(message)s',
        level = logging.DEBUG if args.verbose >= 1 else logging.INFO,
    )

    if args.verbose >= 1:
        # Include the traceback
        check_and_run(args)
    else:
        # Only one error line
        try:
            check_and_run(args)
        except Exception as e:
            logging.error(e)
            sys.exit(1)

def conddb_migrate.make_gt_connection

(

args

)

Definition at line 116 of file conddb_migrate.py.

References digi_MixPreMix_cfi.strip.

Referenced by migrate_gts(), and tags_in_gts().

def make_gt_connection(args):
    logging.info('Fetching global tag list...')
    password = subprocess.check_output('''cat %s | grep -F 'CMS_COND_31X_GLOBALTAG' -2 | tail -1 | cut -d'"' -f4''' % os.path.join(args.authpath, 'readOnlyProd.xml'), shell=True).strip()
    return cx_Oracle.connect('CMS_COND_GENERAL_R', password, 'cms_orcon_adg')

def conddb_migrate.migrate_account

(

args

)

Definition at line 95 of file conddb_migrate.py.

References run_command().

def migrate_account(args):
    command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate -s oracle://cms_orcon_adg/%s -d %s'
    command = command_template % (args.account, args.db)
    run_command(command, os.path.join(args.output, args.account))

def conddb_migrate.migrate_accounts

(

args

)

Definition at line 101 of file conddb_migrate.py.

References genParticles_cff.map.

def migrate_accounts(args):
    def _make_args(args, account):
        newargs = argparse.Namespace(**vars(args))
        newargs.account = account
        return newargs

    multiprocessing.Pool(args.jobs).map(migrate_account, [_make_args(args, account) for account in accounts])

def conddb_migrate.migrate_gt

(

args

)

Definition at line 110 of file conddb_migrate.py.

References run_command().

def migrate_gt(args):
    command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate_gt -s oracle://cms_orcon_adg/CMS_COND_31X_GLOBALTAG -d %s -g %s'
    command = command_template % (args.db, args.gt)
    run_command(command, os.path.join(args.output, args.gt))

def conddb_migrate.migrate_gts

(

args

)

Definition at line 137 of file conddb_migrate.py.

References fetch_gts(), make_gt_connection(), and genParticles_cff.map.

def migrate_gts(args):
    gts = fetch_gts(make_gt_connection(args))

    def _make_args(args, gt):
        newargs = argparse.Namespace(**vars(args))
        newargs.gt = gt
        return newargs

    multiprocessing.Pool(args.jobs).map(migrate_gt, [_make_args(args, gt) for gt in gts])

def conddb_migrate.run_command	(		command,
			output_file
	)

Definition at line 86 of file conddb_migrate.py.

Referenced by migrate_account(), and migrate_gt().

def run_command(command, output_file):
    command = '%s > %s 2>&1' % (command, output_file)
    logging.info('Running %s', command)
    try:
        subprocess.check_call(command, shell=True)
    except subprocess.CalledProcessError as e:
        logging.error('Error while running %s: return code %s', command, e.returncode)

def conddb_migrate.tags_in_gts

(

args

)

Definition at line 148 of file conddb_migrate.py.

References Clusterizer1DCommons.add(), fetch_gts(), and make_gt_connection().

def tags_in_gts(args):
    # Dynamic SQL is used due to the schema
    # This is OK since we trust the input,
    # which is the GT database's tables.

    connection = make_gt_connection(args)
    gts = fetch_gts(connection)
    account_tags = {}

    for i, gt in enumerate(gts):
        logging.info('[%s/%s] Reading %s ...', i+1, len(gts), gt)
        cursor = connection.cursor()
        cursor.execute('''
            select "pfn", "tagname"
            from CMS_COND_31X_GLOBALTAG.TAGINVENTORY_TABLE
            where "tagid" in (
                select "tagid"
                from CMS_COND_31X_GLOBALTAG.TAGTREE_TABLE_%s
            )
        ''' % gt)

        for account, tag in cursor:
            account_tags.setdefault(account, set([])).add(tag)

    for account in sorted(account_tags):
        print account
        for tag in sorted(account_tags[account]):
            print '   ', tag
        print