conddblib Namespace Reference

Classes
class	_Col
class	BoostRunMap
class	Connection
class	DbRef
class	GlobalTag
class	GlobalTagMap
class	IOV
class	Payload
class	RunInfo
class	Synchronization
class	Tag
class	TagLog
class	TagMetadata
class	TimeType

Functions
def	_exists (session, primary_key, value)
def	_getCMSFrontierConnectionString (database)
def	_getCMSSQLAlchemyConnectionString (technology, service, schema_name)
def	_inserted_before (timestamp)
def	connect (url, authPath=None, verbose=0)
def	fq_col (schema, table, column)
def	fq_name (schema_name, table_name)
def	getPayload (session, hash)
def	getSchema (tp)
def	getSessionOnMasterDB (session1, session2)
def	hash (data)
def	listObject (session, name, snapshot=None)
def	make_dbtype (backendName, schemaName, baseType)
def	make_url (database='pro', read_only=True)
def	oracle_connection_string (db_service, db_schema)

Detailed Description

CMS Conditions DB Python library.

Function Documentation

_exists()

def conddblib._exists (   session,   primary_key,   value  )

private

Definition at line 583 of file conddblib.py.

def _exists(session, primary_key, value):
    ret = None
    try: 
        ret = session.query(primary_key).\
            filter(primary_key == value).\
            count() != 0
    except sqlalchemy.exc.OperationalError:
        pass
 
    return ret
 

References submitPVResolutionJobs.count, and ALCARECOTkAlBeamHalo_cff.filter.

Referenced by listObject().

_getCMSFrontierConnectionString()

def conddblib._getCMSFrontierConnectionString (   database )

private

Definition at line 455 of file conddblib.py.

def _getCMSFrontierConnectionString(database):
    import subprocess
    return subprocess.Popen(['cmsGetFnConnect', 'frontier://%s' % database], stdout = subprocess.PIPE).communicate()[0].strip()
 

References communicate(), and digitizers_cfi.strip.

Referenced by _getCMSSQLAlchemyConnectionString().

_getCMSSQLAlchemyConnectionString()

def conddblib._getCMSSQLAlchemyConnectionString (   technology,   service,   schema_name  )

private

Definition at line 459 of file conddblib.py.

def _getCMSSQLAlchemyConnectionString(technology,service,schema_name):
    if technology == 'frontier':
        import urllib
        import sys
        py3k = sys.version_info >= (3, 0)        
        if py3k:
            return '%s://@%s/%s' % ('oracle+frontier', urllib.parse.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
        else:
            return '%s://@%s/%s' % ('oracle+frontier', urllib.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
    elif technology == 'oracle':
        return '%s://%s@%s' % (technology, schema_name, service)
 
# Entry point

References _getCMSFrontierConnectionString().

Referenced by make_url().

_inserted_before()

def conddblib._inserted_before (   timestamp )

private

To be used inside filter().

Definition at line 594 of file conddblib.py.

def _inserted_before(timestamp):
    '''To be used inside filter().
    '''
 
    if timestamp is None:
        # XXX: Returning None does not get optimized (skipped) by SQLAlchemy,
        #      and returning True does not work in Oracle (generates "and 1"
        #      which breaks Oracle but not SQLite). For the moment just use
        #      this dummy condition.
        return sqlalchemy.literal(True) == sqlalchemy.literal(True)
 
    return conddb.IOV.insertion_time <= _parse_timestamp(timestamp)
 

Referenced by listObject().

connect()

def conddblib.connect	(		url,
			authPath = None,
			verbose = 0
	)

Returns a Connection instance to the CMS Condition DB.

See database_help for the description of the database parameter.

The verbosity level is as follows:

    0 = No output (default).
    1 = SQL statements issued, including their parameters.
    2 = In addition, results of the queries (all rows and the column headers).

Definition at line 518 of file conddblib.py.

def connect(url, authPath=None, verbose=0):
    '''Returns a Connection instance to the CMS Condition DB.
 
    See database_help for the description of the database parameter.
 
    The verbosity level is as follows:
 
        0 = No output (default).
        1 = SQL statements issued, including their parameters.
        2 = In addition, results of the queries (all rows and the column headers).
    '''
 
    if url.drivername == 'oracle':
        if url.username is None:
            logging.error('Could not resolve the username for the connection %s. Please provide a connection in the format oracle://[user]:[pass]@[host]' %url )
            raise Exception('Connection format error: %s' %url )
        if url.password is None:
            if authPath is None:
                if authPathEnvVar in os.environ:
                    authPath = os.environ[authPathEnvVar]
            explicit_auth = False
            if authPath is not None:
                dbkey_path = os.path.join(authPath,dbkey_folder)
                if not os.path.exists(dbkey_path):
                    authFile = os.path.join(authPath,'.netrc')
                    if os.path.exists(authFile):
                        entryKey = url.host.lower()+"/"+url.username.lower()
                        logging.debug('Looking up credentials for %s in file %s ' %(entryKey,authFile) )
                        import netrc
                        params = netrc.netrc( authFile ).authenticators(entryKey)
                        if params is not None:
                            (username, account, password) = params
                            url.username = username
                            url.password = password
                        else:
                            msg = 'The entry %s has not been found in the .netrc file.' %entryKey
                            raise TypeError(msg)
                    else:
                        explicit_auth =True
                else:
                    import pluginCondDBPyBind11Interface as credential_store
                    connect_for_update = ( url.username == dbwriter_user_name )
                    connection_string = oracle_connection_string(url.host.lower(),schema_name)
                    logging.debug('Using db key to get credentials for %s' %connection_string )
                    (username,password) = credential_store.get_db_credentials(connection_string,connect_for_update,authPath)
                    url.username = username
                    url.password = password
            else:
                import getpass
                pwd = getpass.getpass('Password for %s: ' % str(url))
                if pwd is None or pwd == '':
                    pwd = getpass.getpass('Password for %s: ' % str(url))
                    if pwd is None or pwd == '':
                        raise Exception('Empty password provided, bailing out...')
                url.password = pwd
 
    if verbose >= 1:
        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
 
    if verbose >= 2:
        logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
 
    return Connection(url)
 
 

References oracle_connection_string(), and str.

Referenced by helperFunctions.getTagsMap(), and conddblib.Connection.is_valid().

fq_col()

def conddblib.fq_col	(		schema,
			table,
			column
	)

Definition at line 184 of file conddblib.py.

def fq_col( schema, table, column ):
    fqn = '%s.%s' %(table, column)
    if schema is not None:
        fqn = '%s.%s' %(schema,fqn)
    return fqn
 

Referenced by make_dbtype().

fq_name()

def conddblib.fq_name	(		schema_name,
			table_name
	)

Definition at line 166 of file conddblib.py.

def fq_name( schema_name, table_name ):
    name = table_name
    if schema_name is not None:
        name = '%s.%s' %(schema_name, table_name)
    return name
 

getPayload()

def conddblib.getPayload	(		session,
			hash
	)

Definition at line 646 of file conddblib.py.

def getPayload(session, hash):
    # get payload from DB:
    data, payloadType = session.query(Payload.data, Payload.object_type).filter(Payload.hash == hash).one()
    return data

References ALCARECOTkAlBeamHalo_cff.filter, and SiPixelPI.one.

getSchema()

def conddblib.getSchema

(

tp

)

Definition at line 224 of file conddblib.py.

def getSchema(tp):
    if tp.__table_args__ is not None:
        return tp.__table_args__['schema']
    return None
 

Referenced by conddblib.Connection.is_valid().

getSessionOnMasterDB()

def conddblib.getSessionOnMasterDB	(		session1,
			session2
	)

Definition at line 433 of file conddblib.py.

def getSessionOnMasterDB( session1, session2 ):
    key = '%s/%s' 
    sessiondict = { }
    sessiondict[key %(session1._url.drivername,session1._url.host)] = session1
    sessiondict[key %(session2._url.drivername,session2._url.host)] = session2
    masterkey = key %('oracle',ONLINEORAPRO)
    if masterkey in sessiondict.keys():
        return sessiondict[masterkey]
    adgkey = key %('oracle',ORAPRO)
    if adgkey in sessiondict.keys():
        return sessiondict[adgkey]
    frontierkey = key %('frontier',PRO)
    if frontierkey in sessiondict.keys():
        return sessiondict[frontierkey]
    # default case: frontier on pro
    conn = Connection(make_url())
    session = conn.session()
    # is it required?
    session._conn = conn
    return session
 
# Connection helpers

References make_url().

hash()

def conddblib.hash

(

data

)

Definition at line 52 of file conddblib.py.

def hash(data):
    return hashlib.sha1(data.encode('ascii')).hexdigest()
 
 
# Constants

listObject()

def conddblib.listObject	(		session,
			name,
			snapshot = None
	)

Definition at line 607 of file conddblib.py.

def listObject(session, name, snapshot=None):
 
    is_tag = _exists(session, Tag.name, name)
    result = {}
    if is_tag:
        result['type'] = 'Tag'
        result['name'] = session.query(Tag).get(name).name
        result['timeType'] = session.query(Tag.time_type).\
                                     filter(Tag.name == name).\
                                     scalar()
 
        result['iovs'] = session.query(IOV.since, IOV.insertion_time, IOV.payload_hash, Payload.object_type).\
                join(IOV.payload).\
                filter(
                    IOV.tag_name == name,
                    _inserted_before(snapshot),
                ).\
                order_by(IOV.since.desc(), IOV.insertion_time.desc()).\
                from_self().\
                order_by(IOV.since, IOV.insertion_time).\
                all()
 
    try:
        is_global_tag = _exists(session, GlobalTag.name, name)
        if is_global_tag:
            result['type'] = 'GlobalTag'
            result['name'] = session.query(GlobalTag).get(name)
            result['tags'] = session.query(GlobalTagMap.record, GlobalTagMap.label, GlobalTagMap.tag_name).\
                                     filter(GlobalTagMap.global_tag_name == name).\
                                     order_by(GlobalTagMap.record, GlobalTagMap.label).\
                                     all()
    except sqlalchemy.exc.OperationalError:
        sys.stderr.write("No table for GlobalTags found in DB.\n\n")
 
    if not is_tag and not is_global_tag:
        raise Exception('There is no tag or global tag named %s in the database.' % name)
 
    return result
 

References _exists(), _inserted_before(), python.cmstools.all(), ALCARECOTkAlBeamHalo_cff.filter, join(), and hitfit.scalar().

make_dbtype()

def conddblib.make_dbtype	(		backendName,
			schemaName,
			baseType
	)

Definition at line 190 of file conddblib.py.

def make_dbtype( backendName, schemaName, baseType ):
    members = {}
    deps_reg = set()
    dbtype_name = '%s_%s' %(baseType.__name__,backendName)
    members['__tablename__'] = baseType.__tablename__
    members['__table_args__'] = None
    if schemaName is not None:
        members['__table_args__'] = {'schema': schemaName }
    for k,v in baseType.columns.items():
        if isinstance(v[0],DbRef):
            refColDbt = v[0].rtype.columns[v[0].rcol][0]
            pk = (True if v[1]==_Col.pk else False)
            if v[1]==_Col.pk:
                members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),primary_key=True)
            else:
                nullable = (False if v[1] == _Col.notNull else True)
                members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),nullable=nullable)
            if v[0].rtype.__name__ not in deps_reg:
                deps_reg.add(v[0].rtype.__name__)
                reftype_name = '%s_%s' %(v[0].rtype.__name__,backendName)
                members[(v[0].rtype.__name__).lower()] = sqlalchemy.orm.relationship(reftype_name)
        else:
            if v[1]==_Col.pk:
                members[k] = sqlalchemy.Column(v[0],primary_key=True)
            else:
                nullable = (True if v[1]==_Col.nullable else False)
                members[k] = sqlalchemy.Column(v[0],nullable=nullable)
    dbType = type(dbtype_name,(_Base,),members)
 
    if backendName not in db_models.keys():
        db_models[backendName] = {}
    db_models[backendName][baseType.__name__] = dbType
    return dbType
 

References fq_col().

Referenced by conddblib.Connection.get_dbtype().

make_url()

def conddblib.make_url	(		database = 'pro',
			read_only = True
	)

Definition at line 472 of file conddblib.py.

def make_url(database='pro',read_only = True):
    if database.startswith('sqlite:') or database.startswith('sqlite_file:'):
        ignore, database = database.split(':',1)
 
    if ':' in database and '://' not in database: # check if we really got a shortcut like "pro:<schema>" (and not a url like proto://...), if so, disentangle
        database, schema = database.split(':')
 
    officialdbs = { 
        # frontier 
        'pro' :         ('frontier','PromptProd',             { 'R': schema_name }, ),
        'arc' :         ('frontier','FrontierArc',            { 'R': schema_name }, ),
        'int' :         ('frontier','FrontierInt',            { 'R': schema_name }, ),
        'dev' :         ('frontier','FrontierPrep',           { 'R': schema_name }, ),
        # oracle adg
        'orapro':       ('oracle',         'cms_orcon_adg',   { 'R': dbreader_user_name }, ),
        'oraarc':       ('oracle',         'cmsarc_lb',       { 'R': dbreader_user_name }, ),
        # oracle masters
        'oraint':       ('oracle',         'cms_orcoff_int',  { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
        'oradev':       ('oracle',         'cms_orcoff_prep', { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
        'onlineorapro': ('oracle',         'cms_orcon_prod',  { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
        'onlineoraint': ('oracle',         'cmsintr_lb',      { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
    }
 
    if database in officialdbs.keys():
        key = ('R' if read_only else 'W')
        mapping = officialdbs[database]
        tech = mapping[0]
        service = mapping[1]
        schema_dict = mapping[2]
        if key in schema_dict.keys():
            database = _getCMSSQLAlchemyConnectionString(tech,service,schema_dict[key])
        else:
            raise Exception("Read-only database %s://%s cannot be accessed in update mode." %(tech,service))
 
    logging.debug('connection string set to "%s"' % database)
 
    try:
        url = sqlalchemy.engine.url.make_url(database)
    except sqlalchemy.exc.ArgumentError:
        url = sqlalchemy.engine.url.make_url('sqlite:///%s' % database)
    return url
 

References _getCMSSQLAlchemyConnectionString().

Referenced by getSessionOnMasterDB(), and helperFunctions.getTagsMap().

oracle_connection_string()

def conddblib.oracle_connection_string	(		db_service,
			db_schema
	)

Definition at line 139 of file conddblib.py.

def oracle_connection_string(db_service, db_schema ):
    return 'oracle://%s/%s'%(db_service,db_schema)
 

Referenced by connect().