conddblib Namespace Reference

Classes
class	_Col
class	BoostRunMap
class	Connection
class	DbRef
class	GlobalTag
class	GlobalTagMap
class	IOV
class	Payload
class	RunInfo
class	Synchronization
class	Tag
class	TagAuthorization
class	TagLog
class	TagMetadata
class	TimeType

Functions
def	_exists (session, primary_key, value)
def	_getCMSFrontierConnectionString (database)
def	_getCMSSQLAlchemyConnectionString (technology, service, schema_name)
def	_inserted_before (timestamp)
def	connect (url, authPath=None, verbose=0, as_admin=False)
def	fq_col (schema, table, column)
def	fq_name (schema_name, table_name)
def	getPayload (session, hash)
def	getSchema (tp)
def	getSessionOnMasterDB (session1, session2)
def	hash (data)
def	listObject (session, name, snapshot=None)
def	make_dbtype (backendName, schemaName, baseType)
def	make_url (database='pro', read_only=True)
def	oracle_connection_string (db_service, db_schema)

Detailed Description

CMS Conditions DB Python library.

Function Documentation

_exists()

def conddblib._exists (   session,   primary_key,   value  )

private

Definition at line 609 of file conddblib.py.

References submitPVResolutionJobs.count, and ALCARECOTkAlBeamHalo_cff.filter.

Referenced by listObject().

def _exists(session, primary_key, value):
    ret = None
    try: 
        ret = session.query(primary_key).\
            filter(primary_key == value).\
            count() != 0
    except sqlalchemy.exc.OperationalError:
        pass

    return ret

_getCMSFrontierConnectionString()

def conddblib._getCMSFrontierConnectionString (   database )

private

Definition at line 470 of file conddblib.py.

References communicate(), and digitizers_cfi.strip.

Referenced by _getCMSSQLAlchemyConnectionString().

def _getCMSFrontierConnectionString(database):
    import subprocess
    return subprocess.Popen(['cmsGetFnConnect', 'frontier://%s' % database], stdout = subprocess.PIPE).communicate()[0].strip()

_getCMSSQLAlchemyConnectionString()

def conddblib._getCMSSQLAlchemyConnectionString (   technology,   service,   schema_name  )

private

Definition at line 474 of file conddblib.py.

References _getCMSFrontierConnectionString().

Referenced by make_url().

def _getCMSSQLAlchemyConnectionString(technology,service,schema_name):
    if technology == 'frontier':
        import urllib
        import sys
        py3k = sys.version_info >= (3, 0)        
        if py3k:
            return '%s://@%s/%s' % ('oracle+frontier', urllib.parse.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
        else:
            return '%s://@%s/%s' % ('oracle+frontier', urllib.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
    elif technology == 'oracle':
        return '%s://%s@%s' % (technology, schema_name, service)

# Entry point

_inserted_before()

def conddblib._inserted_before (   timestamp )

private

To be used inside filter().

Definition at line 620 of file conddblib.py.

Referenced by listObject().

def _inserted_before(timestamp):
    '''To be used inside filter().
    '''

    if timestamp is None:
        # XXX: Returning None does not get optimized (skipped) by SQLAlchemy,
        #      and returning True does not work in Oracle (generates "and 1"
        #      which breaks Oracle but not SQLite). For the moment just use
        #      this dummy condition.
        return sqlalchemy.literal(True) == sqlalchemy.literal(True)

    return conddb.IOV.insertion_time <= _parse_timestamp(timestamp)

connect()

def conddblib.connect	(		url,
			authPath = None,
			verbose = 0,
			as_admin = False
	)

Returns a Connection instance to the CMS Condition DB.

See database_help for the description of the database parameter.

The verbosity level is as follows:

    0 = No output (default).
    1 = SQL statements issued, including their parameters.
    2 = In addition, results of the queries (all rows and the column headers).

Definition at line 533 of file conddblib.py.

References oracle_connection_string(), and str.

Referenced by helperFunctions.getTagsMap(), and conddblib.Connection.is_valid().

def connect(url, authPath=None, verbose=0, as_admin=False):
    '''Returns a Connection instance to the CMS Condition DB.

    See database_help for the description of the database parameter.

    The verbosity level is as follows:

        0 = No output (default).
        1 = SQL statements issued, including their parameters.
        2 = In addition, results of the queries (all rows and the column headers).
    '''

    check_admin = as_admin
    if url.drivername == 'oracle':
        if url.username is None:
            logging.error('Could not resolve the username for the connection %s. Please provide a connection in the format oracle://[user]:[pass]@[host]' %url )
            raise Exception('Connection format error: %s' %url )
        if url.password is None:
            if authPath is None:
                if authPathEnvVar in os.environ:
                    authPath = os.environ[authPathEnvVar]
            explicit_auth = False
            if authPath is not None:
                dbkey_path = os.path.join(authPath,dbkey_folder)
                if not os.path.exists(dbkey_path):
                    authFile = os.path.join(authPath,'.netrc')
                    if os.path.exists(authFile):
                        entryKey = url.host.lower()+"/"+url.username.lower()
                        logging.debug('Looking up credentials for %s in file %s ' %(entryKey,authFile) )
                        import netrc
                        params = netrc.netrc( authFile ).authenticators(entryKey)
                        if params is not None:
                            (username, account, password) = params
                            url.username = username
                            url.password = password
                        else:
                            msg = 'The entry %s has not been found in the .netrc file.' %entryKey
                            raise TypeError(msg)
                    else:
                        explicit_auth =True
                else:
                    import libCondDBPyBind11Interface as auth
                    role_code = auth.reader_role
                    if url.username == dbwriter_user_name:
                        role_code = auth.writer_role
                    if check_admin:
                        role_code = auth.admin_role
                    connection_string = oracle_connection_string(url.host.lower(),schema_name)
                    logging.debug('Using db key to get credentials for %s' %connection_string )
                    (dbuser,username,password) = auth.get_credentials_from_db(connection_string,role_code,authPath)
                    if username=='' or password=='':
                        raise Exception('No credentials found to connect on %s with the required access role.'%connection_string)
                    check_admin = False
                    url.username = username
                    url.password = password
            else:
                import getpass
                pwd = getpass.getpass('Password for %s: ' % str(url))
                if pwd is None or pwd == '':
                    pwd = getpass.getpass('Password for %s: ' % str(url))
                    if pwd is None or pwd == '':
                        raise Exception('Empty password provided, bailing out...')
                url.password = pwd
        if check_admin:
            raise Exception('Admin access has not been granted. Please provide a valid admin db-key.')
    if check_admin:
       raise Exception('Admin access is not available for technology "%s".' %url.drivername)
    if verbose >= 1:
        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

    if verbose >= 2:
        logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)

    return Connection(url)

fq_col()

def conddblib.fq_col	(		schema,
			table,
			column
	)

Definition at line 184 of file conddblib.py.

Referenced by make_dbtype().

def fq_col( schema, table, column ):
    fqn = '%s.%s' %(table, column)
    if schema is not None:
        fqn = '%s.%s' %(schema,fqn)
    return fqn

fq_name()

def conddblib.fq_name	(		schema_name,
			table_name
	)

Definition at line 166 of file conddblib.py.

def fq_name( schema_name, table_name ):
    name = table_name
    if schema_name is not None:
        name = '%s.%s' %(schema_name, table_name)
    return name

getPayload()

def conddblib.getPayload	(		session,
			hash
	)

Definition at line 672 of file conddblib.py.

References ALCARECOTkAlBeamHalo_cff.filter.

def getPayload(session, hash):
    # get payload from DB:
    data, payloadType = session.query(Payload.data, Payload.object_type).filter(Payload.hash == hash).one()
    return data

getSchema()

def conddblib.getSchema

(

tp

)

Definition at line 230 of file conddblib.py.

Referenced by conddblib.Connection.is_valid().

def getSchema(tp):
    if tp.__table_args__ is not None:
        return tp.__table_args__['schema']
    return None

getSessionOnMasterDB()

def conddblib.getSessionOnMasterDB	(		session1,
			session2
	)

Definition at line 448 of file conddblib.py.

References make_url().

def getSessionOnMasterDB( session1, session2 ):
    key = '%s/%s' 
    sessiondict = { }
    sessiondict[key %(session1._url.drivername,session1._url.host)] = session1
    sessiondict[key %(session2._url.drivername,session2._url.host)] = session2
    masterkey = key %('oracle',ONLINEORAPRO)
    if masterkey in sessiondict.keys():
        return sessiondict[masterkey]
    adgkey = key %('oracle',ORAPRO)
    if adgkey in sessiondict.keys():
        return sessiondict[adgkey]
    frontierkey = key %('frontier',PRO)
    if frontierkey in sessiondict.keys():
        return sessiondict[frontierkey]
    # default case: frontier on pro
    conn = Connection(make_url())
    session = conn.session()
    # is it required?
    session._conn = conn
    return session

# Connection helpers

hash()

def conddblib.hash

(

data

)

Definition at line 52 of file conddblib.py.

def hash(data):
    return hashlib.sha1(data.encode('ascii')).hexdigest()


# Constants

listObject()

def conddblib.listObject	(		session,
			name,
			snapshot = None
	)

Definition at line 633 of file conddblib.py.

References _exists(), _inserted_before(), python.cmstools.all(), ALCARECOTkAlBeamHalo_cff.filter, join(), and hitfit.scalar().

def listObject(session, name, snapshot=None):

    is_tag = _exists(session, Tag.name, name)
    result = {}
    if is_tag:
        result['type'] = 'Tag'
        result['name'] = session.query(Tag).get(name).name
        result['timeType'] = session.query(Tag.time_type).\
                                     filter(Tag.name == name).\
                                     scalar()

        result['iovs'] = session.query(IOV.since, IOV.insertion_time, IOV.payload_hash, Payload.object_type).\
                join(IOV.payload).\
                filter(
                    IOV.tag_name == name,
                    _inserted_before(snapshot),
                ).\
                order_by(IOV.since.desc(), IOV.insertion_time.desc()).\
                from_self().\
                order_by(IOV.since, IOV.insertion_time).\
                all()

    try:
        is_global_tag = _exists(session, GlobalTag.name, name)
        if is_global_tag:
            result['type'] = 'GlobalTag'
            result['name'] = session.query(GlobalTag).get(name)
            result['tags'] = session.query(GlobalTagMap.record, GlobalTagMap.label, GlobalTagMap.tag_name).\
                                     filter(GlobalTagMap.global_tag_name == name).\
                                     order_by(GlobalTagMap.record, GlobalTagMap.label).\
                                     all()
    except sqlalchemy.exc.OperationalError:
        sys.stderr.write("No table for GlobalTags found in DB.\n\n")

    if not is_tag and not is_global_tag:
        raise Exception('There is no tag or global tag named %s in the database.' % name)

    return result

make_dbtype()

def conddblib.make_dbtype	(		backendName,
			schemaName,
			baseType
	)

Definition at line 190 of file conddblib.py.

References fq_col().

Referenced by conddblib.Connection.get_dbtype().

def make_dbtype( backendName, schemaName, baseType ):
    members = {}
    deps_reg = set()
    dbtype_name = '%s_%s' %(baseType.__name__,backendName)
    members['__tablename__'] = baseType.__tablename__
    members['__table_args__'] = None
    if schemaName is not None:
        members['__table_args__'] = {'schema': schemaName }
    for k,v in baseType.columns.items():
        defColVal = None
        if len(v)==3:
            defColVal = v[2]
        if isinstance(v[0],DbRef):
            refColDbt = v[0].rtype.columns[v[0].rcol][0]
            pk = (True if v[1]==_Col.pk else False)
            if v[1]==_Col.pk:
                members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),primary_key=True)
            else:
                nullable = (False if v[1] == _Col.notNull else True)
                members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),nullable=nullable)
            if v[0].rtype.__name__ not in deps_reg:
                deps_reg.add(v[0].rtype.__name__)
                reftype_name = '%s_%s' %(v[0].rtype.__name__,backendName)
                members[(v[0].rtype.__name__).lower()] = sqlalchemy.orm.relationship(reftype_name)
        else:
            if v[1]==_Col.pk:
                members[k] = sqlalchemy.Column(v[0],primary_key=True)
            else:
                nullable = (True if v[1]==_Col.nullable else False)
                if defColVal is None:
                    members[k] = sqlalchemy.Column(v[0],nullable=nullable)
                else:
                    members[k] = sqlalchemy.Column(v[0],nullable=nullable, default=defColVal)
    dbType = type(dbtype_name,(_Base,),members)

    if backendName not in db_models.keys():
        db_models[backendName] = {}
    db_models[backendName][baseType.__name__] = dbType
    return dbType

make_url()

def conddblib.make_url	(		database = 'pro',
			read_only = True
	)

Definition at line 487 of file conddblib.py.

References _getCMSSQLAlchemyConnectionString().

Referenced by getSessionOnMasterDB(), and helperFunctions.getTagsMap().

def make_url(database='pro',read_only = True):
    if database.startswith('sqlite:') or database.startswith('sqlite_file:'):
        ignore, database = database.split(':',1)

    if ':' in database and '://' not in database: # check if we really got a shortcut like "pro:<schema>" (and not a url like proto://...), if so, disentangle
        database, schema = database.split(':')

    officialdbs = { 
        # frontier 
        'pro' :         ('frontier','PromptProd',             { 'R': schema_name }, ),
        'arc' :         ('frontier','FrontierArc',            { 'R': schema_name }, ),
        'int' :         ('frontier','FrontierInt',            { 'R': schema_name }, ),
        'dev' :         ('frontier','FrontierPrep',           { 'R': schema_name }, ),
        # oracle adg
        'orapro':       ('oracle',         'cms_orcon_adg',   { 'R': dbreader_user_name }, ),
        'oraarc':       ('oracle',         'cmsarc_lb',       { 'R': dbreader_user_name }, ),
        # oracle masters
        'oraint':       ('oracle',         'cms_orcoff_int',  { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
        'oradev':       ('oracle',         'cms_orcoff_prep', { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
        'onlineorapro': ('oracle',         'cms_orcon_prod',  { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
        'onlineoraint': ('oracle',         'cmsintr_lb',      { 'R': dbreader_user_name,
                                                                'W': dbwriter_user_name }, ),
    }

    if database in officialdbs.keys():
        key = ('R' if read_only else 'W')
        mapping = officialdbs[database]
        tech = mapping[0]
        service = mapping[1]
        schema_dict = mapping[2]
        if key in schema_dict.keys():
            database = _getCMSSQLAlchemyConnectionString(tech,service,schema_dict[key])
        else:
            raise Exception("Read-only database %s://%s cannot be accessed in update mode." %(tech,service))

    logging.debug('connection string set to "%s"' % database)

    try:
        url = sqlalchemy.engine.url.make_url(database)
    except sqlalchemy.exc.ArgumentError:
        url = sqlalchemy.engine.url.make_url('sqlite:///%s' % database)
    return url

oracle_connection_string()

def conddblib.oracle_connection_string	(		db_service,
			db_schema
	)

Definition at line 139 of file conddblib.py.

Referenced by connect().

def oracle_connection_string(db_service, db_schema ):
    return 'oracle://%s/%s'%(db_service,db_schema)

---

Generated for CMSSW Reference Manual by  1.8.14