Classes
class	_Col

class	BoostRunMap

class	Connection

class	DbRef

class	GlobalTag

class	GlobalTagMap

class	IOV

class	Payload

class	RunInfo

class	Synchronization

class	Tag

class	TagAuthorization

class	TagLog

class	TagMetadata

class	TimeType

Functions
def	_exists (session, primary_key, value)

def	_getCMSFrontierConnectionString (database)

def	_getCMSSQLAlchemyConnectionString (technology, service, schema_name)

def	_inserted_before (timestamp)

def	connect (url, authPath=None, verbose=0, as_admin=False)

def	fq_col (schema, table, column)

def	fq_name (schema_name, table_name)

def	getPayload (session, hash)

def	getSchema (tp)

def	getSessionOnMasterDB (session1, session2)

def	hash (data)

def	listObject (session, name, snapshot=None)

def	make_dbtype (backendName, schemaName, baseType)

def	make_url (database='pro', read_only=True)

def	oracle_connection_string (db_service, db_schema)

Variables
string	__author__ = 'Miguel Ojeda'

string	__copyright__ = 'Copyright 2013, CERN'

list	__credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']

string	__email__ = 'giacomo.govi@cern.ch'

string	__license__ = 'Unknown'

string	__maintainer__ = 'Giacomo Govi'

	_Base = sqlalchemy.ext.declarative.declarative_base()

string	ARC = 'FrontierArc'

string	authPathEnvVar = 'COND_AUTH_PATH'

string	contact_help = 'If you need assistance, please write an email to %s and %s. If you need immediate/urgent assistance, you can call the Offline DB expert on call (%s).' % (offline_db_experts_email, web_experts_email, offline_db_experts_phone)

string	database_help

dictionary	db_models = {}

string	dbkey_filename = 'db.key'

	dbkey_folder = os.path.join('.cms_cond',dbkey_filename)

string	dbreader_user_name = 'cms_cond_general_r'

string	dbuser_name = 'cms_conditions'

string	dbwriter_user_name = 'cms_cond_general_w'

int	description_length = 4000

string	DEV = 'FrontierPrep'

string	empty_label = '-'

	hash_length = len(hash(''))

string	INT = 'FrontierInt'

	logger = logging.getLogger(__name__)

int	name_length = 100

string	offline_db_experts_email = 'cms-offlinedb-exp@cern.ch'

string	offline_db_experts_phone = '+41 22 76 70817, or 70817 from CERN; check https://twiki.cern.ch/twiki/bin/viewauth/CMS/DBShifterHelpPage if it does not work; availability depends on the state of the LHC'

string	ONLINEORAINT = 'cmsintr_lb'

string	ONLINEORAPRO = 'cms_orcon_prod'

string	ORAARC = 'cmsarc_lb'

string	ORADEV = 'cms_orcoff_prep'

string	ORAINT = 'cms_orcoff_int'

string	ORAPRO = 'cms_orcon_adg'

string	PRO = 'PromptProd'

string	schema_name = 'cms_conditions'

	synch_list = list(x.value for x in list(Synchronization))

string	web_experts_email = 'cms-cond-dev@cern.ch'

Detailed Description

CMS Conditions DB Python library.

Function Documentation

◆ _exists()

def conddblib._exists	(	session,
		primary_key,
		value
	)

private

Definition at line 609 of file conddblib.py.

References submitPVResolutionJobs.count, and ALCARECOTkAlBeamHalo_cff.filter.

Referenced by listObject().

 def _exists(session, primary_key, value):
     ret = None
     try: 
         ret = session.query(primary_key).\
             filter(primary_key == value).\
             count() != 0
     except sqlalchemy.exc.OperationalError:
         pass
 
     return ret
 

◆ _getCMSFrontierConnectionString()

def conddblib._getCMSFrontierConnectionString ( database )

private

Definition at line 470 of file conddblib.py.

References communicate(), and nano_mu_digi_cff.strip.

Referenced by _getCMSSQLAlchemyConnectionString().

 def _getCMSFrontierConnectionString(database):
     import subprocess
     return subprocess.Popen(['cmsGetFnConnect', 'frontier://%s' % database], stdout = subprocess.PIPE).communicate()[0].strip()
 

◆ _getCMSSQLAlchemyConnectionString()

def conddblib._getCMSSQLAlchemyConnectionString	(	technology,
		service,
		schema_name
	)

private

Definition at line 474 of file conddblib.py.

References _getCMSFrontierConnectionString().

Referenced by make_url().

 def _getCMSSQLAlchemyConnectionString(technology,service,schema_name):
     if technology == 'frontier':
         import urllib
         import sys
         py3k = sys.version_info >= (3, 0)        
         if py3k:
             return '%s://@%s/%s' % ('oracle+frontier', urllib.parse.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
         else:
             return '%s://@%s/%s' % ('oracle+frontier', urllib.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
     elif technology == 'oracle':
         return '%s://%s@%s' % (technology, schema_name, service)
 
 # Entry point

◆ _inserted_before()

def conddblib._inserted_before ( timestamp )

private

To be used inside filter().

Definition at line 620 of file conddblib.py.

References conddb2hdf5._parse_timestamp().

Referenced by listObject().

 def _inserted_before(timestamp):
     '''To be used inside filter().
     '''
 
     if timestamp is None:
         # XXX: Returning None does not get optimized (skipped) by SQLAlchemy,
         #      and returning True does not work in Oracle (generates "and 1"
         #      which breaks Oracle but not SQLite). For the moment just use
         #      this dummy condition.
         return sqlalchemy.literal(True) == sqlalchemy.literal(True)
 
     return conddb.IOV.insertion_time <= _parse_timestamp(timestamp)
 

◆ connect()

def conddblib.connect	(	url,
		authPath = `None`,
		verbose = `0`,
		as_admin = `False`
	)

Returns a Connection instance to the CMS Condition DB.

See database_help for the description of the database parameter.

The verbosity level is as follows:

    0 = No output (default).
    1 = SQL statements issued, including their parameters.
    2 = In addition, results of the queries (all rows and the column headers).

Definition at line 533 of file conddblib.py.

References oracle_connection_string(), and str.

Referenced by conddblib.Connection.is_valid().

 def connect(url, authPath=None, verbose=0, as_admin=False):
     '''Returns a Connection instance to the CMS Condition DB.
 
     See database_help for the description of the database parameter.
 
     The verbosity level is as follows:
 
         0 = No output (default).
         1 = SQL statements issued, including their parameters.
         2 = In addition, results of the queries (all rows and the column headers).
     '''
 
     check_admin = as_admin
     if url.drivername == 'oracle':
         if url.username is None:
             logging.error('Could not resolve the username for the connection %s. Please provide a connection in the format oracle://[user]:[pass]@[host]' %url )
             raise Exception('Connection format error: %s' %url )
         if url.password is None:
             if authPath is None:
                 if authPathEnvVar in os.environ:
                     authPath = os.environ[authPathEnvVar]
             explicit_auth = False
             if authPath is not None:
                 dbkey_path = os.path.join(authPath,dbkey_folder)
                 if not os.path.exists(dbkey_path):
                     authFile = os.path.join(authPath,'.netrc')
                     if os.path.exists(authFile):
                         entryKey = url.host.lower()+"/"+url.username.lower()
                         logging.debug('Looking up credentials for %s in file %s ' %(entryKey,authFile) )
                         import netrc
                         params = netrc.netrc( authFile ).authenticators(entryKey)
                         if params is not None:
                             (username, account, password) = params
                             url.username = username
                             url.password = password
                         else:
                             msg = 'The entry %s has not been found in the .netrc file.' %entryKey
                             raise TypeError(msg)
                     else:
                         explicit_auth =True
                 else:
                     import libCondDBPyBind11Interface as auth
                     role_code = auth.reader_role
                     if url.username == dbwriter_user_name:
                         role_code = auth.writer_role
                     if check_admin:
                         role_code = auth.admin_role
                     connection_string = oracle_connection_string(url.host.lower(),schema_name)
                     logging.debug('Using db key to get credentials for %s' %connection_string )
                     (dbuser,username,password) = auth.get_credentials_from_db(connection_string,role_code,authPath)
                     if username=='' or password=='':
                         raise Exception('No credentials found to connect on %s with the required access role.'%connection_string)
                     check_admin = False
                     url.username = username
                     url.password = password
             else:
                 import getpass
                 pwd = getpass.getpass('Password for %s: ' % str(url))
                 if pwd is None or pwd == '':
                     pwd = getpass.getpass('Password for %s: ' % str(url))
                     if pwd is None or pwd == '':
                         raise Exception('Empty password provided, bailing out...')
                 url.password = pwd
         if check_admin:
             raise Exception('Admin access has not been granted. Please provide a valid admin db-key.')
     if check_admin:
        raise Exception('Admin access is not available for technology "%s".' %url.drivername)
     if verbose >= 1:
         logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
 
     if verbose >= 2:
         logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
 
     return Connection(url)
 
 

◆ fq_col()

def conddblib.fq_col	(	schema,
		table,
		column
	)

Definition at line 184 of file conddblib.py.

Referenced by make_dbtype().

 def fq_col( schema, table, column ):
     fqn = '%s.%s' %(table, column)
     if schema is not None:
         fqn = '%s.%s' %(schema,fqn)
     return fqn
 

◆ fq_name()

def conddblib.fq_name	(	schema_name,
		table_name
	)

Definition at line 166 of file conddblib.py.

 def fq_name( schema_name, table_name ):
     name = table_name
     if schema_name is not None:
         name = '%s.%s' %(schema_name, table_name)
     return name
 

◆ getPayload()

def conddblib.getPayload	(	session,
		hash
	)

Definition at line 672 of file conddblib.py.

References ALCARECOTkAlBeamHalo_cff.filter.

Referenced by TotemT2FramePosition.printXML().

 def getPayload(session, hash):
     # get payload from DB:
     data, payloadType = session.query(Payload.data, Payload.object_type).filter(Payload.hash == hash).one()
     return data

◆ getSchema()

def conddblib.getSchema ( tp )

Definition at line 230 of file conddblib.py.

Referenced by conddblib.Connection.is_valid().

 def getSchema(tp):
     if tp.__table_args__ is not None:
         return tp.__table_args__['schema']
     return None
 

◆ getSessionOnMasterDB()

def conddblib.getSessionOnMasterDB	(	session1,
		session2
	)

Definition at line 448 of file conddblib.py.

References make_url().

 def getSessionOnMasterDB( session1, session2 ):
     key = '%s/%s' 
     sessiondict = { }
     sessiondict[key %(session1._url.drivername,session1._url.host)] = session1
     sessiondict[key %(session2._url.drivername,session2._url.host)] = session2
     masterkey = key %('oracle',ONLINEORAPRO)
     if masterkey in sessiondict.keys():
         return sessiondict[masterkey]
     adgkey = key %('oracle',ORAPRO)
     if adgkey in sessiondict.keys():
         return sessiondict[adgkey]
     frontierkey = key %('frontier',PRO)
     if frontierkey in sessiondict.keys():
         return sessiondict[frontierkey]
     # default case: frontier on pro
     conn = Connection(make_url())
     session = conn.session()
     # is it required?
     session._conn = conn
     return session
 
 # Connection helpers

◆ hash()

def conddblib.hash ( data )

Definition at line 52 of file conddblib.py.

 def hash(data):
     return hashlib.sha1(data.encode('ascii')).hexdigest()
 
 
 # Constants

◆ listObject()

def conddblib.listObject	(	session,
		name,
		snapshot = `None`
	)

Definition at line 633 of file conddblib.py.

References _exists(), _inserted_before(), python.cmstools.all(), ALCARECOTkAlBeamHalo_cff.filter, and join().

 def listObject(session, name, snapshot=None):
 
     is_tag = _exists(session, Tag.name, name)
     result = {}
     if is_tag:
         result['type'] = 'Tag'
         result['name'] = session.query(Tag).get(name).name
         result['timeType'] = session.query(Tag.time_type).\
                                      filter(Tag.name == name).\
                                      scalar()
 
         result['iovs'] = session.query(IOV.since, IOV.insertion_time, IOV.payload_hash, Payload.object_type).\
                 join(IOV.payload).\
                 filter(
                     IOV.tag_name == name,
                     _inserted_before(snapshot),
                 ).\
                 order_by(IOV.since.desc(), IOV.insertion_time.desc()).\
                 from_self().\
                 order_by(IOV.since, IOV.insertion_time).\
                 all()
 
     try:
         is_global_tag = _exists(session, GlobalTag.name, name)
         if is_global_tag:
             result['type'] = 'GlobalTag'
             result['name'] = session.query(GlobalTag).get(name)
             result['tags'] = session.query(GlobalTagMap.record, GlobalTagMap.label, GlobalTagMap.tag_name).\
                                      filter(GlobalTagMap.global_tag_name == name).\
                                      order_by(GlobalTagMap.record, GlobalTagMap.label).\
                                      all()
     except sqlalchemy.exc.OperationalError:
         sys.stderr.write("No table for GlobalTags found in DB.\n\n")
 
     if not is_tag and not is_global_tag:
         raise Exception('There is no tag or global tag named %s in the database.' % name)
 
     return result
 

◆ make_dbtype()

def conddblib.make_dbtype	(	backendName,
		schemaName,
		baseType
	)

Definition at line 190 of file conddblib.py.

References fq_col().

Referenced by conddblib.Connection.get_dbtype().

 def make_dbtype( backendName, schemaName, baseType ):
     members = {}
     deps_reg = set()
     dbtype_name = '%s_%s' %(baseType.__name__,backendName)
     members['__tablename__'] = baseType.__tablename__
     members['__table_args__'] = None
     if schemaName is not None:
         members['__table_args__'] = {'schema': schemaName }
     for k,v in baseType.columns.items():
         defColVal = None
         if len(v)==3:
             defColVal = v[2]
         if isinstance(v[0],DbRef):
             refColDbt = v[0].rtype.columns[v[0].rcol][0]
             pk = (True if v[1]==_Col.pk else False)
             if v[1]==_Col.pk:
                 members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),primary_key=True)
             else:
                 nullable = (False if v[1] == _Col.notNull else True)
                 members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),nullable=nullable)
             if v[0].rtype.__name__ not in deps_reg:
                 deps_reg.add(v[0].rtype.__name__)
                 reftype_name = '%s_%s' %(v[0].rtype.__name__,backendName)
                 members[(v[0].rtype.__name__).lower()] = sqlalchemy.orm.relationship(reftype_name)
         else:
             if v[1]==_Col.pk:
                 members[k] = sqlalchemy.Column(v[0],primary_key=True)
             else:
                 nullable = (True if v[1]==_Col.nullable else False)
                 if defColVal is None:
                     members[k] = sqlalchemy.Column(v[0],nullable=nullable)
                 else:
                     members[k] = sqlalchemy.Column(v[0],nullable=nullable, default=defColVal)
     dbType = type(dbtype_name,(_Base,),members)
 
     if backendName not in db_models.keys():
         db_models[backendName] = {}
     db_models[backendName][baseType.__name__] = dbType
     return dbType
 

◆ make_url()

def conddblib.make_url	(	database = `'pro'`,
		read_only = `True`
	)

Definition at line 487 of file conddblib.py.

References _getCMSSQLAlchemyConnectionString().

Referenced by getSessionOnMasterDB().

 def make_url(database='pro',read_only = True):
     if database.startswith('sqlite:') or database.startswith('sqlite_file:'):
         ignore, database = database.split(':',1)
 
     if ':' in database and '://' not in database: # check if we really got a shortcut like "pro:<schema>" (and not a url like proto://...), if so, disentangle
         database, schema = database.split(':')
 
     officialdbs = { 
         # frontier 
         'pro' :         ('frontier','PromptProd',             { 'R': schema_name }, ),
         'arc' :         ('frontier','FrontierArc',            { 'R': schema_name }, ),
         'int' :         ('frontier','FrontierInt',            { 'R': schema_name }, ),
         'dev' :         ('frontier','FrontierPrep',           { 'R': schema_name }, ),
         # oracle adg
         'orapro':       ('oracle',         'cms_orcon_adg',   { 'R': dbreader_user_name }, ),
         'oraarc':       ('oracle',         'cmsarc_lb',       { 'R': dbreader_user_name }, ),
         # oracle masters
         'oraint':       ('oracle',         'cms_orcoff_int',  { 'R': dbreader_user_name,
                                                                 'W': dbwriter_user_name }, ),
         'oradev':       ('oracle',         'cms_orcoff_prep', { 'R': dbreader_user_name,
                                                                 'W': dbwriter_user_name }, ),
         'onlineorapro': ('oracle',         'cms_orcon_prod',  { 'R': dbreader_user_name,
                                                                 'W': dbwriter_user_name }, ),
         'onlineoraint': ('oracle',         'cmsintr_lb',      { 'R': dbreader_user_name,
                                                                 'W': dbwriter_user_name }, ),
     }
 
     if database in officialdbs.keys():
         key = ('R' if read_only else 'W')
         mapping = officialdbs[database]
         tech = mapping[0]
         service = mapping[1]
         schema_dict = mapping[2]
         if key in schema_dict.keys():
             database = _getCMSSQLAlchemyConnectionString(tech,service,schema_dict[key])
         else:
             raise Exception("Read-only database %s://%s cannot be accessed in update mode." %(tech,service))
 
     logging.debug('connection string set to "%s"' % database)
 
     try:
         url = sqlalchemy.engine.url.make_url(database)
     except sqlalchemy.exc.ArgumentError:
         url = sqlalchemy.engine.url.make_url('sqlite:///%s' % database)
     return url
 

◆ oracle_connection_string()

def conddblib.oracle_connection_string	(	db_service,
		db_schema
	)

Definition at line 139 of file conddblib.py.

Referenced by connect().

 def oracle_connection_string(db_service, db_schema ):
     return 'oracle://%s/%s'%(db_service,db_schema)
 

Variable Documentation

◆ author

string conddblib.__author__ = 'Miguel Ojeda'

private

Definition at line 4 of file conddblib.py.

◆ copyright

private

Definition at line 5 of file conddblib.py.

◆ credits

list conddblib.__credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']

private

Definition at line 6 of file conddblib.py.

◆ email

string conddblib.__email__ = 'giacomo.govi@cern.ch'

private

Definition at line 9 of file conddblib.py.

◆ license

string conddblib.__license__ = 'Unknown'

private

Definition at line 7 of file conddblib.py.

◆ maintainer

string conddblib.__maintainer__ = 'Giacomo Govi'

private

Definition at line 8 of file conddblib.py.

◆ _Base

conddblib._Base = sqlalchemy.ext.declarative.declarative_base()

private

Definition at line 164 of file conddblib.py.

◆ ARC

string conddblib.ARC = 'FrontierArc'

Definition at line 34 of file conddblib.py.

◆ authPathEnvVar

string conddblib.authPathEnvVar = 'COND_AUTH_PATH'

Definition at line 28 of file conddblib.py.

◆ contact_help

string conddblib.contact_help = 'If you need assistance, please write an email to %s and %s. If you need immediate/urgent assistance, you can call the Offline DB expert on call (%s).' % (offline_db_experts_email, web_experts_email, offline_db_experts_phone)

Definition at line 67 of file conddblib.py.

◆ database_help

string conddblib.database_help

Definition at line 68 of file conddblib.py.

◆ db_models

dictionary conddblib.db_models = {}

Definition at line 172 of file conddblib.py.

◆ dbkey_filename

string conddblib.dbkey_filename = 'db.key'

Definition at line 29 of file conddblib.py.

◆ dbkey_folder

conddblib.dbkey_folder = os.path.join('.cms_cond',dbkey_filename)

Definition at line 30 of file conddblib.py.

◆ dbreader_user_name

string conddblib.dbreader_user_name = 'cms_cond_general_r'

Definition at line 23 of file conddblib.py.

◆ dbuser_name

string conddblib.dbuser_name = 'cms_conditions'

Definition at line 22 of file conddblib.py.

◆ dbwriter_user_name

string conddblib.dbwriter_user_name = 'cms_cond_general_w'

Definition at line 24 of file conddblib.py.

◆ description_length

int conddblib.description_length = 4000

Definition at line 60 of file conddblib.py.

◆ DEV

string conddblib.DEV = 'FrontierPrep'

Definition at line 36 of file conddblib.py.

◆ empty_label

string conddblib.empty_label = '-'

Definition at line 57 of file conddblib.py.

◆ hash_length

conddblib.hash_length = len(hash(''))

Definition at line 61 of file conddblib.py.

◆ INT

string conddblib.INT = 'FrontierInt'

Definition at line 35 of file conddblib.py.

◆ logger

conddblib.logger = logging.getLogger(__name__)

Definition at line 25 of file conddblib.py.

Referenced by cond::service::OnlineDBOutputService.getLastLumiProcessed(), cond::persistency::MsgDispatcher.MsgDispatcher(), and cscdqm::Configuration.printStats().

◆ name_length

int conddblib.name_length = 100

Definition at line 59 of file conddblib.py.

◆ offline_db_experts_email

string conddblib.offline_db_experts_email = 'cms-offlinedb-exp@cern.ch'

Definition at line 64 of file conddblib.py.

◆ offline_db_experts_phone

string conddblib.offline_db_experts_phone = '+41 22 76 70817, or 70817 from CERN; check https://twiki.cern.ch/twiki/bin/viewauth/CMS/DBShifterHelpPage if it does not work; availability depends on the state of the LHC'

Definition at line 65 of file conddblib.py.

◆ ONLINEORAINT

string conddblib.ONLINEORAINT = 'cmsintr_lb'

Definition at line 44 of file conddblib.py.

◆ ONLINEORAPRO

string conddblib.ONLINEORAPRO = 'cms_orcon_prod'

Definition at line 43 of file conddblib.py.

◆ ORAARC

string conddblib.ORAARC = 'cmsarc_lb'

Definition at line 39 of file conddblib.py.

◆ ORADEV

string conddblib.ORADEV = 'cms_orcoff_prep'

Definition at line 42 of file conddblib.py.

◆ ORAINT

string conddblib.ORAINT = 'cms_orcoff_int'

Definition at line 41 of file conddblib.py.

◆ ORAPRO

string conddblib.ORAPRO = 'cms_orcon_adg'

Definition at line 38 of file conddblib.py.

◆ PRO

string conddblib.PRO = 'PromptProd'

Definition at line 33 of file conddblib.py.

◆ schema_name

string conddblib.schema_name = 'cms_conditions'

Definition at line 21 of file conddblib.py.

◆ synch_list

conddblib.synch_list = list(x.value for x in list(Synchronization))

Definition at line 153 of file conddblib.py.

◆ web_experts_email

string conddblib.web_experts_email = 'cms-cond-dev@cern.ch'

Definition at line 63 of file conddblib.py.

Classes

Functions

Variables

Detailed Description

Function Documentation

◆ _exists()

◆ _getCMSFrontierConnectionString()

◆ _getCMSSQLAlchemyConnectionString()

◆ _inserted_before()

◆ connect()

◆ fq_col()

◆ fq_name()

◆ getPayload()

◆ getSchema()

◆ getSessionOnMasterDB()

◆ hash()

◆ listObject()

◆ make_dbtype()

◆ make_url()

◆ oracle_connection_string()

Variable Documentation

◆ __author__

◆ __copyright__

◆ __credits__

◆ __email__

◆ __license__

◆ __maintainer__

◆ _Base

◆ ARC

◆ authPathEnvVar

◆ contact_help

◆ database_help

◆ db_models

◆ dbkey_filename

◆ dbkey_folder

◆ dbreader_user_name

◆ dbuser_name

◆ dbwriter_user_name

◆ description_length

◆ DEV

◆ empty_label

◆ hash_length

◆ INT

◆ logger

◆ name_length

◆ offline_db_experts_email

◆ offline_db_experts_phone

◆ ONLINEORAINT

◆ ONLINEORAPRO

◆ ORAARC

◆ ORADEV

◆ ORAINT

◆ ORAPRO

◆ PRO

◆ schema_name

◆ synch_list

◆ web_experts_email

◆ author

◆ copyright

◆ credits

◆ email

◆ license

◆ maintainer