CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
conddblib Namespace Reference

Classes

class  _Col
 
class  BoostRunMap
 
class  Connection
 
class  DbRef
 
class  GlobalTag
 
class  GlobalTagMap
 
class  IOV
 
class  Payload
 
class  RunInfo
 
class  Synchronization
 
class  Tag
 
class  TagAuthorization
 
class  TagLog
 
class  TagMetadata
 
class  TimeType
 

Functions

def _exists
 
def _getCMSFrontierConnectionString
 
def _getCMSSQLAlchemyConnectionString
 
def _inserted_before
 
def connect
 
def fq_col
 
def fq_name
 
def getPayload
 
def getSchema
 
def getSessionOnMasterDB
 
def hash
 
def listObject
 
def make_dbtype
 
def make_url
 
def oracle_connection_string
 

Variables

string __author__ = 'Miguel Ojeda'
 
string __copyright__ = 'Copyright 2013, CERN'
 
list __credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']
 
string __email__ = 'giacomo.govi@cern.ch'
 
string __license__ = 'Unknown'
 
string __maintainer__ = 'Giacomo Govi'
 
tuple _Base = sqlalchemy.ext.declarative.declarative_base()
 
string ARC = 'FrontierArc'
 
string authPathEnvVar = 'COND_AUTH_PATH'
 
string contact_help = 'If you need assistance, please write an email to %s and %s. If you need immediate/urgent assistance, you can call the Offline DB expert on call (%s).'
 
string database_help
 
dictionary db_models = {}
 
string dbkey_filename = 'db.key'
 
tuple dbkey_folder = os.path.join('.cms_cond',dbkey_filename)
 
string dbreader_user_name = 'cms_cond_general_r'
 
string dbuser_name = 'cms_conditions'
 
string dbwriter_user_name = 'cms_cond_general_w'
 
int description_length = 4000
 
string DEV = 'FrontierPrep'
 
string empty_label = '-'
 
tuple hash_length = len(hash(''))
 
string INT = 'FrontierInt'
 
tuple logger = logging.getLogger(__name__)
 
int name_length = 100
 
string offline_db_experts_email = 'cms-offlinedb-exp@cern.ch'
 
string offline_db_experts_phone = '+41 22 76 70817, or 70817 from CERN; check https://twiki.cern.ch/twiki/bin/viewauth/CMS/DBShifterHelpPage if it does not work; availability depends on the state of the LHC'
 
string ONLINEORAINT = 'cmsintr_lb'
 
string ONLINEORAPRO = 'cms_orcon_prod'
 
string ORAARC = 'cmsarc_lb'
 
string ORADEV = 'cms_orcoff_prep'
 
string ORAINT = 'cms_orcoff_int'
 
string ORAPRO = 'cms_orcon_adg'
 
string PRO = 'PromptProd'
 
string schema_name = 'cms_conditions'
 
tuple synch_list = list(x.value for x in list(Synchronization))
 
string web_experts_email = 'cms-cond-dev@cern.ch'
 

Detailed Description

CMS Conditions DB Python library.

Function Documentation

def conddblib._exists (   session,
  primary_key,
  value 
)
private

Definition at line 609 of file conddblib.py.

References submitPVResolutionJobs.count, and alcazmumu_cfi.filter.

Referenced by listObject().

610 def _exists(session, primary_key, value):
611  ret = None
612  try:
613  ret = session.query(primary_key).\
614  filter(primary_key == value).\
615  count() != 0
616  except sqlalchemy.exc.OperationalError:
617  pass
618 
619  return ret
def _exists
Definition: conddblib.py:609
def conddblib._getCMSFrontierConnectionString (   database)
private

Definition at line 470 of file conddblib.py.

References communicate(), and digitizers_cfi.strip.

Referenced by _getCMSSQLAlchemyConnectionString().

471 def _getCMSFrontierConnectionString(database):
472  import subprocess
473  return subprocess.Popen(['cmsGetFnConnect', 'frontier://%s' % database], stdout = subprocess.PIPE).communicate()[0].strip()
static void * communicate(void *obj)
Definition: DQMNet.cc:1049
def _getCMSFrontierConnectionString
Definition: conddblib.py:470
def conddblib._getCMSSQLAlchemyConnectionString (   technology,
  service,
  schema_name 
)
private

Definition at line 474 of file conddblib.py.

References _getCMSFrontierConnectionString().

Referenced by make_url().

475 def _getCMSSQLAlchemyConnectionString(technology,service,schema_name):
476  if technology == 'frontier':
477  import urllib
478  import sys
479  py3k = sys.version_info >= (3, 0)
480  if py3k:
481  return '%s://@%s/%s' % ('oracle+frontier', urllib.parse.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
482  else:
483  return '%s://@%s/%s' % ('oracle+frontier', urllib.quote_plus(_getCMSFrontierConnectionString(service)), schema_name )
484  elif technology == 'oracle':
485  return '%s://%s@%s' % (technology, schema_name, service)
486 
# Entry point
def _getCMSSQLAlchemyConnectionString
Definition: conddblib.py:474
def _getCMSFrontierConnectionString
Definition: conddblib.py:470
def conddblib._inserted_before (   timestamp)
private
To be used inside filter().

Definition at line 620 of file conddblib.py.

Referenced by listObject().

621 def _inserted_before(timestamp):
622  '''To be used inside filter().
623  '''
624 
625  if timestamp is None:
626  # XXX: Returning None does not get optimized (skipped) by SQLAlchemy,
627  # and returning True does not work in Oracle (generates "and 1"
628  # which breaks Oracle but not SQLite). For the moment just use
629  # this dummy condition.
630  return sqlalchemy.literal(True) == sqlalchemy.literal(True)
631 
632  return conddb.IOV.insertion_time <= _parse_timestamp(timestamp)
def _inserted_before
Definition: conddblib.py:620
def conddblib.connect (   url,
  authPath = None,
  verbose = 0,
  as_admin = False 
)
Returns a Connection instance to the CMS Condition DB.

See database_help for the description of the database parameter.

The verbosity level is as follows:

    0 = No output (default).
    1 = SQL statements issued, including their parameters.
    2 = In addition, results of the queries (all rows and the column headers).

Definition at line 533 of file conddblib.py.

References oracle_connection_string(), and str.

Referenced by helperFunctions.getTagsMap().

534 def connect(url, authPath=None, verbose=0, as_admin=False):
535  '''Returns a Connection instance to the CMS Condition DB.
536 
537  See database_help for the description of the database parameter.
538 
539  The verbosity level is as follows:
540 
541  0 = No output (default).
542  1 = SQL statements issued, including their parameters.
543  2 = In addition, results of the queries (all rows and the column headers).
544  '''
545 
546  check_admin = as_admin
547  if url.drivername == 'oracle':
548  if url.username is None:
549  logging.error('Could not resolve the username for the connection %s. Please provide a connection in the format oracle://[user]:[pass]@[host]' %url )
550  raise Exception('Connection format error: %s' %url )
551  if url.password is None:
552  if authPath is None:
553  if authPathEnvVar in os.environ:
554  authPath = os.environ[authPathEnvVar]
555  explicit_auth = False
556  if authPath is not None:
557  dbkey_path = os.path.join(authPath,dbkey_folder)
558  if not os.path.exists(dbkey_path):
559  authFile = os.path.join(authPath,'.netrc')
560  if os.path.exists(authFile):
561  entryKey = url.host.lower()+"/"+url.username.lower()
562  logging.debug('Looking up credentials for %s in file %s ' %(entryKey,authFile) )
563  import netrc
564  params = netrc.netrc( authFile ).authenticators(entryKey)
565  if params is not None:
566  (username, account, password) = params
567  url.username = username
568  url.password = password
569  else:
570  msg = 'The entry %s has not been found in the .netrc file.' %entryKey
571  raise TypeError(msg)
572  else:
573  explicit_auth =True
574  else:
575  import libCondDBPyBind11Interface as auth
576  role_code = auth.reader_role
577  if url.username == dbwriter_user_name:
578  role_code = auth.writer_role
579  if check_admin:
580  role_code = auth.admin_role
581  connection_string = oracle_connection_string(url.host.lower(),schema_name)
582  logging.debug('Using db key to get credentials for %s' %connection_string )
583  (dbuser,username,password) = auth.get_credentials_from_db(connection_string,role_code,authPath)
584  if username=='' or password=='':
585  raise Exception('No credentials found to connect on %s with the required access role.'%connection_string)
586  check_admin = False
587  url.username = username
588  url.password = password
589  else:
590  import getpass
591  pwd = getpass.getpass('Password for %s: ' % str(url))
592  if pwd is None or pwd == '':
593  pwd = getpass.getpass('Password for %s: ' % str(url))
594  if pwd is None or pwd == '':
595  raise Exception('Empty password provided, bailing out...')
596  url.password = pwd
597  if check_admin:
598  raise Exception('Admin access has not been granted. Please provide a valid admin db-key.')
599  if check_admin:
600  raise Exception('Admin access is not available for technology "%s".' %url.drivername)
601  if verbose >= 1:
602  logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
603 
604  if verbose >= 2:
605  logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
606 
607  return Connection(url)
608 
def oracle_connection_string
Definition: conddblib.py:139
def connect
Definition: conddblib.py:533
#define str(s)
def conddblib.fq_col (   schema,
  table,
  column 
)

Definition at line 184 of file conddblib.py.

Referenced by make_dbtype().

185 def fq_col( schema, table, column ):
186  fqn = '%s.%s' %(table, column)
187  if schema is not None:
188  fqn = '%s.%s' %(schema,fqn)
189  return fqn
def fq_col
Definition: conddblib.py:184
def conddblib.fq_name (   schema_name,
  table_name 
)

Definition at line 166 of file conddblib.py.

167 def fq_name( schema_name, table_name ):
168  name = table_name
169  if schema_name is not None:
170  name = '%s.%s' %(schema_name, table_name)
171  return name
def fq_name
Definition: conddblib.py:166
def conddblib.getPayload (   session,
  hash 
)

Definition at line 672 of file conddblib.py.

References alcazmumu_cfi.filter.

673 def getPayload(session, hash):
674  # get payload from DB:
675  data, payloadType = session.query(Payload.data, Payload.object_type).filter(Payload.hash == hash).one()
676  return data
def getPayload
Definition: conddblib.py:672
def conddblib.getSchema (   tp)

Definition at line 230 of file conddblib.py.

Referenced by conddblib.Connection.is_valid().

231 def getSchema(tp):
232  if tp.__table_args__ is not None:
233  return tp.__table_args__['schema']
234  return None
def getSchema
Definition: conddblib.py:230
def conddblib.getSessionOnMasterDB (   session1,
  session2 
)

Definition at line 448 of file conddblib.py.

References make_url().

449 def getSessionOnMasterDB( session1, session2 ):
450  key = '%s/%s'
451  sessiondict = { }
452  sessiondict[key %(session1._url.drivername,session1._url.host)] = session1
453  sessiondict[key %(session2._url.drivername,session2._url.host)] = session2
454  masterkey = key %('oracle',ONLINEORAPRO)
455  if masterkey in sessiondict.keys():
456  return sessiondict[masterkey]
457  adgkey = key %('oracle',ORAPRO)
458  if adgkey in sessiondict.keys():
459  return sessiondict[adgkey]
460  frontierkey = key %('frontier',PRO)
461  if frontierkey in sessiondict.keys():
462  return sessiondict[frontierkey]
463  # default case: frontier on pro
464  conn = Connection(make_url())
465  session = conn.session()
466  # is it required?
467  session._conn = conn
468  return session
469 
# Connection helpers
def getSessionOnMasterDB
Definition: conddblib.py:448
def make_url
Definition: conddblib.py:487
def conddblib.hash (   data)

Definition at line 52 of file conddblib.py.

52 
53 def hash(data):
54  return hashlib.sha1(data.encode('ascii')).hexdigest()
55 
56 
# Constants
def hash
Definition: conddblib.py:52
def conddblib.listObject (   session,
  name,
  snapshot = None 
)

Definition at line 633 of file conddblib.py.

References _exists(), _inserted_before(), python.cmstools.all(), alcazmumu_cfi.filter, join(), and hitfit.scalar().

634 def listObject(session, name, snapshot=None):
635 
636  is_tag = _exists(session, Tag.name, name)
637  result = {}
638  if is_tag:
639  result['type'] = 'Tag'
640  result['name'] = session.query(Tag).get(name).name
641  result['timeType'] = session.query(Tag.time_type).\
642  filter(Tag.name == name).\
643  scalar()
644 
645  result['iovs'] = session.query(IOV.since, IOV.insertion_time, IOV.payload_hash, Payload.object_type).\
646  join(IOV.payload).\
647  filter(
648  IOV.tag_name == name,
649  _inserted_before(snapshot),
650  ).\
651  order_by(IOV.since.desc(), IOV.insertion_time.desc()).\
652  from_self().\
653  order_by(IOV.since, IOV.insertion_time).\
654  all()
655 
656  try:
657  is_global_tag = _exists(session, GlobalTag.name, name)
658  if is_global_tag:
659  result['type'] = 'GlobalTag'
660  result['name'] = session.query(GlobalTag).get(name)
661  result['tags'] = session.query(GlobalTagMap.record, GlobalTagMap.label, GlobalTagMap.tag_name).\
662  filter(GlobalTagMap.global_tag_name == name).\
663  order_by(GlobalTagMap.record, GlobalTagMap.label).\
664  all()
665  except sqlalchemy.exc.OperationalError:
666  sys.stderr.write("No table for GlobalTags found in DB.\n\n")
667 
668  if not is_tag and not is_global_tag:
669  raise Exception('There is no tag or global tag named %s in the database.' % name)
670 
671  return result
def _exists
Definition: conddblib.py:609
def _inserted_before
Definition: conddblib.py:620
def all
workaround iterator generators for ROOT classes
Definition: cmstools.py:25
def listObject
Definition: conddblib.py:633
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
double scalar(const CLHEP::HepGenMatrix &m)
Return the matrix as a scalar. Raise an assertion if the matris is not .
Definition: matutil.cc:166
def conddblib.make_dbtype (   backendName,
  schemaName,
  baseType 
)

Definition at line 190 of file conddblib.py.

References fq_col().

Referenced by conddblib.Connection.get_dbtype().

191 def make_dbtype( backendName, schemaName, baseType ):
192  members = {}
193  deps_reg = set()
194  dbtype_name = '%s_%s' %(baseType.__name__,backendName)
195  members['__tablename__'] = baseType.__tablename__
196  members['__table_args__'] = None
197  if schemaName is not None:
198  members['__table_args__'] = {'schema': schemaName }
199  for k,v in baseType.columns.items():
200  defColVal = None
201  if len(v)==3:
202  defColVal = v[2]
203  if isinstance(v[0],DbRef):
204  refColDbt = v[0].rtype.columns[v[0].rcol][0]
205  pk = (True if v[1]==_Col.pk else False)
206  if v[1]==_Col.pk:
207  members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),primary_key=True)
208  else:
209  nullable = (False if v[1] == _Col.notNull else True)
210  members[k] = sqlalchemy.Column(refColDbt,sqlalchemy.ForeignKey(fq_col(schemaName,v[0].rtype.__tablename__,v[0].rcol)),nullable=nullable)
211  if v[0].rtype.__name__ not in deps_reg:
212  deps_reg.add(v[0].rtype.__name__)
213  reftype_name = '%s_%s' %(v[0].rtype.__name__,backendName)
214  members[(v[0].rtype.__name__).lower()] = sqlalchemy.orm.relationship(reftype_name)
215  else:
216  if v[1]==_Col.pk:
217  members[k] = sqlalchemy.Column(v[0],primary_key=True)
218  else:
219  nullable = (True if v[1]==_Col.nullable else False)
220  if defColVal is None:
221  members[k] = sqlalchemy.Column(v[0],nullable=nullable)
222  else:
223  members[k] = sqlalchemy.Column(v[0],nullable=nullable, default=defColVal)
224  dbType = type(dbtype_name,(_Base,),members)
225 
226  if backendName not in db_models.keys():
227  db_models[backendName] = {}
228  db_models[backendName][baseType.__name__] = dbType
229  return dbType
def make_dbtype
Definition: conddblib.py:190
def fq_col
Definition: conddblib.py:184
def conddblib.make_url (   database = 'pro',
  read_only = True 
)

Definition at line 487 of file conddblib.py.

References _getCMSSQLAlchemyConnectionString().

Referenced by getSessionOnMasterDB(), and helperFunctions.getTagsMap().

488 def make_url(database='pro',read_only = True):
489  if database.startswith('sqlite:') or database.startswith('sqlite_file:'):
490  ignore, database = database.split(':',1)
491 
492  if ':' in database and '://' not in database: # check if we really got a shortcut like "pro:<schema>" (and not a url like proto://...), if so, disentangle
493  database, schema = database.split(':')
494 
495  officialdbs = {
496  # frontier
497  'pro' : ('frontier','PromptProd', { 'R': schema_name }, ),
498  'arc' : ('frontier','FrontierArc', { 'R': schema_name }, ),
499  'int' : ('frontier','FrontierInt', { 'R': schema_name }, ),
500  'dev' : ('frontier','FrontierPrep', { 'R': schema_name }, ),
501  # oracle adg
502  'orapro': ('oracle', 'cms_orcon_adg', { 'R': dbreader_user_name }, ),
503  'oraarc': ('oracle', 'cmsarc_lb', { 'R': dbreader_user_name }, ),
504  # oracle masters
505  'oraint': ('oracle', 'cms_orcoff_int', { 'R': dbreader_user_name,
506  'W': dbwriter_user_name }, ),
507  'oradev': ('oracle', 'cms_orcoff_prep', { 'R': dbreader_user_name,
508  'W': dbwriter_user_name }, ),
509  'onlineorapro': ('oracle', 'cms_orcon_prod', { 'R': dbreader_user_name,
510  'W': dbwriter_user_name }, ),
511  'onlineoraint': ('oracle', 'cmsintr_lb', { 'R': dbreader_user_name,
512  'W': dbwriter_user_name }, ),
513  }
514 
515  if database in officialdbs.keys():
516  key = ('R' if read_only else 'W')
517  mapping = officialdbs[database]
518  tech = mapping[0]
519  service = mapping[1]
520  schema_dict = mapping[2]
521  if key in schema_dict.keys():
522  database = _getCMSSQLAlchemyConnectionString(tech,service,schema_dict[key])
523  else:
524  raise Exception("Read-only database %s://%s cannot be accessed in update mode." %(tech,service))
525 
526  logging.debug('connection string set to "%s"' % database)
527 
528  try:
529  url = sqlalchemy.engine.url.make_url(database)
530  except sqlalchemy.exc.ArgumentError:
531  url = sqlalchemy.engine.url.make_url('sqlite:///%s' % database)
532  return url
def _getCMSSQLAlchemyConnectionString
Definition: conddblib.py:474
def make_url
Definition: conddblib.py:487
def conddblib.oracle_connection_string (   db_service,
  db_schema 
)

Definition at line 139 of file conddblib.py.

Referenced by connect().

140 def oracle_connection_string(db_service, db_schema ):
141  return 'oracle://%s/%s'%(db_service,db_schema)
def oracle_connection_string
Definition: conddblib.py:139

Variable Documentation

string conddblib.__author__ = 'Miguel Ojeda'

Definition at line 4 of file conddblib.py.

string conddblib.__copyright__ = 'Copyright 2013, CERN'

Definition at line 5 of file conddblib.py.

list conddblib.__credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']

Definition at line 6 of file conddblib.py.

string conddblib.__email__ = 'giacomo.govi@cern.ch'

Definition at line 9 of file conddblib.py.

string conddblib.__license__ = 'Unknown'

Definition at line 7 of file conddblib.py.

string conddblib.__maintainer__ = 'Giacomo Govi'

Definition at line 8 of file conddblib.py.

tuple conddblib._Base = sqlalchemy.ext.declarative.declarative_base()

Definition at line 164 of file conddblib.py.

string conddblib.ARC = 'FrontierArc'

Definition at line 34 of file conddblib.py.

string conddblib.authPathEnvVar = 'COND_AUTH_PATH'

Definition at line 28 of file conddblib.py.

string conddblib.contact_help = 'If you need assistance, please write an email to %s and %s. If you need immediate/urgent assistance, you can call the Offline DB expert on call (%s).'

Definition at line 67 of file conddblib.py.

string conddblib.database_help

Definition at line 68 of file conddblib.py.

dictionary conddblib.db_models = {}

Definition at line 172 of file conddblib.py.

string conddblib.dbkey_filename = 'db.key'

Definition at line 29 of file conddblib.py.

tuple conddblib.dbkey_folder = os.path.join('.cms_cond',dbkey_filename)

Definition at line 30 of file conddblib.py.

string conddblib.dbreader_user_name = 'cms_cond_general_r'

Definition at line 23 of file conddblib.py.

string conddblib.dbuser_name = 'cms_conditions'

Definition at line 22 of file conddblib.py.

string conddblib.dbwriter_user_name = 'cms_cond_general_w'

Definition at line 24 of file conddblib.py.

int conddblib.description_length = 4000

Definition at line 60 of file conddblib.py.

string conddblib.DEV = 'FrontierPrep'

Definition at line 36 of file conddblib.py.

string conddblib.empty_label = '-'

Definition at line 57 of file conddblib.py.

tuple conddblib.hash_length = len(hash(''))

Definition at line 61 of file conddblib.py.

string conddblib.INT = 'FrontierInt'

Definition at line 35 of file conddblib.py.

tuple conddblib.logger = logging.getLogger(__name__)

Definition at line 25 of file conddblib.py.

Referenced by dqm::implementation::DQMStore.debugTrackME(), cond::service::OnlineDBOutputService.getLastLumiProcessed(), cond::persistency::MsgDispatcher.MsgDispatcher(), cscdqm::Configuration.printStats(), and dqm::implementation::DQMStore.printTrace().

int conddblib.name_length = 100

Definition at line 59 of file conddblib.py.

string conddblib.offline_db_experts_email = 'cms-offlinedb-exp@cern.ch'

Definition at line 64 of file conddblib.py.

string conddblib.offline_db_experts_phone = '+41 22 76 70817, or 70817 from CERN; check https://twiki.cern.ch/twiki/bin/viewauth/CMS/DBShifterHelpPage if it does not work; availability depends on the state of the LHC'

Definition at line 65 of file conddblib.py.

string conddblib.ONLINEORAINT = 'cmsintr_lb'

Definition at line 44 of file conddblib.py.

string conddblib.ONLINEORAPRO = 'cms_orcon_prod'

Definition at line 43 of file conddblib.py.

string conddblib.ORAARC = 'cmsarc_lb'

Definition at line 39 of file conddblib.py.

string conddblib.ORADEV = 'cms_orcoff_prep'

Definition at line 42 of file conddblib.py.

string conddblib.ORAINT = 'cms_orcoff_int'

Definition at line 41 of file conddblib.py.

string conddblib.ORAPRO = 'cms_orcon_adg'

Definition at line 38 of file conddblib.py.

string conddblib.PRO = 'PromptProd'

Definition at line 33 of file conddblib.py.

string conddblib.schema_name = 'cms_conditions'

Definition at line 21 of file conddblib.py.

tuple conddblib.synch_list = list(x.value for x in list(Synchronization))

Definition at line 153 of file conddblib.py.

string conddblib.web_experts_email = 'cms-cond-dev@cern.ch'

Definition at line 63 of file conddblib.py.