Inheritance diagram for conddb_version_mgr.conddb_tool:

Public Member Functions
def	__init__ (self)

def	connect (self)

def	insert_boost_run (self)

def	list_boost_run (self)

def	process_tag_boost_version (self, t, timetype, tagBoostVersion, minIov, timeCut, validate)

def	show_tag_boost_version (self)

def	update_tag_boost_version_in_db (self, t, tagBoostVersion, minIov, update)

def	update_tags (self)

def	validate_boost_version (self, t, timetype, tagBoostVersion)

Public Attributes
	args

	db

	iovs

	logger

	version_db

	versionIovs

Detailed Description

Definition at line 139 of file conddb_version_mgr.py.

Constructor & Destructor Documentation

def conddb_version_mgr.conddb_tool.__init__ ( self )

Definition at line 140 of file conddb_version_mgr.py.

     def __init__( self ):
         self.db = None
         self.version_db = None
         self.args = None
         self.logger = logging.getLogger()        
         self.logger.setLevel(logLevel)
         consoleHandler = logging.StreamHandler(sys.stdout) 
         consoleHandler.setFormatter(logFormatter)
         self.logger.addHandler(consoleHandler)
         self.iovs = None
         self.versionIovs = None
 

Member Function Documentation

def conddb_version_mgr.conddb_tool.connect ( self )

Definition at line 152 of file conddb_version_mgr.py.

References conddb_version_mgr.version_db.db, conddb_version_mgr.conddb_tool.db, HcalLutManager.db, and relativeConstraints.keys.

     def connect( self ):
         if self.args.db is None:
             self.args.db = 'pro'
         if self.args.db == 'dev' or self.args.db == 'oradev' :
             db_service = dev_db_service
         elif self.args.db == 'orapro':
             db_service = adg_db_service    
         elif self.args.db != 'onlineorapro' or self.args.db != 'pro':
             db_service = prod_db_service
         else:
             raise Exception("Database '%s' is not known." %args.db )
         if self.args.accessType not in db_service[1].keys():
             raise Exception('The specified database connection %s does not support the requested action.' %db_service[0])
         service = db_service[1][self.args.accessType]
         creds = auth.get_credentials( authPathEnvVar, service, self.args.auth )
         if creds is None:
             raise Exception("Could not find credentials for service %s" %service)
         (username, account, pwd) = creds
         connStr =  '%s/%s@%s' %(username,pwd,db_service[0])
         self.db = cx_Oracle.connect(connStr)
         logging.info('Connected to %s as user %s' %(db_service[0],username))
         self.db.current_schema = schema_name
 

def conddb_version_mgr.conddb_tool.insert_boost_run ( self )

Definition at line 368 of file conddb_version_mgr.py.

References conddb_version_mgr.version_db.db, conddb_version_mgr.conddb_tool.db, HcalLutManager.db, and conddb_version_mgr.conddb_tool.version_db.

     def insert_boost_run( self ):
         cursor = self.db.cursor()
         self.version_db = version_db( self.db )
         self.version_db.insert_boost_run_range( self.args.since, self.args.label )
         self.db.commit()
         logging.info('boost version %s inserted with since %s' %(self.args.label,self.args.since))
 

def conddb_version_mgr.conddb_tool.list_boost_run ( self )

Definition at line 375 of file conddb_version_mgr.py.

References conddb_version_mgr.version_db.db, conddb_version_mgr.conddb_tool.db, HcalLutManager.db, conddb_version_mgr.print_table(), and conddb_version_mgr.conddb_tool.version_db.

     def list_boost_run( self ):
         cursor = self.db.cursor()
         self.version_db = version_db( self.db )
         self.version_db.fetch_boost_run_map()
         headers = ['Run','Run start time','Boost Version','Insertion time']
         print_table( headers, self.version_db.boost_run_map ) 
 

def conddb_version_mgr.conddb_tool.process_tag_boost_version	(	self,
		t,
		timetype,
		tagBoostVersion,
		minIov,
		timeCut,
		validate
	)

Definition at line 175 of file conddb_version_mgr.py.

References conddb_version_mgr.conddb_tool.iovs, str, and conddb_version_mgr.conddb_tool.versionIovs.

Referenced by conddb_version_mgr.conddb_tool.show_tag_boost_version(), conddb_version_mgr.conddb_tool.update_tags(), and conddb_version_mgr.conddb_tool.validate_boost_version().

     def process_tag_boost_version( self, t, timetype, tagBoostVersion, minIov, timeCut, validate ):
         if self.iovs is None:
             self.iovs = []
             cursor = self.db.cursor()
             stmt = 'SELECT IOV.SINCE SINCE, IOV.INSERTION_TIME INSERTION_TIME, P.STREAMER_INFO STREAMER_INFO FROM TAG, IOV, PAYLOAD P WHERE TAG.NAME = IOV.TAG_NAME AND P.HASH = IOV.PAYLOAD_HASH AND TAG.NAME = :TAG_NAME'
             params = (t,)
             if timeCut and tagBoostVersion is not None and not validate:
                 whereClauseOnSince = ' AND IOV.INSERTION_TIME>:TIME_CUT' 
                 stmt = stmt +  whereClauseOnSince
                 params = params + (timeCut,)               
             stmt = stmt + ' ORDER BY SINCE'
             logging.debug('Executing: "%s"' %stmt)
             cursor.execute(stmt,params)
             for r in cursor:
                 streamer_info = str(r[2].read())
                 self.iovs.append((r[0],r[1],streamer_info))
         niovs = 0
         self.versionIovs = []
         lastBoost = None
         update = False
         if tagBoostVersion is not None:
             update = True        
         for iov in self.iovs:            
             if validate and timeCut is not None and  timeCut < iov[1]:
                 continue
             niovs += 1
             iovBoostVersion, tagBoostVersion = sm.update_tag_boost_version( tagBoostVersion, minIov, iov[2], iov[0], timetype, self.version_db.boost_run_map )
             if minIov is None or iov[0]<minIov:
                 minIov = iov[0]
             logging.debug('iov: %s - inserted on %s - streamer: %s' %(iov[0],iov[1],iov[2]))
             logging.debug('current tag boost version: %s minIov: %s' %(tagBoostVersion,minIov))
             if lastBoost is None or lastBoost!=iovBoostVersion:
                 self.versionIovs.append((iov[0],iovBoostVersion))
                 lastBoost = iovBoostVersion
 
         if tagBoostVersion is None:
             if niovs == 0:
                 logging.warning( 'No iovs found. boost version cannot be determined.')
                 return None, None
             else:
                 logging.error('Could not determine the tag boost version.' )
                 return None, None
         else:
             if niovs == 0:
                 logging.info('Tag boost version has not changed.')
             else:
                 msg = 'Found tag boost version %s ( min iov: %s ) combining payloads from %s iovs' %(tagBoostVersion,minIov,niovs)
                 if timeCut is not None:
                     if update:
                         msg += ' (iov insertion time>%s)' %str(timeCut)
                     else:
                         msg += ' (iov insertion time<%s)' %str(timeCut)
                 logging.info( msg ) 
         return tagBoostVersion, minIov
 

def conddb_version_mgr.conddb_tool.show_tag_boost_version ( self )

Definition at line 382 of file conddb_version_mgr.py.

References conddb_version_mgr.version_db.db, conddb_version_mgr.conddb_tool.db, HcalLutManager.db, conddb_version_mgr.conddb_tool.iovs, edm.print(), conddb_version_mgr.print_table(), conddb_version_mgr.conddb_tool.process_tag_boost_version(), str, conddb_version_mgr.conddb_tool.version_db, and conddb_version_mgr.conddb_tool.versionIovs.

     def show_tag_boost_version( self ):
         cursor = self.db.cursor()
         tag = self.args.tag_name
         cursor.execute('SELECT TIME_TYPE FROM TAG WHERE NAME= :TAG_NAME',(tag,))
         rows = cursor.fetchall()
         timeType = None
         t_modificationTime = None
         for r in rows:
             timeType = r[0]
         if timeType is None:
             raise Exception("Tag %s does not exist in the database." %tag)
         cursor.execute('SELECT MAX(INSERTION_TIME) FROM IOV WHERE TAG_NAME= :TAG_NAME',(tag,))
         rows = cursor.fetchall()
         for r in rows:
             t_modificationTime = r[0]
         if t_modificationTime is None:
             raise Exception("Tag %s does not have any iov stored." %tag)
         logging.info('Tag %s - timetype: %s' %(tag,timeType))
         cursor.execute('SELECT MIN_SERIALIZATION_V, MIN_SINCE, MODIFICATION_TIME FROM TAG_METADATA WHERE TAG_NAME= :TAG_NAME',(tag,))
         rows = cursor.fetchall()
         tagBoostVersion = None
         minIov = None
         v_modificationTime = None
         for r in rows:
             tagBoostVersion = r[0]
             minIov = r[1]
             v_modificationTime = r[2]
         if v_modificationTime is not None:
             if t_modificationTime > v_modificationTime:
                 logging.warning('The minimum boost version stored is out of date.')
             else:
                 logging.info('The minimum boost version stored is up to date.')
         mt = '-'
         if v_modificationTime is not None:
             mt = str(v_modificationTime)
         r_tagBoostVersion = None
         if self.args.rebuild or self.args.full:
             self.version_db = version_db( self.db )
             self.version_db.fetch_boost_run_map()
             timeCut = None
             logging.info('Calculating minimum boost version for the available iovs...')
             r_tagBoostVersion, r_minIov = self.process_tag_boost_version( tag, timeType, tagBoostVersion, minIov, timeCut )
         print('# Currently stored: %s (min iov:%s)' %(tagBoostVersion,minIov))
         print('# Last update: %s' %mt)
         print('# Last update on the iovs: %s' %str(t_modificationTime))
         if self.args.rebuild or self.args.full:
             print('# Based on the %s available IOVs: %s (min iov:%s)' %(len(self.iovs),r_tagBoostVersion,r_minIov))
             if self.args.full:
                 headers = ['Run','Boost Version']
                 print_table( headers, self.versionIovs ) 
 

def conddb_version_mgr.conddb_tool.update_tag_boost_version_in_db	(	self,
		t,
		tagBoostVersion,
		minIov,
		update
	)

Definition at line 271 of file conddb_version_mgr.py.

Referenced by conddb_version_mgr.conddb_tool.update_tags().

     def update_tag_boost_version_in_db( self, t, tagBoostVersion, minIov, update ):
         cursor = self.db.cursor()
         now = datetime.datetime.utcnow()
         if update:
             cursor.execute('UPDATE TAG_METADATA SET MIN_SERIALIZATION_V=:BOOST_V, MIN_SINCE=:MIN_IOV, MODIFICATION_TIME=:NOW WHERE TAG_NAME = :NAME',( tagBoostVersion,minIov,now,t))
         else:
             cursor.execute('INSERT INTO TAG_METADATA ( TAG_NAME, MIN_SERIALIZATION_V, MIN_SINCE, MODIFICATION_TIME ) VALUES ( :NAME, :BOOST_V, :MIN_IOV, :NOW )',(t, tagBoostVersion,minIov,now))
         logging.info('Minimum boost version for the tag updated.')
         

def conddb_version_mgr.conddb_tool.update_tags ( self )

Definition at line 280 of file conddb_version_mgr.py.

References conddb_version_mgr.version_db.db, conddb_version_mgr.conddb_tool.db, HcalLutManager.db, conddb_version_mgr.conddb_tool.iovs, conddb_version_mgr.conddb_tool.process_tag_boost_version(), str, conddb_version_mgr.conddb_tool.update_tag_boost_version_in_db(), conddb_version_mgr.conddb_tool.validate_boost_version(), conddb_version_mgr.conddb_tool.version_db, and conddb_version_mgr.conddb_tool.versionIovs.

     def update_tags( self ):
         cursor = self.db.cursor()
         self.version_db = version_db( self.db )
         self.version_db.fetch_cmssw_boost_map()
         self.version_db.fetch_boost_run_map()
         tags = {}
         wpars = ()
         if self.args.name is not None:
             stmt0 = 'SELECT NAME FROM TAG WHERE NAME = :TAG_NAME'
             wpars = (self.args.name,)
             cursor.execute(stmt0,wpars);
             rows = cursor.fetchall()
             found = False
             for r in rows:
                 found = True
                 break
             if not found:
                 raise Exception('Tag %s does not exists in the database.' %self.args.name )
             tags[self.args.name] = None
             stmt1 = 'SELECT MIN_SERIALIZATION_V, MIN_SINCE, CAST(MODIFICATION_TIME AS TIMESTAMP(0)) FROM  TAG_METADATA WHERE TAG_NAME = :NAME'
             cursor.execute(stmt1,wpars);
             rows = cursor.fetchall()
             for r in rows:
                 tags[self.args.name] = (r[0],r[1],r[2])
         else:
             stmt0 = 'SELECT NAME FROM TAG WHERE NAME NOT IN ( SELECT TAG_NAME FROM TAG_METADATA) ORDER BY NAME' 
             nmax = 100
             if self.args.max is not None:
                 nmax = self.args.max
             if self.args.all:
                 nmax = -1
             if nmax >=0:
                 stmt0 = 'SELECT NAME FROM (SELECT NAME FROM TAG WHERE NAME NOT IN ( SELECT TAG_NAME FROM TAG_METADATA ) ORDER BY NAME) WHERE ROWNUM<= :MAXR'
                 wpars = (nmax,)
             cursor.execute(stmt0,wpars);
             rows = cursor.fetchall()
             for r in rows:
                 tags[r[0]] = None
             stmt1 = 'SELECT T.NAME NAME, TM.MIN_SERIALIZATION_V MIN_SERIALIZATION_V, TM.MIN_SINCE MIN_SINCE, CAST(TM.MODIFICATION_TIME AS TIMESTAMP(0)) MODIFICATION_TIME FROM TAG T, TAG_METADATA TM WHERE T.NAME=TM.TAG_NAME AND CAST(TM.MODIFICATION_TIME AS TIMESTAMP(0)) < (SELECT MAX(INSERTION_TIME) FROM IOV WHERE IOV.TAG_NAME=TM.TAG_NAME) ORDER BY NAME' 
             nmax = nmax-len(tags)
             if nmax >=0:
                 stmt1 = 'SELECT NAME, MIN_SERIALIZATION_V, MIN_SINCE, MODIFICATION_TIME FROM (SELECT T.NAME NAME, TM.MIN_SERIALIZATION_V MIN_SERIALIZATION_V, TM.MIN_SINCE MIN_SINCE, CAST(TM.MODIFICATION_TIME AS TIMESTAMP(0)) MODIFICATION_TIME FROM TAG T, TAG_METADATA TM WHERE T.NAME=TM.TAG_NAME AND CAST(TM.MODIFICATION_TIME AS TIMESTAMP(0)) < (SELECT MAX(INSERTION_TIME) FROM IOV WHERE IOV.TAG_NAME=TM.TAG_NAME) ORDER BY NAME) WHERE ROWNUM<= :MAXR'
                 wpars = (nmax,)
             cursor.execute(stmt1,wpars);
             rows = cursor.fetchall()
             i = 0
             for r in rows:
                 i += 1
                 if nmax >=0 and i>nmax:
                     break
                 tags[r[0]] = (r[1],r[2],r[3])
         logging.info( 'Processing boost version for %s tags' %len(tags))
         count = 0
         for t in sorted(tags.keys()):
             count += 1
             try:
                 update = False
                 cursor.execute('SELECT TIME_TYPE FROM TAG WHERE NAME= :TAG_NAME',(t,))
                 timetype = cursor.fetchone()[0]
                 self.iovs = None
                 logging.info('************************************************************************')
                 logging.info('Tag [%s] %s - timetype: %s' %(count,t,timetype))
                 tagBoostVersion = None
                 minIov = None
                 timeCut = None
                 if tags[t] is not None:
                     update = True
                     tagBoostVersion = tags[t][0]
                     minIov = tags[t][1]
                     timeCut = tags[t][2]
                 tagBoostVersion, minIov = self.process_tag_boost_version( t, timetype, tagBoostVersion, minIov, timeCut, self.args.validate )
                 if tagBoostVersion is None:
                     continue
                 logging.debug('boost versions in the %s iovs: %s' %(len(self.iovs),str(self.versionIovs)))
                 if self.args.validate: 
                     invalid_gts = self.validate_boost_version( t, timetype, tagBoostVersion )
                     if len(invalid_gts)>0:
                         with open('invalid_tags_in_gts.txt','a') as error_file:
                             for gt in invalid_gts:
                                 error_file.write('Tag %s (boost %s) is invalid for GT %s ( boost %s) \n' %(t,tagBoostVersion,gt[0],gt[1]))
                 if len(self.iovs):
                     if self.iovs[0][0]<minIov:
                         minIov = self.iovs[0]
                 self.update_tag_boost_version_in_db( t, tagBoostVersion, minIov, update )
                 self.db.commit()
             except Exception as e:
                 logging.error(str(e))
 

def conddb_version_mgr.conddb_tool.validate_boost_version	(	self,
		t,
		timetype,
		tagBoostVersion
	)

Definition at line 230 of file conddb_version_mgr.py.

References conddb_version_mgr.conddb_tool.process_tag_boost_version(), and str.

Referenced by conddb_version_mgr.conddb_tool.update_tags().

     def validate_boost_version( self, t, timetype, tagBoostVersion ):
         cursor = self.db.cursor()
         cursor.execute('SELECT GT.NAME, GT.RELEASE, GT.SNAPSHOT_TIME FROM GLOBAL_TAG GT, GLOBAL_TAG_MAP GTM WHERE GT.NAME = GTM.GLOBAL_TAG_NAME AND GTM.TAG_NAME = :TAG_NAME',(t,))
         rows = cursor.fetchall()
         invalid_gts = []
         ngt = 0
         gts = []
         for r in rows:
             gts.append((r[0],r[1],r[2]))
         if len(gts)>0:
             logging.info('validating %s gts.' %len(gts))
         boost_snapshot_map = {}
         for gt in gts:
             ngt += 1
             logging.debug('Validating for GT %s (release %s)' %(gt[0],gt[1])) 
             gtCMSSWVersion =  sm.check_cmssw_version( gt[1] )
             gtBoostVersion = self.version_db.lookup_boost_in_cmssw( gtCMSSWVersion  )
             if sm.cmp_boost_version( gtBoostVersion, tagBoostVersion )<0:
                 logging.warning( 'The boost version computed from all the iovs in the tag (%s) is incompatible with the gt [%s] %s (consuming ver: %s, snapshot: %s)' %(tagBoostVersion,ngt,gt[0],gtBoostVersion,str(gt[2])))
                 if str(gt[2]) not in boost_snapshot_map.keys():
                     tagSnapshotBoostVersion = None
                     minIov = None
                     tagSnapshotBoostVersion, minIov = self.process_tag_boost_version(t, timetype, tagSnapshotBoostVersion, minIov, gt[2])
                     if tagSnapshotBoostVersion is not None:
                         boost_snapshot_map[str(gt[2])] = tagSnapshotBoostVersion
                     else:
                         continue
                 else:
                     tagSnapshotBoostVersion = boost_snapshot_map[str(gt[2])] 
                 if sm.cmp_boost_version( gtBoostVersion, tagSnapshotBoostVersion )<0:
                     logging.error('The snapshot from tag used by gt %s (consuming ver: %s) has an incompatible combined boost version %s' %(gt[0],gtBoostVersion,tagSnapshotBoostVersion))
                     invalid_gts.append( ( gt[0], gtBoostVersion ) )
         if len(invalid_gts)==0:
             if ngt>0:
                 logging.info('boost version for the tag validated in %s referencing Gts' %(ngt))
             else:
                 logging.info('No GT referencing this tag found.')
         else:
             logging.error( 'boost version for the tag is invalid.')
         return invalid_gts
 