00001
00002 VERSION='1.00'
00003 import os,sys,datetime
00004 import coral
00005 from RecoLuminosity.LumiDB import argparse,selectionParser,csvSelectionParser
00006
00007 '''
00008 --on wbm db
00009 select LUMISEGMENTNR,DEADTIMEBEAMACTIVE from cms_wbm.LEVEL1_TRIGGER_CONDITIONS where RUNNUMBER=:runnumber order by LUMISEGMENTNR;
00010 --on lumidb
00011 update TRG set DEADTIME=:deadtimebeamactive where RUNNUM=:runnum and CMSLSNUM=:lsnum
00012 --reapply calibration to inst lumi
00013 update LUMISUMMARY set INSTLUMI=1.006319*INSTLUMI where RUNNUM in (1,3,57,90)
00014 '''
00015
00016 class constants(object):
00017 def __init__(self):
00018 self.debug=False
00019 self.isdryrun=None
00020 self.runinfoschema='CMS_RUNINFO'
00021 self.wbmschema='CMS_WBM'
00022 self.wbmdeadtable='LEVEL1_TRIGGER_CONDITIONS'
00023 self.gtmonschema='CMS_GT_MON'
00024 self.gtdeadview='GT_MON_TRIG_DEAD_VIEW'
00025 self.lumitrgtable='TRG'
00026 self.lumisummarytable='LUMISUMMARY'
00027 self.runsummarytable='CMSRUNSUMMARY'
00028 def missingTimeRuns(dbsession,c):
00029 '''return all the runs with starttime or stoptime column NULL in lumi db
00030 select runnum from CMSRUNSUMMARY where starttime is NULL or stoptime is NULL
00031 '''
00032 result=[]
00033 try:
00034 emptyBindVarList=coral.AttributeList()
00035 dbsession.transaction().start(True)
00036 schema=dbsession.nominalSchema()
00037 if not schema:
00038 raise 'cannot connect to schema '
00039 if not schema.existsTable(c.runsummarytable):
00040 raise 'non-existing table '+c.runsummarytable
00041 query=schema.newQuery()
00042 query.addToTableList(c.runsummarytable)
00043 query.addToOutputList('RUNNUM','runnum')
00044 query.setCondition('STARTTIME IS NULL AND STOPTIME IS NULL',emptyBindVarList)
00045 query.addToOrderList('runnum')
00046 queryOutput=coral.AttributeList()
00047 queryOutput.extend('runnum','unsigned int')
00048 query.defineOutput(queryOutput)
00049 cursor=query.execute()
00050 while cursor.next():
00051 result.append(cursor.currentRow()['runnum'].data())
00052 del query
00053 dbsession.transaction().commit()
00054 except Exception,e:
00055 print str(e)
00056 dbsession.transaction().rollback()
00057 del dbsession
00058 return result
00059 def getTimeForRun(dbsession,c,runnums):
00060 '''
00061 get start stop time of run from runinfo database
00062 select time from cms_runinfo.runsession_parameter where runnumber=:runnum and name='CMS.LVL0:START_TIME_T';
00063 select time from cms_runinfo.runsession_parameter where runnumber=:runnum and name='CMS.LVL0:STOP_TIME_T';
00064 '''
00065 result={}
00066 tableName='RUNSESSION_PARAMETER'
00067 try:
00068 dbsession.transaction().start(True)
00069 schema=dbsession.nominalSchema()
00070 if not schema:
00071 raise 'cannot connect to schema '
00072 if not schema.existsTable(tableName):
00073 raise 'non-existing table '+tableName
00074
00075 startTime=''
00076 stopTime=''
00077 for runnum in runnums:
00078 startTQuery=schema.newQuery()
00079 startTQuery.addToTableList(tableName)
00080 startTQuery.addToOutputList('TIME','starttime')
00081 stopTQuery=schema.newQuery()
00082 stopTQuery.addToTableList(tableName)
00083 stopTQuery.addToOutputList('TIME','stoptime')
00084 startTQueryCondition=coral.AttributeList()
00085 stopTQueryCondition=coral.AttributeList()
00086 startTQueryOutput=coral.AttributeList()
00087 stopTQueryOutput=coral.AttributeList()
00088 startTQueryCondition.extend('runnum','unsigned int')
00089 startTQueryCondition.extend('name','string')
00090 startTQueryOutput.extend('starttime','time stamp')
00091 stopTQueryCondition.extend('runnum','unsigned int')
00092 stopTQueryCondition.extend('name','string')
00093 stopTQueryOutput.extend('stoptime','time stamp')
00094 startTQueryCondition['runnum'].setData(int(runnum))
00095 startTQueryCondition['name'].setData('CMS.LVL0:START_TIME_T')
00096 startTQuery.setCondition('RUNNUMBER=:runnum AND NAME=:name',startTQueryCondition)
00097 startTQuery.defineOutput(startTQueryOutput)
00098 startTCursor=startTQuery.execute()
00099 while startTCursor.next():
00100 startTime=startTCursor.currentRow()['starttime'].data()
00101 stopTQueryCondition['runnum'].setData(int(runnum))
00102 stopTQueryCondition['name'].setData('CMS.LVL0:STOP_TIME_T')
00103 stopTQuery.setCondition('RUNNUMBER=:runnum AND NAME=:name',stopTQueryCondition)
00104 stopTQuery.defineOutput(stopTQueryOutput)
00105 stopTCursor=stopTQuery.execute()
00106 while stopTCursor.next():
00107 stopTime=stopTCursor.currentRow()['stoptime'].data()
00108 if not startTime or not stopTime:
00109 print 'Warning: no startTime or stopTime found for run ',runnum
00110 else:
00111 result[runnum]=(startTime,stopTime)
00112 del startTQuery
00113 del stopTQuery
00114 dbsession.transaction().commit()
00115 except Exception,e:
00116 print str(e)
00117 dbsession.transaction().rollback()
00118 del dbsession
00119 return result
00120
00121 def addTimeForRun(dbsession,c,runtimedict):
00122 '''
00123 Input runtimedict{runnumber:(startTimeT,stopTimeT)}
00124 update CMSRUNSUMMARY set STARTTIME=:starttime,STOPTIME=:stoptime where RUNNUM=:runnum
00125 #update CMSRUNSUMMARY set STOPTIME=:stoptime where RUNNUM=:runnum
00126 '''
00127 nchanged=0
00128 totalchanged=0
00129 try:
00130 dbsession.transaction().start(False)
00131 schema=dbsession.nominalSchema()
00132 if not schema:
00133 raise 'cannot connect to schema'
00134 if not schema.existsTable(c.runsummarytable):
00135 raise 'non-existing table '+c.runsummarytable
00136 inputData=coral.AttributeList()
00137 inputData.extend('starttime','time stamp')
00138 inputData.extend('stoptime','time stamp')
00139 inputData.extend('runnum','unsigned int')
00140 runs=runtimedict.keys()
00141 runs.sort()
00142 for runnum in runs:
00143 (startTimeT,stopTimeT)=runtimedict[runnum]
00144 inputData['starttime'].setData(startTimeT)
00145 inputData['stoptime'].setData(stopTimeT)
00146 inputData['runnum'].setData(int(runnum))
00147 nchanged=schema.tableHandle(c.runsummarytable).dataEditor().updateRows('STARTTIME=:starttime,STOPTIME=:stoptime','RUNNUM=:runnum',inputData)
00148 print 'run '+str(runnum)+' update '+str(nchanged)+' row with starttime ,stoptime'
00149 print startTimeT,stopTimeT
00150 totalchanged=totalchanged+nchanged
00151 if c.isdryrun:
00152 dbsession.transaction().rollback()
00153 else:
00154 dbsession.transaction().commit()
00155 except Exception,e:
00156 print str(e)
00157 dbsession.transaction().rollback()
00158 del dbsession
00159 print 'total number of rows changed: ',totalchanged
00160
00161 def recalibrateLumiForRun(dbsession,c,delta,runnums):
00162 '''
00163 update LUMISUMMARY set INSTLUMI=:delta*INSTLUMI where RUNNUM in (1,3,57,90)
00164 '''
00165 updaterows=0
00166 try:
00167 dbsession.transaction().start(False)
00168 schema=dbsession.nominalSchema()
00169 if not schema:
00170 raise 'cannot connect to schema'
00171 if not schema.existsTable(c.lumisummarytable):
00172 raise 'non-existing table '+c.lumisummarytable
00173 runliststring=','.join([str(x) for x in runnums])
00174 print 'applying delta '+delta+' on run list '+runliststring
00175 nchanged=0
00176 inputData=coral.AttributeList()
00177 inputData.extend('delta','float')
00178 inputData['delta'].setData(float(delta))
00179 nchanged=schema.tableHandle(c.lumisummarytable).dataEditor().updateRows('INSTLUMI=INSTLUMI*:delta','RUNNUM in ('+runliststring+')',inputData)
00180 print 'total number of row changed ',nchanged
00181 if c.isdryrun:
00182 dbsession.transaction().rollback()
00183 else:
00184 dbsession.transaction().commit()
00185 return nchanged
00186 except Exception,e:
00187 print str(e)
00188 dbsession.transaction().rollback()
00189 del dbsession
00190
00191 def GTdeadtimeBeamActiveForRun(dbsession,c,runnum):
00192 '''
00193 select lsnr,counts from cms_gt_mon.gt_mon_trig_dead_view where runnr=:runnumber and deadcounter='DeadtimeBeamActive' order by lsnr;
00194 return result{lumisection:deadtimebeamactive}
00195
00196 '''
00197 result={}
00198 try:
00199 dbsession.transaction().start(True)
00200 schema=dbsession.schema(c.gtmonschema)
00201
00202 if not schema:
00203 raise Exception('cannot connect to schema '+c.gtmonschema)
00204 if not schema.existsView(c.gtdeadview):
00205 raise Exception('non-existing view '+c.gtdeadview)
00206
00207 deadOutput=coral.AttributeList()
00208 deadOutput.extend("lsnr","unsigned int")
00209 deadOutput.extend("deadcount","unsigned long long")
00210
00211 deadBindVarList=coral.AttributeList()
00212 deadBindVarList.extend("runnumber","unsigned int")
00213 deadBindVarList.extend("countername","string")
00214 deadBindVarList["runnumber"].setData(int(runnum))
00215 deadBindVarList["countername"].setData('DeadtimeBeamActive')
00216
00217 query=schema.newQuery()
00218 query.addToTableList(c.gtdeadview)
00219 query.addToOutputList('LSNR','lsnr')
00220 query.addToOutputList('COUNTS','deadcount')
00221 query.setCondition('RUNNR=:runnumber AND DEADCOUNTER=:countername',deadBindVarList)
00222 query.addToOrderList('lsnr')
00223 query.defineOutput(deadOutput)
00224
00225 cursor=query.execute()
00226 while cursor.next():
00227 cmslsnum=cursor.currentRow()['lsnr'].data()
00228 deadcount=cursor.currentRow()['deadcount'].data()
00229 result[cmslsnum]=deadcount
00230
00231 del query
00232 return result
00233 except Exception,e:
00234 print str(e)
00235 dbsession.transaction().rollback()
00236 del dbsession
00237
00238 def WBMdeadtimeBeamActiveForRun(dbsession,c,runnum):
00239 '''
00240 select LUMISEGMENTNR,DEADTIMEBEAMACTIVE from cms_wbm.LEVEL1_TRIGGER_CONDITIONS where RUNNUMBER=:runnum order by LUMISEGMENTNR;
00241 return result{lumisection:deadtimebeamactive}
00242
00243 '''
00244 result={}
00245 try:
00246 dbsession.transaction().start(True)
00247 schema=dbsession.nominalSchema()
00248 if not schema:
00249 raise Exception('cannot connect to schema'+c.wbmschema)
00250 if not schema.existsTable(c.wbmdeadtable):
00251 raise Exception('non-existing table'+c.wbmdeadtable)
00252
00253 deadOutput=coral.AttributeList()
00254 deadOutput.extend("lsnr","unsigned int")
00255 deadOutput.extend("deadcount","unsigned long long")
00256
00257 deadBindVarList=coral.AttributeList()
00258 deadBindVarList.extend("runnum","unsigned int")
00259 deadBindVarList["runnum"].setData(int(runnum))
00260
00261 query=schema.newQuery()
00262 query.addToTableList(c.wbmdeadtable)
00263 query.addToOutputList('LUMISEGMENTNR','lsnr')
00264 query.addToOutputList('DEADTIMEBEAMACTIVE','deadcount')
00265 query.setCondition('RUNNUMBER=:runnum',deadBindVarList)
00266 query.addToOrderList('LUMISEGMENTNR')
00267 query.defineOutput(deadOutput)
00268
00269 cursor=query.execute()
00270 while cursor.next():
00271 cmslsnum=cursor.currentRow()['lsnr'].data()
00272 deadcount=cursor.currentRow()['deadcount'].data()
00273 result[cmslsnum]=deadcount
00274
00275 del query
00276 return result
00277 except Exception,e:
00278 print str(e)
00279 dbsession.transaction().rollback()
00280 del dbsession
00281
00282 def patchDeadtimeForRun(dbsession,c,runnum,deadtimeDict):
00283 '''
00284 input: deadtimeDict{ls:deadtimebeamactive}
00285 loop over input
00286 update TRG set DEADTIME=:deadtimebeamactive where RUNNUM=:runnum and CMSLSNUM=:lsnum
00287 output: number of rows changed
00288 '''
00289 totalchanged=0
00290 try:
00291 dbsession.transaction().start(False)
00292 schema=dbsession.nominalSchema()
00293 if not schema:
00294 raise Exception('cannot connect to schema ')
00295 if not schema.existsTable(c.lumitrgtable):
00296 raise Exception('non-existing table '+c.lumitrgtable)
00297 for lsnum,deadtimebeamactive in deadtimeDict.items():
00298 nchanged=0
00299 inputData=coral.AttributeList()
00300 inputData.extend('deadtimebeamactive','unsigned int')
00301 inputData.extend('runnum','unsigned int')
00302 inputData.extend('lsnum','unsigned int')
00303 inputData['deadtimebeamactive'].setData(deadtimebeamactive)
00304 inputData['runnum'].setData(runnum)
00305 inputData['lsnum'].setData(lsnum)
00306 nchanged=schema.tableHandle(c.lumitrgtable).dataEditor().updateRows('DEADTIME=:deadtimebeamactive','RUNNUM=:runnum AND CMSLSNUM=:lsnum',inputData)
00307 print 'rows changed for ls ',str(lsnum),str(nchanged)
00308 totalchanged+=nchanged
00309 dbsession.transaction().commit()
00310 return totalchanged
00311 except Exception,e:
00312 print str(e)
00313 dbsession.transaction().rollback()
00314 del dbsession
00315
00316 def main():
00317 c=constants()
00318 parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Patch LumiData")
00319 parser.add_argument('-c',dest='destination',action='store',required=True,help='destination lumi db (required)')
00320 parser.add_argument('-s',dest='source',action='store',required=False,help='source db (required except for lumicalib)')
00321 parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file (required)')
00322 parser.add_argument('-r',dest='runnumber',action='store',required=False,help='run number (optional)')
00323 parser.add_argument('-i',dest='inputfile',action='store',required=False,help='run selection file(optional)')
00324 parser.add_argument('-delta',dest='delta',action='store',required=False,help='calibration factor wrt old data in lumiDB (required for lumicalib)')
00325 parser.add_argument('action',choices=['deadtimeGT','deadtimeWBM','lumicalib','runtimestamp'],help='deadtimeGT: patch deadtime to deadtimebeamactive,\ndeadtimeWBM: patch deadtimeWBM to deadtimebeamactive,\nlumicalib: recalibrate inst lumi by delta where delta>1\n runtimestamp: add start,stop run timestamp where empty')
00326 parser.add_argument('--dryrun',dest='dryrun',action='store_true',help='only print datasource query result, do not update destination')
00327
00328 parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
00329 args=parser.parse_args()
00330 runnumber=args.runnumber
00331 destConnect=args.destination
00332 sourceConnect=args.source
00333 if args.authpath and len(args.authpath)!=0:
00334 os.environ['CORAL_AUTH_PATH']=args.authpath
00335 svc=coral.ConnectionService()
00336 sourcesession=None
00337 if sourceConnect:
00338 sourcesession=svc.connect(sourceConnect,accessMode=coral.access_ReadOnly)
00339 sourcesession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
00340 sourcesession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
00341 destsession=svc.connect(destConnect,accessMode=coral.access_Update)
00342 destsession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
00343 destsession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
00344 if args.debug:
00345 msg=coral.MessageStream('')
00346 msg.setMsgVerbosity(coral.message_Level_Debug)
00347 if args.dryrun:
00348 c.isdryrun=True
00349 else:
00350 c.isdryrun=False
00351
00352 deadresult={}
00353
00354 if args.action == 'deadtimeGT':
00355 if not sourceConnect:
00356 raise Exception('deadtimeGT action requies -s option for source connection string')
00357 deadresult=GTdeadtimeBeamActiveForRun(sourcesession,c,runnumber)
00358 print 'reading from ',sourceConnect
00359 print 'run : ',runnumber
00360 print 'LS:deadtimebeamactive'
00361
00362 if deadresult and len(deadresult)!=0:
00363 for cmsls,deadtimebeamactive in deadresult.items():
00364 print cmsls,deadtimebeamactive
00365 else:
00366 print 'no deadtime found for run ',runnumber
00367 print 'exit'
00368 return
00369 print 'total LS: ',len(deadresult)
00370
00371 if len(deadresult)!=max( [ x for x in deadresult.keys() ] ):
00372 print 'total ls: ',len(deadresult)
00373
00374 print 'alert: missing Lumi Sections in the middle'
00375 for x in range(1,max( [ x for x in deadresult.keys()] ) ):
00376 if not deadresult.has_key(x):
00377 print 'filling up LS deadtime with 0: LS : ',x
00378 deadresult[x]=0
00379
00380 if not args.dryrun:
00381 print 'updating ',destConnect
00382 nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult)
00383 print 'number of updated rows ',nupdated
00384 elif args.action == 'deadtimeWBM':
00385 if not sourceConnect:
00386 raise Exception('deadtimeWBM action requies -s option for source connection string')
00387 deadresult=WBMdeadtimeBeamActiveForRun(sourcesession,c,runnumber)
00388 print 'reading from ',sourceConnect
00389 print 'run : ',runnumber
00390 print 'LS:deadtimebeamactive'
00391
00392 if deadresult and len(deadresult)!=0:
00393 for cmsls,deadtimebeamactive in deadresult.items():
00394 print cmsls,deadtimebeamactive
00395 else:
00396 print 'no deadtime found for run ',runnumber
00397 print 'exit'
00398 return
00399 print 'total LS: ',len(deadresult)
00400 if len(deadresult)!=max( [ (deadresult[x],x) for x in deadresult])[1]:
00401 print 'alert: missing Lumi Sections in the middle'
00402 for x in range(1,max( [ (deadresult[x],x) for x in deadresult])[1]):
00403 if not deadresult.has_key(x):
00404 print 'filling up LS deadtime with 0: LS : ',x
00405 deadresult[x]=0
00406 print deadresult
00407 if not args.dryrun:
00408 print 'updating ',destConnect
00409 nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult)
00410 print 'number of updated rows ',nupdated
00411 elif args.action == 'lumicalib':
00412 if not args.delta or args.delta==0:
00413 raise Exception('Must provide non-zero -delta argument')
00414 runnums=[]
00415 if args.runnumber:
00416 runnums.append(args.runnumber)
00417 elif args.inputfile:
00418 basename,extension=os.path.splitext(args.inputfile)
00419 if extension=='.csv':
00420 fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile)
00421 else:
00422 f=open(args.inputfile,'r')
00423 inputfilecontent=f.read()
00424 fileparsingResult=selectionParser.selectionParser(inputfilecontent)
00425 if not fileparsingResult:
00426 raise Exception('failed to parse the input file '+ifilename)
00427
00428 runnums=fileparsingResult.runs()
00429
00430 else:
00431 raise Exception('Must provide -r or -i argument as input')
00432 nupdated=recalibrateLumiForRun(destsession,c,args.delta,runnums)
00433 elif args.action == 'runtimestamp':
00434 if not sourceConnect:
00435 raise Exception('runtimestamp action requies -s option for source connection string')
00436 if not args.runnumber and not args.inputfile:
00437 runnums=missingTimeRuns(destsession,c)
00438 print 'these runs miss start/stop time: ',runnums
00439 print 'total : ',len(runnums)
00440 elif args.runnumber:
00441 runnums=[int(args.runnumber)]
00442 elif args.inputfile:
00443 basename,extension=os.path.splitext(args.inputfile)
00444 if extension=='.csv':
00445 fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile)
00446 else:
00447 f=open(args.inputfile,'r')
00448 inputfilecontent=f.read()
00449 fileparsingResult=selectionParser.selectionParser(inputfilecontent)
00450 if not fileparsingResult:
00451 raise Exception('failed to parse the input file '+ifilename)
00452 runnums=fileparsingResult.runs()
00453 result=getTimeForRun(sourcesession,c,runnums)
00454
00455
00456
00457
00458
00459
00460
00461 addTimeForRun(destsession,c,result)
00462 if sourcesession:
00463 del sourcesession
00464 del destsession
00465 del svc
00466
00467 if __name__=='__main__':
00468 main()
00469
1.7.3