Reference Manual

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 VERSION='2.00'
00003 import os,sys,time
00004 import coral
00005 #import optparse
00006 from RecoLuminosity.LumiDB import lumiTime,inputFilesetParser,csvSelectionParser, selectionParser,csvReporter,argparse,CommonUtil,lumiQueryAPI
00007 #import RecoLuminosity.LumiDB.lumiQueryAPI as LumiQueryAPI
00008 #from pprint import pprint
00009 
00010 def getPerLSData(dbsession,inputRange,lumiversion='0001'):
00011     result={}#{run:[[cmslsnum,orbittime,orbittimestamp,delivered,recorded]]}
00012     datacollector={}
00013     if isinstance(inputRange, str):
00014         datacollector[int(inputRange)]=[]
00015     else:
00016         for run in inputRange.runs():
00017             datacollector[run]=[]
00018     try:
00019         dbsession.transaction().start(True)
00020         schema=dbsession.nominalSchema()
00021         for run in datacollector.keys():
00022             runsummaryOut=[]  #[fillnum,sequence,hltkey,starttime,stoptime]
00023             lumisummaryOut=[] #[[cmslsnum,instlumi,numorbit,startorbit,beamstatus,beamenergy,cmsalive]]
00024             trgOut={} #{cmslsnum:[trgcount,deadtime,bitname,prescale]}
00025             q=schema.newQuery()
00026             runsummaryOut=lumiQueryAPI.runsummaryByrun(q,run)
00027             del q
00028             q=schema.newQuery()
00029             lumisummaryOut=lumiQueryAPI.lumisummaryByrun(q,run,lumiversion)
00030             del q
00031             q=schema.newQuery()
00032             trgOut=lumiQueryAPI.trgbitzeroByrun(q,run,)
00033             del q
00034             if len(runsummaryOut)!=0 and len(lumisummaryOut)!=0 and len(trgOut)!=0:
00035                 datacollector[run].append(runsummaryOut)
00036                 datacollector[run].append(lumisummaryOut)
00037                 datacollector[run].append(trgOut)
00038         dbsession.transaction().commit()
00039     except Exception, e:
00040         dbsession.transaction().rollback()
00041         del dbsession
00042         raise Exception, 'lumiCalc.getPerLSData:'+str(e)
00043     for run,perrundata in datacollector.items():
00044         result[run]=[]
00045         if len(perrundata)==0:
00046             continue
00047         runsummary=perrundata[0]
00048         lumisummary=perrundata[1]
00049         trg=perrundata[2]
00050         starttimestr=runsummaryOut[3]
00051         t=lumiTime.lumiTime()
00052         for dataperls in lumisummary:
00053             cmslsnum=dataperls[0]
00054             instlumi=dataperls[1]
00055             numorbit=dataperls[2]
00056             dellumi=instlumi*float(numorbit)*3564.0*25.0e-09
00057             startorbit=dataperls[3]
00058             orbittime=t.OrbitToTime(starttimestr,startorbit)
00059             orbittimestamp=time.mktime(orbittime.timetuple())+orbittime.microsecond/1e6
00060             trgcount=0
00061             deadtime=0
00062             prescale=0
00063             deadfrac=1.0
00064             if trg.has_key(cmslsnum):
00065                 trgcount=trg[cmslsnum][0]
00066                 deadtime=trg[cmslsnum][1]
00067                 prescale=trg[cmslsnum][3]
00068                 if trgcount!=0 and prescale!=0:
00069                     deadfrac=float(deadtime)/(float(trgcount)*float(prescale))
00070                 recordedlumi=dellumi*(1.0-deadfrac)
00071             result[run].append( [cmslsnum,orbittime,orbittimestamp,dellumi,recordedlumi] )
00072     return result
00073 
00074 def getValidationData(dbsession,run=None,cmsls=None):
00075     '''retrieve validation data per run or all
00076     input: runnum, if not runnum, retrive all
00077     output: {run:[[cmslsnum,flag,comment]]}
00078     '''
00079     try:
00080         dbsession.transaction().start(True)
00081         schema=dbsession.nominalSchema()
00082         queryHandle=dbsession.nominalSchema().newQuery()
00083         result=lumiQueryAPI.validation(queryHandle,run,cmsls)
00084         del queryHandle
00085         dbsession.transaction().commit()
00086     except Exception, e:
00087         dbsession.transaction().rollback()
00088         del dbsession
00089         raise Exception, 'lumiValidate.getValidationData:'+str(e)
00090     return result
00091 
00092 ##############################
00093 ## ######################## ##
00094 ## ## ################## ## ##
00095 ## ## ## Main Program ## ## ##
00096 ## ## ################## ## ##
00097 ## ######################## ##
00098 ##############################
00099 
00100 if __name__ == '__main__':
00101     parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description = "Lumi Calculations",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
00102     allowedActions = ['overview', 'delivered', 'recorded', 'lumibyls','lumibylstime','lumibylsXing','status']
00103     beamModeChoices = [ "stable", "quiet", "either"]
00104     # parse arguments
00105     parser.add_argument('action',choices=allowedActions,help='command actions')
00106     parser.add_argument('-c',dest='connect',action='store',required=False,help='connect string to lumiDB,optional',default='frontier://LumiCalc/CMS_LUMI_PROD')
00107     parser.add_argument('-P',dest='authpath',action='store',help='path to authentication file,optional')
00108     parser.add_argument('-n',dest='normfactor',action='store',type=float,default=1.0,help='normalization factor,optional')
00109     parser.add_argument('-r',dest='runnumber',action='store',type=int,help='run number,optional')
00110     parser.add_argument('-i',dest='inputfile',action='store',help='lumi range selection file,optional')
00111     parser.add_argument('-o',dest='outputfile',action='store',help='output to csv file,optional')
00112     parser.add_argument('-b',dest='beammode',action='store',choices=beamModeChoices,required=False,help='beam mode choice')
00113     parser.add_argument('-lumiversion',dest='lumiversion',action='store',default='0001',help='lumi data version, optional')
00114     parser.add_argument('-hltpath',dest='hltpath',action='store',default='all',help='specific hltpath to calculate the recorded luminosity,optional')
00115     parser.add_argument('-siteconfpath',dest='siteconfpath',action='store',help='specific path to site-local-config.xml file, optional. If path undefined, fallback to cern proxy&server')
00116     parser.add_argument('-xingMinLum', dest = 'xingMinLum', type=float, default=1e-3,required=False,help='Minimum luminosity considered for "lumibylsXing" action')
00117     parser.add_argument('--verbose',dest='verbose',action='store_true',help='verbose mode for printing' )
00118     parser.add_argument('--nowarning',dest='nowarning',action='store_true',help='suppress bad for lumi warnings' )
00119     parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
00120     options=parser.parse_args()
00121     if options.authpath:
00122         os.environ['CORAL_AUTH_PATH'] = options.authpath
00123     ## Let's start the fun
00124     if not options.inputfile and not options.runnumber:
00125         print "must specify either a run (-r) or an input run selection file (-i)"
00126         sys.exit()
00127 
00128     ## Save what we need in the parameters object
00129     parameters = lumiQueryAPI.ParametersObject()
00130     parameters.verbose     = options.verbose
00131     parameters.noWarnings  = options.nowarning
00132     parameters.norm        = options.normfactor
00133     parameters.lumiversion = options.lumiversion
00134     if options.beammode=='stable':
00135         parameters.beammode    = 'STABLE BEAMS'
00136     parameters.xingMinLum  = options.xingMinLum
00137     session,svc =  lumiQueryAPI.setupSession (options.connect or \
00138                                               'frontier://LumiCalc/CMS_LUMI_PROD',
00139                                                options.siteconfpath,parameters,options.debug)
00140     lumiXing = False
00141     if options.action in ['lumibylsXing','delivered','recorded','overview','lumibyls','lumibylstime']:
00142         if options.action == 'lumibylsXing':
00143            #action = 'lumibyls'
00144            parameters.lumiXing = True
00145            # we can't have lumiXing mode if we're not writing to a CSV
00146            # file
00147            #if not options.outputfile:
00148            #    raise RuntimeError, "You must specify an outputt file in 'lumibylsXing' mode"
00149         if options.runnumber:
00150             inputRange=str(options.runnumber)
00151         else:
00152             inputRange=inputFilesetParser.inputFilesetParser(options.inputfile)
00153         if not inputRange:
00154             print 'failed to parse the input file', options.inputfile
00155             raise 
00156 
00157         # Delivered
00158         if options.action ==  'delivered':
00159             lumidata=lumiQueryAPI.deliveredLumiForRange(session, parameters, inputRange)    
00160             if not options.outputfile:
00161                 lumiQueryAPI.printDeliveredLumi (lumidata, '')
00162             else:
00163                 lumidata.insert (0, ['run', 'nls', 'delivered', 'beammode'])
00164                 lumiQueryAPI.dumpData (lumidata, options.outputfile)
00165 
00166         # Recorded
00167         if options.action ==  'recorded':
00168             hltpath = ''
00169             if options.hltpath:
00170                 hltpath = options.hltpath
00171                 lumidata =  lumiQueryAPI.recordedLumiForRange (session, parameters, inputRange)
00172             if not options.outputfile:
00173                 lumiQueryAPI.printRecordedLumi (lumidata, parameters.verbose, hltpath)
00174             else:
00175                 todump = lumiQueryAPI.dumpRecordedLumi (lumidata, hltpath)
00176                 todump.insert (0, ['run', 'hltpath', 'recorded'])
00177                 lumiQueryAPI.dumpData (todump, options.outputfile)
00178                 
00179                 # Overview
00180         if options.action ==  'overview':
00181             hltpath=''
00182             if options.hltpath:
00183                 hltpath=options.hltpath
00184             delivereddata=lumiQueryAPI.deliveredLumiForRange(session, parameters, inputRange)
00185             recordeddata=lumiQueryAPI.recordedLumiForRange(session, parameters, inputRange)
00186             if not options.outputfile:
00187                 lumiQueryAPI.printOverviewData (delivereddata, recordeddata, hltpath)
00188             else:
00189                 todump = lumiQueryAPI.dumpOverview (delivereddata, recordeddata, hltpath)
00190                 if not hltpath:
00191                     hltpath = 'all'
00192                 todump.insert (0, ['run', 'delivered', 'recorded', 'hltpath:'+hltpath])
00193                 lumiQueryAPI.dumpData (todump, options.outputfile)
00194 
00195                 # Lumi by lumisection
00196         if options.action == 'lumibylstime':
00197             lsdata=getPerLSData(session,inputRange)#{run:[[ls,orbittime,orbittimestamp,delivered,recorded],[]]}
00198             runs=lsdata.keys()
00199             runs.sort()
00200             if not options.outputfile:
00201                 print 'run,cmslsnum, utctime, unixtimestamp, delivered, recorded'
00202                 for run in runs:
00203                     if len(lsdata[run])==0:continue #empty or non-existing run
00204                     for perlsdata in lsdata[run]:
00205                         if len(perlsdata)==0:
00206                             continue
00207                         print run,perlsdata[0],perlsdata[1],perlsdata[2],perlsdata[3],perlsdata[4]
00208                             
00209             else:
00210                 report=csvReporter.csvReporter(options.outputfile)
00211                 report.writeRow(['run','cmslsnum','utctime','unixtimestamp','delivered','recorded'])
00212                 for run in runs:
00213                     if len(lsdata[run])==0:continue #empty or non-existing run
00214                     for perlsdata in lsdata[run]:
00215                         if len(perlsdata)==0:
00216                             continue
00217                         report.writeRow([run,perlsdata[0],perlsdata[1],perlsdata[2],perlsdata[3],perlsdata[4]])
00218                             
00219         if options.action=='lumibyls' or options.action=='lumibylsXing':
00220             recordeddata=lumiQueryAPI.recordedLumiForRange(session, parameters, inputRange)
00221             # we got it, now we got to decide what to do with it
00222             if not options.outputfile:
00223                 lumiQueryAPI.printPerLSLumi (recordeddata, parameters.verbose)
00224             else:
00225                 todump = lumiQueryAPI.dumpPerLSLumi(recordeddata)
00226                 todump.insert (0, ['run', 'ls', 'delivered', 'recorded'])
00227                 lumiQueryAPI.dumpData (todump, options.outputfile)
00228         if not options.nowarning:
00229             result={}
00230             if isinstance(inputRange,str):
00231                 result=getValidationData(session,run=int(inputRange))
00232             else:
00233                 runsandls=inputRange.runsandls()
00234                 for runnum,lslist in runsandls.items():
00235                     dataperrun=getValidationData(session,run=runnum,cmsls=lslist)
00236                     if dataperrun.has_key(runnum):
00237                         result[runnum]=dataperrun[runnum]
00238             for run,perrundata in result.items():
00239                 totalsuspect=0
00240                 totalbad=0
00241                 totalunknown=0
00242                 for lsdata in perrundata:
00243                     if lsdata[1]=='UNKNOWN':
00244                         totalunknown+=1
00245                     if lsdata[1]=='SUSPECT':
00246                         totalsuspect+=1
00247                     if lsdata[1]=='BAD':
00248                         totalbad+=1
00249                 if totalsuspect!=0 or totalbad!=0 or totalunknown!=0:
00250                     print '[WARNING] : run '+str(run)+' : total non-GOOD LS: UNKNOWN '+str(totalunknown)+', BAD '+str(totalbad)+', SUSPECT '+str(totalsuspect)
00251     # relate to validation status
00252     elif options.action=='status':
00253         result={}
00254         if options.inputfile:
00255             p=inputFilesetParser.inputFilesetParser(options.inputfile)
00256             runsandls=p.runsandls()
00257             for runnum,lslist in runsandls.items():
00258                 dataperrun=getValidationData(session,run=runnum,cmsls=lslist)
00259                 result[runnum]=dataperrun[runnum]
00260         else:
00261             result=getValidationData(session,run=options.runnumber)
00262         runs=result.keys()
00263         runs.sort()
00264         if options.outputfile:
00265             r=csvReporter.csvReporter(options.outputfile)
00266             for run in runs:
00267                 for perrundata in result[run]:
00268                     r.writeRow([str(run),str(perrundata[0]),perrundata[1],perrundata[2]])
00269         else:
00270             for run in runs:
00271                 print '== ='
00272                 for lsdata in result[run]:
00273                     print str(run)+','+str(lsdata[0])+','+lsdata[1]+','+lsdata[2]
00274     del session
00275     del svc
/data/refman/pasoursint/CMSSW_4_4_5_patch3/src/RecoLuminosity/LumiDB/scripts/lumiCalc.py
