Reference Manual

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 ########################################################################
00004 # Command to calculate luminosity from HF measurement stored in lumiDB #
00005 #                                                                      #
00006 # Author:      Zhen Xie                                                #
00007 ########################################################################
00008 
00009 import os,sys,time
00010 from RecoLuminosity.LumiDB import sessionManager,lumiTime,inputFilesetParser,csvSelectionParser,selectionParser,csvReporter,argparse,CommonUtil,revisionDML,lumiCalcAPI,lumiReport,RegexValidator,normDML
00011         
00012 beamChoices=['PROTPHYS','IONPHYS','PAPHYS']
00013 
00014 def parseInputFiles(inputfilename):
00015     '''
00016     output ({run:[cmsls,cmsls,...]},[[resultlines]])
00017     '''
00018     selectedrunlsInDB={}
00019     resultlines=[]
00020     p=inputFilesetParser.inputFilesetParser(inputfilename)
00021     runlsbyfile=p.runsandls()
00022     selectedProcessedRuns=p.selectedRunsWithresult()
00023     selectedNonProcessedRuns=p.selectedRunsWithoutresult()
00024     resultlines=p.resultlines()
00025     for runinfile in selectedNonProcessedRuns:
00026         selectedrunlsInDB[runinfile]=runlsbyfile[runinfile]
00027     return (selectedrunlsInDB,resultlines)
00028 
00029 ##############################
00030 ## ######################## ##
00031 ## ## ################## ## ##
00032 ## ## ## Main Program ## ## ##
00033 ## ## ################## ## ##
00034 ## ######################## ##
00035 ##############################
00036 
00037 if __name__ == '__main__':
00038     parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description = "Lumi Calculation",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
00039     allowedActions = ['overview', 'delivered', 'recorded', 'lumibyls','lumibylsXing']
00040     beamModeChoices = [ "stable"]
00041     amodetagChoices = [ "PROTPHYS","IONPHYS",'PAPHYS' ]
00042     xingAlgoChoices =[ "OCC1","OCC2","ET"]
00043 
00044     #
00045     # parse arguments
00046     #  
00047     ################################################
00048     # basic arguments
00049     ################################################
00050     parser.add_argument('action',choices=allowedActions,
00051                         help='command actions')
00052     parser.add_argument('-c',dest='connect',action='store',
00053                         required=False,
00054                         help='connect string to lumiDB,optional',
00055                         default='frontier://LumiCalc/CMS_LUMI_PROD')
00056     parser.add_argument('-P',dest='authpath',action='store',
00057                         required=False,
00058                         help='path to authentication file')
00059     parser.add_argument('-r',dest='runnumber',action='store',
00060                         type=int,
00061                         required=False,
00062                         help='run number')
00063     parser.add_argument('-o',dest='outputfile',action='store',
00064                         required=False,
00065                         help='output to csv file' )
00066     
00067     #################################################
00068     #arg to select exact run and ls
00069     #################################################
00070     parser.add_argument('-i',dest='inputfile',action='store',
00071                         required=False,
00072                         help='lumi range selection file')
00073     #################################################
00074     #arg to select exact hltpath or pattern
00075     #################################################
00076     parser.add_argument('--hltpath',dest='hltpath',action='store',
00077                         default=None,required=False,
00078                         help='specific hltpath or hltpath pattern to calculate the effectived luminosity')
00079     #################################################
00080     #versions control
00081     #################################################
00082     parser.add_argument('--normtag',dest='normtag',action='store',
00083                         required=False,
00084                         help='version of lumi norm/correction')
00085     parser.add_argument('--datatag',dest='datatag',action='store',
00086                         required=False,
00087                         help='version of lumi/trg/hlt data')
00088 
00089     ###############################################
00090     # run filters
00091     ###############################################
00092     parser.add_argument('-f','--fill',dest='fillnum',action='store',
00093                         default=None,required=False,
00094                         help='fill number (optional) ')
00095     parser.add_argument('--amodetag',dest='amodetag',action='store',
00096                         choices=amodetagChoices,
00097                         required=False,
00098                         help='specific accelerator mode choices [PROTOPHYS,IONPHYS,PAPHYS] (optional)')
00099     parser.add_argument('--beamenergy',dest='beamenergy',action='store',
00100                         type=float,
00101                         default=None,
00102                         help='nominal beam energy in GeV')
00103     parser.add_argument('--beamfluctuation',dest='beamfluctuation',
00104                         type=float,action='store',
00105                         default=0.2,
00106                         required=False,
00107                         help='fluctuation in fraction allowed to nominal beam energy, default 0.2, to be used together with -beamenergy  (optional)')
00108                         
00109     parser.add_argument('--begin',dest='begin',action='store',
00110                         default=None,
00111                         required=False,
00112                         type=RegexValidator.RegexValidator("^\d\d/\d\d/\d\d \d\d:\d\d:\d\d$|^\d{6}$|^\d{4}$","wrong format"),
00113                         help='min run start time (mm/dd/yy hh:mm:ss),min fill or min run'
00114                         )                        
00115     parser.add_argument('--end',dest='end',action='store',
00116                         default=None,
00117                         required=False,
00118                         type=RegexValidator.RegexValidator("^\d\d/\d\d/\d\d \d\d:\d\d:\d\d$|^\d{6}$|^\d{4}$","wrong format"),
00119                         help='max run start time (mm/dd/yy hh:mm:ss),max fill or max run'
00120                         )
00121     parser.add_argument('--minBiasXsec',dest='minbiasxsec',action='store',
00122                         default=69300.0,
00123                         type=float,
00124                         required=False,
00125                         help='minbias cross-section in ub'
00126                         )
00127     #############################################
00128     #ls filter 
00129     #############################################
00130     parser.add_argument('-b',dest='beammode',action='store',
00131                         choices=beamModeChoices,
00132                         required=False,
00133                         help='beam mode choices [stable]')
00134 
00135     parser.add_argument('--xingMinLum', dest = 'xingMinLum',
00136                         type=float,
00137                         default=1e-03,
00138                         required=False,
00139                         help='Minimum perbunch luminosity to print, default=1e-03/ub')
00140     
00141     parser.add_argument('--xingAlgo', dest = 'xingAlgo',
00142                         default='OCC1',
00143                         required=False,
00144                         help='algorithm name for per-bunch lumi ')
00145     
00146     #############################################
00147     #global scale factor
00148     #############################################        
00149     parser.add_argument('-n',dest='scalefactor',action='store',
00150                         type=float,
00151                         default=1.0,
00152                         required=False,
00153                         help='user defined global scaling factor on displayed lumi values,optional')
00154 
00155     #################################################
00156     #command configuration 
00157     #################################################
00158     parser.add_argument('--siteconfpath',dest='siteconfpath',action='store',
00159                         default=None,
00160                         required=False,
00161                         help='specific path to site-local-config.xml file, optional. If path undefined, fallback to cern proxy&server')
00162     
00163     parser.add_argument('--headerfile',dest='headerfile',action='store',
00164                         default=None,
00165                         required=False,
00166                         help='write command header output to specified file'
00167                        )
00168     #################################################
00169     #switches
00170     #################################################
00171     parser.add_argument('--without-correction',
00172                         dest='withoutNorm',
00173                         action='store_true',
00174                         help='without any correction/calibration'
00175                         )
00176     parser.add_argument('--without-checkforupdate',
00177                         dest='withoutCheckforupdate',
00178                         action='store_true',
00179                         help='without check for update'
00180                         )                    
00181     #parser.add_argument('--verbose',
00182     #                    dest='verbose',
00183     #                    action='store_true',
00184     #                    help='verbose mode for printing'
00185     #                    )
00186     parser.add_argument('--nowarning',
00187                         dest='nowarning',
00188                         action='store_true',
00189                         help='suppress bad for lumi warnings'
00190                         )
00191     parser.add_argument('--debug',
00192                         dest='debug',
00193                         action='store_true',
00194                         help='debug'
00195                         )
00196     options=parser.parse_args()
00197     if not options.runnumber and not options.inputfile and not options.fillnum and not options.begin:
00198         raise RuntimeError('at least one run selection argument in [-r,-f,-i,--begin] is required')
00199     reqrunmin=None
00200     reqfillmin=None
00201     reqtimemin=None
00202     reqrunmax=None
00203     reqfillmax=None
00204     reqtimemax=None
00205     timeFilter=[None,None]
00206     noWarning=options.nowarning
00207     pbeammode = None
00208     if options.beammode=='stable':
00209         pbeammode = 'STABLE BEAMS'
00210     iresults=[]
00211     reqTrg=False
00212     reqHlt=False
00213     if options.action=='overview' or options.action=='lumibyls' or options.action=='lumibylsXing':
00214         reqTrg=True
00215     if options.action=='recorded':
00216         reqTrg=True
00217         reqHlt=True
00218     if options.runnumber:
00219         reqrunmax=options.runnumber
00220         reqrunmin=options.runnumber
00221     if options.fillnum:
00222         reqfillmin=options.fillnum
00223         reqfillmax=options.fillnum
00224         
00225     
00226     if options.begin:
00227         (runbeg,fillbeg,timebeg)=CommonUtil.parseTime(options.begin)
00228         if runbeg: #there's --begin runnum #priority run,fill,time
00229             if not reqrunmin:# there's no -r, then take this
00230                 reqrunmin=runbeg
00231         elif fillbeg:
00232             if not reqfillmin:
00233                 reqfillmin=fillbeg
00234         elif timebeg:
00235             reqtimemin=timebeg
00236         if reqtimemin:
00237             lute=lumiTime.lumiTime()
00238             reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S')
00239             timeFilter[0]=reqtimeminT
00240     if options.end:
00241         (runend,fillend,timeend)=CommonUtil.parseTime(options.end)
00242         if runend:
00243             if not reqrunmax:#priority run,fill,time
00244                 reqrunmax=runend
00245         elif fillend:
00246             if not reqfillmax:
00247                 reqfillmax=fillend
00248         elif timeend:
00249             reqtimemax=timeend
00250         if reqtimemax:
00251             lute=lumiTime.lumiTime()
00252             reqtimemaxT=lute.StrToDatetime(reqtimemax,customfm='%m/%d/%y %H:%M:%S')
00253             timeFilter[1]=reqtimemaxT
00254     if options.inputfile and (reqtimemax or reqtimemin):
00255         #if use time and file filter together, there's no point of warning about missing LS,switch off
00256         noWarning=True
00257         
00258     ##############################################################
00259     # check working environment
00260     ##############################################################
00261     workingversion='UNKNOWN'
00262     updateversion='NONE'
00263     thiscmmd=sys.argv[0]
00264     if not options.withoutCheckforupdate:
00265         from RecoLuminosity.LumiDB import checkforupdate
00266         cmsswWorkingBase=os.environ['CMSSW_BASE']
00267         if not cmsswWorkingBase:
00268             print 'Please check out RecoLuminosity/LumiDB from CVS,scram b,cmsenv'
00269             sys.exit(11)
00270         c=checkforupdate.checkforupdate()
00271         workingversion=c.runningVersion(cmsswWorkingBase,'lumiCalc2.py',isverbose=False)
00272         if workingversion:
00273             updateversionList=c.checkforupdate(workingversion,isverbose=False)
00274             if updateversionList:
00275                 updateversion=updateversionList[-1][0]
00276     #
00277     # check DB environment
00278     #
00279     if options.authpath:
00280         os.environ['CORAL_AUTH_PATH'] = options.authpath
00281         
00282     #############################################################
00283     #pre-check option compatibility
00284     #############################################################
00285 
00286     if options.action=='recorded':
00287         if not options.hltpath:
00288             raise RuntimeError('argument --hltpath pathname is required for recorded action')
00289         
00290     svc=sessionManager.sessionManager(options.connect,
00291                                       authpath=options.authpath,
00292                                       siteconfpath=options.siteconfpath,
00293                                       debugON=options.debug)
00294     session=svc.openSession(isReadOnly=True,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
00295 
00296     ##############################################################
00297     # check run/ls list
00298     ##############################################################
00299     irunlsdict={}
00300     rruns=[]
00301     session.transaction().start(True)
00302     filerunlist=None
00303     if options.inputfile:
00304         (irunlsdict,iresults)=parseInputFiles(options.inputfile)
00305         filerunlist=irunlsdict.keys()
00306         
00307     
00308     #if not irunlsdict: #no file
00309     #    irunlsdict=dict(zip(rruns,[None]*len(rruns)))
00310     #else:
00311     #    for selectedrun in irunlsdict.keys():#if there's further filter on the runlist,clean input dict
00312     #        if selectedrun not in rruns:
00313     #            del irunlsdict[selectedrun]
00314     
00315     ##############################################################
00316     # check datatag
00317     # #############################################################       
00318     datatagname=options.datatag
00319     if not datatagname:
00320         (datatagid,datatagname)=revisionDML.currentDataTag(session.nominalSchema())
00321     else:
00322         datatagid=revisionDML.getDataTagId(session.nominalSchema(),datatagname)
00323 
00324     dataidmap=lumiCalcAPI.runList(session.nominalSchema(),datatagid,runmin=reqrunmin,runmax=reqrunmax,fillmin=reqfillmin,fillmax=reqfillmax,startT=reqtimemin,stopT=reqtimemax,l1keyPattern=None,hltkeyPattern=None,amodetag=options.amodetag,nominalEnergy=options.beamenergy,energyFlut=options.beamfluctuation,requiretrg=reqTrg,requirehlt=reqHlt,preselectedruns=filerunlist)
00325     if not dataidmap:
00326         print '[INFO] No qualified run found, do nothing'
00327         sys.exit(14)
00328     rruns=[]
00329     #crosscheck dataid value
00330     for irun,(lid,tid,hid) in dataidmap.items():
00331         if not lid:
00332             print '[INFO] No qualified lumi data found for run, ',irun
00333         if reqTrg and not tid:
00334             print '[INFO] No qualified trg data found for run ',irun
00335             continue
00336         if reqHlt and not hid:
00337             print '[INFO] No qualified hlt data found for run ',irun
00338             continue
00339         rruns.append(irun)
00340     if not irunlsdict: #no file
00341         irunlsdict=dict(zip(rruns,[None]*len(rruns)))
00342     else:
00343         for selectedrun in irunlsdict.keys():#if there's further filter on the runlist,clean input dict
00344             if selectedrun not in rruns:
00345                 del irunlsdict[selectedrun]
00346     if not irunlsdict:
00347         print '[INFO] No qualified run found, do nothing'
00348         sys.exit(13)
00349     ###############################################################
00350     # check normtag and get norm values if required
00351     ###############################################################
00352     normname='NONE'
00353     normid=0
00354     normvalueDict={}
00355     if not options.withoutNorm:
00356         normname=options.normtag
00357         if not normname:
00358             normmap=normDML.normIdByType(session.nominalSchema(),lumitype='HF',defaultonly=True)
00359             if len(normmap):
00360                 normname=normmap.keys()[0]
00361                 normid=normmap[normname]
00362         else:
00363             normid=normDML.normIdByName(session.nominalSchema(),normname)
00364         if not normid:
00365             raise RuntimeError('[ERROR] cannot resolve norm/correction')
00366             sys.exit(12)
00367         normvalueDict=normDML.normValueById(session.nominalSchema(),normid) #{since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]}
00368     session.transaction().commit()
00369     lumiReport.toScreenHeader(thiscmmd,datatagname,normname,workingversion,updateversion,'HF',toFile=options.headerfile)
00370 
00371                 
00372     ##################
00373     # ls level       #
00374     ##################
00375     session.transaction().start(True)
00376     
00377     GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict,dataidmap,lumitype='HF')
00378     if options.action == 'delivered':
00379         result=lumiCalcAPI.deliveredLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF')
00380         lumiReport.toScreenTotDelivered(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00381     if options.action == 'overview':
00382         result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF')
00383         lumiReport.toScreenOverview(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00384     if options.action == 'lumibyls':
00385         if not options.hltpath:
00386             result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF',minbiasXsec=options.minbiasxsec)
00387             lumiReport.toScreenLumiByLS(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)            
00388         else:
00389             hltname=options.hltpath
00390             hltpat=None
00391             if hltname=='*' or hltname=='all':
00392                 hltname=None
00393             elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
00394                 hltpat=hltname
00395                 hltname=None
00396             result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=options.xingMinLum,withBeamIntensity=False,lumitype='HF')
00397             lumiReport.toScreenLSEffective(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00398     if options.action == 'recorded':#recorded actually means effective because it needs to show all the hltpaths...
00399         hltname=options.hltpath
00400         hltpat=None
00401         if hltname is not None:
00402             if hltname=='*' or hltname=='all':
00403                 hltname=None
00404             elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
00405                 hltpat=hltname
00406                 hltname=None
00407         result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=options.xingMinLum,withBeamIntensity=False,lumitype='HF')
00408         lumiReport.toScreenTotEffective(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00409     if options.action == 'lumibylsXing':
00410          result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,withBXInfo=True,bxAlgo=options.xingAlgo,xingMinLum=options.xingMinLum,withBeamIntensity=False,lumitype='HF')
00411          outfile=options.outputfile
00412          if not outfile:
00413              print '[WARNING] no output file given. lumibylsXing writes per-bunch lumi only to default file lumibylsXing.csv'
00414              outfile='lumibylsXing.csv'           
00415          lumiReport.toCSVLumiByLSXing(result,options.scalefactor,outfile,irunlsdict=irunlsdict,noWarning=noWarning)
00416     session.transaction().commit()
00417     del session
00418     del svc
/data/refman/pasoursint/CMSSW_5_3_10_patch1/src/RecoLuminosity/LumiDB/scripts/lumiCalc2.py
