Reference Manual

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 ########################################################################
00004 # Command to calculate luminosity from HF measurement stored in lumiDB #
00005 #                                                                      #
00006 # Author:      Zhen Xie                                                #
00007 ########################################################################
00008 
00009 import os,sys,time
00010 from RecoLuminosity.LumiDB import sessionManager,lumiTime,inputFilesetParser,csvSelectionParser,selectionParser,csvReporter,argparse,CommonUtil,revisionDML,lumiCalcAPI,lumiReport,RegexValidator,normDML
00011         
00012 beamChoices=['PROTPHYS','IONPHYS','PAPHYS']
00013 
00014 def parseInputFiles(inputfilename):
00015     '''
00016     output ({run:[cmsls,cmsls,...]},[[resultlines]])
00017     '''
00018     selectedrunlsInDB={}
00019     resultlines=[]
00020     p=inputFilesetParser.inputFilesetParser(inputfilename)
00021     runlsbyfile=p.runsandls()
00022     selectedProcessedRuns=p.selectedRunsWithresult()
00023     selectedNonProcessedRuns=p.selectedRunsWithoutresult()
00024     resultlines=p.resultlines()
00025     for runinfile in selectedNonProcessedRuns:
00026         selectedrunlsInDB[runinfile]=runlsbyfile[runinfile]
00027     return (selectedrunlsInDB,resultlines)
00028 
00029 ##############################
00030 ## ######################## ##
00031 ## ## ################## ## ##
00032 ## ## ## Main Program ## ## ##
00033 ## ## ################## ## ##
00034 ## ######################## ##
00035 ##############################
00036 
00037 if __name__ == '__main__':
00038     parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description = "Lumi Calculation",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
00039     allowedActions = ['overview', 'delivered', 'recorded', 'lumibyls','lumibylsXing']
00040     beamModeChoices = [ "stable"]
00041     amodetagChoices = [ "PROTPHYS","IONPHYS",'PAPHYS' ]
00042     xingAlgoChoices =[ "OCC1","OCC2","ET"]
00043 
00044     #
00045     # parse arguments
00046     #  
00047     ################################################
00048     # basic arguments
00049     ################################################
00050     parser.add_argument('action',choices=allowedActions,
00051                         help='command actions')
00052     parser.add_argument('-c',dest='connect',action='store',
00053                         required=False,
00054                         help='connect string to lumiDB,optional',
00055                         default='frontier://LumiCalc/CMS_LUMI_PROD')
00056     parser.add_argument('-P',dest='authpath',action='store',
00057                         required=False,
00058                         help='path to authentication file')
00059     parser.add_argument('-r',dest='runnumber',action='store',
00060                         type=int,
00061                         required=False,
00062                         help='run number')
00063     parser.add_argument('-o',dest='outputfile',action='store',
00064                         required=False,
00065                         help='output to csv file' )
00066     
00067     #################################################
00068     #arg to select exact run and ls
00069     #################################################
00070     parser.add_argument('-i',dest='inputfile',action='store',
00071                         required=False,
00072                         help='lumi range selection file')
00073     #################################################
00074     #arg to select exact hltpath or pattern
00075     #################################################
00076     parser.add_argument('--hltpath',dest='hltpath',action='store',
00077                         default=None,required=False,
00078                         help='specific hltpath or hltpath pattern to calculate the effectived luminosity')
00079     #################################################
00080     #versions control
00081     #################################################
00082     parser.add_argument('--normtag',dest='normtag',action='store',
00083                         required=False,
00084                         help='version of lumi norm/correction')
00085     parser.add_argument('--datatag',dest='datatag',action='store',
00086                         required=False,
00087                         help='version of lumi/trg/hlt data')
00088 
00089     ###############################################
00090     # run filters
00091     ###############################################
00092     parser.add_argument('-f','--fill',dest='fillnum',action='store',
00093                         default=None,required=False,
00094                         help='fill number (optional) ')
00095     parser.add_argument('--amodetag',dest='amodetag',action='store',
00096                         choices=amodetagChoices,
00097                         required=False,
00098                         help='specific accelerator mode choices [PROTOPHYS,IONPHYS,PAPHYS] (optional)')
00099     parser.add_argument('--beamenergy',dest='beamenergy',action='store',
00100                         type=float,
00101                         default=None,
00102                         help='nominal beam energy in GeV')
00103     parser.add_argument('--beamfluctuation',dest='beamfluctuation',
00104                         type=float,action='store',
00105                         default=0.2,
00106                         required=False,
00107                         help='fluctuation in fraction allowed to nominal beam energy, default 0.2, to be used together with -beamenergy  (optional)')
00108                         
00109     parser.add_argument('--begin',dest='begin',action='store',
00110                         default=None,
00111                         required=False,
00112                         type=RegexValidator.RegexValidator("^\d\d/\d\d/\d\d \d\d:\d\d:\d\d$|^\d{6}$|^\d{4}$","wrong format"),
00113                         help='min run start time (mm/dd/yy hh:mm:ss),min fill or min run'
00114                         )                        
00115     parser.add_argument('--end',dest='end',action='store',
00116                         default=None,
00117                         required=False,
00118                         type=RegexValidator.RegexValidator("^\d\d/\d\d/\d\d \d\d:\d\d:\d\d$|^\d{6}$|^\d{4}$","wrong format"),
00119                         help='max run start time (mm/dd/yy hh:mm:ss),max fill or max run'
00120                         )
00121     parser.add_argument('--minBiasXsec',dest='minbiasxsec',action='store',
00122                         default=69300.0,
00123                         type=float,
00124                         required=False,
00125                         help='minbias cross-section in ub'
00126                         )
00127     #############################################
00128     #ls filter 
00129     #############################################
00130     parser.add_argument('-b',dest='beammode',action='store',
00131                         choices=beamModeChoices,
00132                         required=False,
00133                         help='beam mode choices [stable]')
00134 
00135     parser.add_argument('--xingMinLum', dest = 'xingMinLum',
00136                         type=float,
00137                         default=1e-03,
00138                         required=False,
00139                         help='Minimum perbunch luminosity to print, default=1e-03/ub')
00140     
00141     parser.add_argument('--xingAlgo', dest = 'xingAlgo',
00142                         default='OCC1',
00143                         required=False,
00144                         help='algorithm name for per-bunch lumi ')
00145     
00146     #############################################
00147     #global scale factor
00148     #############################################        
00149     parser.add_argument('-n',dest='scalefactor',action='store',
00150                         type=float,
00151                         default=1.0,
00152                         required=False,
00153                         help='user defined global scaling factor on displayed lumi values,optional')
00154 
00155     #################################################
00156     #command configuration 
00157     #################################################
00158     parser.add_argument('--siteconfpath',dest='siteconfpath',action='store',
00159                         default=None,
00160                         required=False,
00161                         help='specific path to site-local-config.xml file, optional. If path undefined, fallback to cern proxy&server')
00162     
00163     parser.add_argument('--headerfile',dest='headerfile',action='store',
00164                         default=None,
00165                         required=False,
00166                         help='write command header output to specified file'
00167                        )
00168     #################################################
00169     #switches
00170     #################################################
00171     parser.add_argument('--without-correction',
00172                         dest='withoutNorm',
00173                         action='store_true',
00174                         help='without any correction/calibration'
00175                         )
00176     parser.add_argument('--without-checkforupdate',
00177                         dest='withoutCheckforupdate',
00178                         action='store_true',
00179                         help='without check for update'
00180                         )                    
00181     #parser.add_argument('--verbose',
00182     #                    dest='verbose',
00183     #                    action='store_true',
00184     #                    help='verbose mode for printing'
00185     #                    )
00186     parser.add_argument('--nowarning',
00187                         dest='nowarning',
00188                         action='store_true',
00189                         help='suppress bad for lumi warnings'
00190                         )
00191     parser.add_argument('--debug',
00192                         dest='debug',
00193                         action='store_true',
00194                         help='debug'
00195                         )
00196     options=parser.parse_args()
00197     if not options.runnumber and not options.inputfile and not options.fillnum and not options.begin:
00198         raise RuntimeError('at least one run selection argument in [-r,-f,-i,--begin] is required')
00199     reqrunmin=None
00200     reqfillmin=None
00201     reqtimemin=None
00202     reqrunmax=None
00203     reqfillmax=None
00204     reqtimemax=None
00205     timeFilter=[None,None]
00206     noWarning=options.nowarning
00207     pbeammode = None
00208     if options.beammode=='stable':
00209         pbeammode = 'STABLE BEAMS'
00210     iresults=[]
00211     reqTrg=False
00212     reqHlt=False
00213     if options.action=='overview' or options.action=='lumibyls' or options.action=='lumibylsXing':
00214         reqTrg=True
00215         if options.action=='lumibyls' and options.hltpath:
00216             reqHlt=True
00217     if options.action=='recorded':
00218         reqTrg=True
00219         reqHlt=True
00220     if options.runnumber:
00221         reqrunmax=options.runnumber
00222         reqrunmin=options.runnumber
00223     if options.fillnum:
00224         reqfillmin=options.fillnum
00225         reqfillmax=options.fillnum
00226         
00227     
00228     if options.begin:
00229         (runbeg,fillbeg,timebeg)=CommonUtil.parseTime(options.begin)
00230         if runbeg: #there's --begin runnum #priority run,fill,time
00231             if not reqrunmin:# there's no -r, then take this
00232                 reqrunmin=runbeg
00233         elif fillbeg:
00234             if not reqfillmin:
00235                 reqfillmin=fillbeg
00236         elif timebeg:
00237             reqtimemin=timebeg
00238         if reqtimemin:
00239             lute=lumiTime.lumiTime()
00240             reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S')
00241             timeFilter[0]=reqtimeminT
00242     if options.end:
00243         (runend,fillend,timeend)=CommonUtil.parseTime(options.end)
00244         if runend:
00245             if not reqrunmax:#priority run,fill,time
00246                 reqrunmax=runend
00247         elif fillend:
00248             if not reqfillmax:
00249                 reqfillmax=fillend
00250         elif timeend:
00251             reqtimemax=timeend
00252         if reqtimemax:
00253             lute=lumiTime.lumiTime()
00254             reqtimemaxT=lute.StrToDatetime(reqtimemax,customfm='%m/%d/%y %H:%M:%S')
00255             timeFilter[1]=reqtimemaxT
00256     if options.inputfile and (reqtimemax or reqtimemin):
00257         #if use time and file filter together, there's no point of warning about missing LS,switch off
00258         noWarning=True
00259         
00260     ##############################################################
00261     # check working environment
00262     ##############################################################
00263     workingversion='UNKNOWN'
00264     updateversion='NONE'
00265     thiscmmd=sys.argv[0]
00266     if not options.withoutCheckforupdate:
00267         from RecoLuminosity.LumiDB import checkforupdate
00268         cmsswWorkingBase=os.environ['CMSSW_BASE']
00269         if not cmsswWorkingBase:
00270             print 'Please check out RecoLuminosity/LumiDB from CVS,scram b,cmsenv'
00271             sys.exit(11)
00272         c=checkforupdate.checkforupdate()
00273         workingversion=c.runningVersion(cmsswWorkingBase,'lumiCalc2.py',isverbose=False)
00274         if workingversion:
00275             updateversionList=c.checkforupdate(workingversion,isverbose=False)
00276             if updateversionList:
00277                 updateversion=updateversionList[-1][0]
00278     #
00279     # check DB environment
00280     #
00281     if options.authpath:
00282         os.environ['CORAL_AUTH_PATH'] = options.authpath
00283         
00284     #############################################################
00285     #pre-check option compatibility
00286     #############################################################
00287 
00288     if options.action=='recorded':
00289         if not options.hltpath:
00290             raise RuntimeError('argument --hltpath pathname is required for recorded action')
00291         
00292     svc=sessionManager.sessionManager(options.connect,
00293                                       authpath=options.authpath,
00294                                       siteconfpath=options.siteconfpath,
00295                                       debugON=options.debug)
00296     session=svc.openSession(isReadOnly=True,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
00297 
00298     ##############################################################
00299     # check run/ls list
00300     ##############################################################
00301     irunlsdict={}
00302     rruns=[]
00303     session.transaction().start(True)
00304     filerunlist=None
00305     if options.inputfile:
00306         (irunlsdict,iresults)=parseInputFiles(options.inputfile)
00307         filerunlist=irunlsdict.keys()
00308         
00309     
00310     #if not irunlsdict: #no file
00311     #    irunlsdict=dict(zip(rruns,[None]*len(rruns)))
00312     #else:
00313     #    for selectedrun in irunlsdict.keys():#if there's further filter on the runlist,clean input dict
00314     #        if selectedrun not in rruns:
00315     #            del irunlsdict[selectedrun]
00316     
00317     ##############################################################
00318     # check datatag
00319     # #############################################################       
00320     datatagname=options.datatag
00321     if not datatagname:
00322         (datatagid,datatagname)=revisionDML.currentDataTag(session.nominalSchema())
00323     else:
00324         datatagid=revisionDML.getDataTagId(session.nominalSchema(),datatagname)
00325 
00326     dataidmap=lumiCalcAPI.runList(session.nominalSchema(),datatagid,runmin=reqrunmin,runmax=reqrunmax,fillmin=reqfillmin,fillmax=reqfillmax,startT=reqtimemin,stopT=reqtimemax,l1keyPattern=None,hltkeyPattern=None,amodetag=options.amodetag,nominalEnergy=options.beamenergy,energyFlut=options.beamfluctuation,requiretrg=reqTrg,requirehlt=reqHlt,preselectedruns=filerunlist)
00327     if not dataidmap:
00328         print '[INFO] No qualified run found, do nothing'
00329         sys.exit(14)
00330     rruns=[]
00331     #crosscheck dataid value
00332     for irun,(lid,tid,hid) in dataidmap.items():
00333         if not lid:
00334             print '[INFO] No qualified lumi data found for run, ',irun
00335         if reqTrg and not tid:
00336             print '[INFO] No qualified trg data found for run ',irun
00337             continue
00338         if reqHlt and not hid:
00339             print '[INFO] No qualified hlt data found for run ',irun
00340             continue
00341         rruns.append(irun)
00342     if not irunlsdict: #no file
00343         irunlsdict=dict(zip(rruns,[None]*len(rruns)))
00344     else:
00345         for selectedrun in irunlsdict.keys():#if there's further filter on the runlist,clean input dict
00346             if selectedrun not in rruns:
00347                 del irunlsdict[selectedrun]
00348     if not irunlsdict:
00349         print '[INFO] No qualified run found, do nothing'
00350         sys.exit(13)
00351 
00352     ###############################################################
00353     # check normtag and get norm values if required
00354     ###############################################################
00355     normname='NONE'
00356     normid=0
00357     normvalueDict={}
00358     if not options.withoutNorm:
00359         normname=options.normtag
00360         if not normname:
00361             normmap=normDML.normIdByType(session.nominalSchema(),lumitype='HF',defaultonly=True)
00362             if len(normmap):
00363                 normname=normmap.keys()[0]
00364                 normid=normmap[normname]
00365         else:
00366             normid=normDML.normIdByName(session.nominalSchema(),normname)
00367         if not normid:
00368             raise RuntimeError('[ERROR] cannot resolve norm/correction')
00369             sys.exit(12)
00370         normvalueDict=normDML.normValueById(session.nominalSchema(),normid) #{since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]}
00371     session.transaction().commit()
00372     lumiReport.toScreenHeader(thiscmmd,datatagname,normname,workingversion,updateversion,'HF',toFile=options.headerfile)
00373 
00374                 
00375     ##################
00376     # ls level       #
00377     ##################
00378     session.transaction().start(True)
00379     
00380     GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict,dataidmap,lumitype='HF')
00381     if options.action == 'delivered':
00382         result=lumiCalcAPI.deliveredLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF')
00383         lumiReport.toScreenTotDelivered(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00384     if options.action == 'overview':
00385         result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF')
00386         lumiReport.toScreenOverview(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00387     if options.action == 'lumibyls':
00388         if not options.hltpath:
00389             result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF',minbiasXsec=options.minbiasxsec)
00390             lumiReport.toScreenLumiByLS(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)            
00391         else:
00392             hltname=options.hltpath
00393             hltpat=None
00394             if hltname=='*' or hltname=='all':
00395                 hltname=None
00396             elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
00397                 hltpat=hltname
00398                 hltname=None
00399             result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=options.xingMinLum,withBeamIntensity=False,lumitype='HF')
00400             lumiReport.toScreenLSEffective(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00401     if options.action == 'recorded':#recorded actually means effective because it needs to show all the hltpaths...
00402         hltname=options.hltpath
00403         hltpat=None
00404         if hltname is not None:
00405             if hltname=='*' or hltname=='all':
00406                 hltname=None
00407             elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
00408                 hltpat=hltname
00409                 hltname=None
00410         result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=options.xingMinLum,withBeamIntensity=False,lumitype='HF')
00411         lumiReport.toScreenTotEffective(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00412     if options.action == 'lumibylsXing':
00413          result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,withBXInfo=True,bxAlgo=options.xingAlgo,xingMinLum=options.xingMinLum,withBeamIntensity=False,lumitype='HF')
00414          outfile=options.outputfile
00415          if not outfile:
00416              print '[WARNING] no output file given. lumibylsXing writes per-bunch lumi only to default file lumibylsXing.csv'
00417              outfile='lumibylsXing.csv'           
00418          lumiReport.toCSVLumiByLSXing(result,options.scalefactor,outfile,irunlsdict=irunlsdict,noWarning=noWarning)
00419     session.transaction().commit()
00420     del session
00421     del svc
/afs/cern.ch/work/a/aaltunda/public/www/CMSSW_6_2_5/src/RecoLuminosity/LumiDB/scripts/lumiCalc2.py
