Reference Manual

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 ########################################################################
00004 # Command to calculate luminosity from HF measurement stored in lumiDB #
00005 #                                                                      #
00006 # Author:      Zhen Xie                                                #
00007 ########################################################################
00008 
00009 import os,sys,time
00010 from RecoLuminosity.LumiDB import sessionManager,lumiTime,inputFilesetParser,csvSelectionParser,selectionParser,csvReporter,argparse,CommonUtil,lumiCalcAPI,revisionDML,normDML,lumiReport,lumiCorrections,RegexValidator
00011 
00012 def parseInputFiles(inputfilename):
00013     '''
00014     output ({run:[cmsls,cmsls,...]},[[resultlines]])
00015     '''
00016     selectedrunlsInDB={}
00017     resultlines=[]
00018     p=inputFilesetParser.inputFilesetParser(inputfilename)
00019     runlsbyfile=p.runsandls()
00020     selectedProcessedRuns=p.selectedRunsWithresult()
00021     selectedNonProcessedRuns=p.selectedRunsWithoutresult()
00022     resultlines=p.resultlines()
00023     for runinfile in selectedNonProcessedRuns:
00024         selectedrunlsInDB[runinfile]=runlsbyfile[runinfile]
00025     return (selectedrunlsInDB,resultlines)
00026 
00027 ##############################
00028 ## ######################## ##
00029 ## ## ################## ## ##
00030 ## ## ## Main Program ## ## ##
00031 ## ## ################## ## ##
00032 ## ######################## ##
00033 ##############################
00034 
00035 if __name__ == '__main__':
00036     parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description = "Lumi Calculation Based on Pixel",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
00037     allowedActions = ['overview', 'recorded', 'lumibyls']
00038     #
00039     # parse arguments
00040     #  
00041 
00042     # basic arguments
00043     #
00044     parser.add_argument('action',choices=allowedActions,
00045                         help='command actions')
00046     parser.add_argument('-c',dest='connect',action='store',
00047                         required=False,
00048                         help='connect string to lumiDB,optional',
00049                         default='frontier://LumiCalc/CMS_LUMI_PROD')
00050     parser.add_argument('-P',dest='authpath',action='store',
00051                         required=False,
00052                         help='path to authentication file (optional)')
00053     parser.add_argument('-r',dest='runnumber',action='store',
00054                         type=int,
00055                         required=False,
00056                         help='run number (optional)')
00057     parser.add_argument('-o',dest='outputfile',action='store',
00058                         required=False,
00059                         help='output to csv file (optional)')
00060     #################################################
00061     #arg to select exact run and ls
00062     #################################################
00063     parser.add_argument('-i',dest='inputfile',action='store',
00064                         required=False,
00065                         help='lumi range selection file (optional)')
00066     #################################################
00067     #arg to select exact hltpath or pattern
00068     #################################################
00069     parser.add_argument('--hltpath',dest='hltpath',action='store',
00070                         default=None,required=False,
00071                         help='specific hltpath or hltpath pattern to calculate the effectived luminosity (optional)')
00072     #################################################
00073     #versions control
00074     #################################################
00075     parser.add_argument('--normtag',dest='normtag',action='store',
00076                         required=False,
00077                         help='version of lumi norm/correction')
00078     parser.add_argument('--datatag',dest='datatag',action='store',
00079                         required=False,
00080                         help='version of lumi/trg/hlt data')
00081     ###############################################
00082     # run filters
00083     ###############################################
00084     parser.add_argument('-f','--fill',dest='fillnum',action='store',
00085                         default=None,required=False,
00086                         help='fill number (optional) ')
00087     
00088     parser.add_argument('--begin',dest='begin',action='store',
00089                         default=None,
00090                         required=False,
00091                         type=RegexValidator.RegexValidator("^\d\d/\d\d/\d\d \d\d:\d\d:\d\d$|^\d{6}$|^\d{4}$","wrong format"),
00092                         help='min run start time (mm/dd/yy hh:mm:ss),min fill or min run'
00093                         ) 
00094     parser.add_argument('--end',dest='end',action='store',
00095                         required=False,
00096                         type=RegexValidator.RegexValidator("^\d\d/\d\d/\d\d \d\d:\d\d:\d\d$|^\d{6}$|^\d{4}$","wrong format"),
00097                         help='max run start time (mm/dd/yy hh:mm:ss),max fill or max run'
00098                         )
00099     parser.add_argument('--minBiasXsec',dest='minbiasxsec',action='store',
00100                         default=69300.0,
00101                         type=float,
00102                         required=False,
00103                         help='minbias cross-section in ub'
00104                         )
00105     #############################################
00106     #global scale factor
00107     #############################################       
00108     parser.add_argument('-n',dest='scalefactor',action='store',
00109                         type=float,
00110                         default=1.0,
00111                         required=False,
00112                         help='user defined global scaling factor on displayed lumi values,optional')
00113     #################################################
00114     #command configuration 
00115     #################################################
00116     parser.add_argument('--siteconfpath',dest='siteconfpath',action='store',
00117                         default=None,
00118                         required=False,
00119                         help='specific path to site-local-config.xml file, optional. If path undefined, fallback to cern proxy&server')
00120     
00121     parser.add_argument('--headerfile',dest='headerfile',action='store',
00122                         default=None,
00123                         required=False,
00124                         help='write command header output to specified file'
00125                        )
00126     
00127     #################################################
00128     #switches
00129     #################################################
00130     parser.add_argument('--without-correction',
00131                         dest='withoutNorm',
00132                         action='store_true',
00133                         help='without afterglow correction'
00134                         )
00135     parser.add_argument('--without-checkforupdate',
00136                         dest='withoutCheckforupdate',
00137                         action='store_true',
00138                         help='without check for update'
00139                         )         
00140     #parser.add_argument('--verbose',dest='verbose',
00141     #                    action='store_true',
00142     #                    help='verbose mode for printing' )
00143     parser.add_argument('--nowarning',
00144                         dest='nowarning',
00145                         action='store_true',
00146                         help='suppress bad for lumi warnings' )
00147     parser.add_argument('--debug',dest='debug',
00148                         action='store_true',
00149                         help='debug')
00150     
00151     options=parser.parse_args()
00152     if not options.runnumber and not options.inputfile and not options.fillnum and not options.begin:
00153         raise RuntimeError('at least one run selection argument in [-r,-f,-i,--begin] is required')
00154     #
00155     # check working environment
00156     #
00157     reqrunmin=None
00158     reqfillmin=None
00159     reqtimemin=None
00160     reqrunmax=None
00161     reqfillmax=None
00162     reqtimemax=None
00163     timeFilter=[None,None]
00164     noWarning=options.nowarning
00165     iresults=[]
00166     reqTrg=False
00167     reqHlt=False
00168     if options.action=='overview' or options.action=='lumibyls':
00169         reqTrg=True
00170     if options.action=='recorded':
00171         reqTrg=True
00172         reqHlt=True
00173     if options.runnumber:
00174         reqrunmax=options.runnumber
00175         reqrunmin=options.runnumber
00176     if options.fillnum:
00177         reqfillmin=options.fillnum
00178         reqfillmax=options.fillnum
00179 
00180     if options.begin:
00181         (runbeg,fillbeg,timebeg)=CommonUtil.parseTime(options.begin)
00182         if runbeg: #there's --begin runnum #priority run,fill,time
00183             if not reqrunmin:# there's no -r, then take this
00184                 reqrunmin=runbeg
00185         elif fillbeg:
00186             if not reqfillmin:
00187                 reqfillmin=fillbeg
00188         elif timebeg:
00189             reqtimemin=timebeg
00190         if reqtimemin:
00191             lute=lumiTime.lumiTime()
00192             reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S')
00193             timeFilter[0]=reqtimeminT
00194     if options.end:
00195         (runend,fillend,timeend)=CommonUtil.parseTime(options.end)
00196         if runend:
00197             if not reqrunmax:#priority run,fill,time
00198                 reqrunmax=runend
00199         elif fillend:
00200             if not reqfillmax:
00201                 reqfillmax=fillend
00202         elif timeend:
00203             reqtimemax=timeend
00204         if reqtimemax:
00205             lute=lumiTime.lumiTime()
00206             reqtimemaxT=lute.StrToDatetime(reqtimemax,customfm='%m/%d/%y %H:%M:%S')
00207             timeFilter[1]=reqtimemaxT
00208     if options.inputfile and (reqtimemax or reqtimemin):
00209         #if use time and file filter together, there's no point of warning about missing LS,switch off
00210         noWarning=True
00211         
00212     ##############################################################
00213     # check working environment
00214     ##############################################################            
00215     workingversion='UNKNOWN'
00216     updateversion='NONE'
00217     thiscmmd=sys.argv[0]
00218     if not options.withoutCheckforupdate:
00219         from RecoLuminosity.LumiDB import checkforupdate
00220         cmsswWorkingBase=os.environ['CMSSW_BASE']
00221         if not cmsswWorkingBase:
00222             print 'Please check out RecoLuminosity/LumiDB from CVS,scram b,cmsenv'
00223             sys.exit(11)
00224         c=checkforupdate.checkforupdate('pixeltagstatus.txt')
00225         workingversion=c.runningVersion(cmsswWorkingBase,'pixelLumiCalc.py',isverbose=False)
00226         if workingversion:
00227             updateversionList=c.checkforupdate(workingversion,isverbose=False)
00228             if updateversionList:
00229                 updateversion=updateversionList[-1][0]
00230     #
00231     # check DB environment
00232     #   
00233     if options.authpath:
00234         os.environ['CORAL_AUTH_PATH'] = options.authpath
00235     #############################################################
00236     #pre-check option compatibility
00237     #############################################################
00238     if options.action=='recorded':
00239         if not options.hltpath:
00240             raise RuntimeError('argument --hltpath pathname is required for recorded action')                
00241     svc=sessionManager.sessionManager(options.connect,
00242                                       authpath=options.authpath,
00243                                       siteconfpath=options.siteconfpath,
00244                                       debugON=options.debug)
00245 
00246     session=svc.openSession(isReadOnly=True,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
00247     ##############################################################
00248     # check run/ls list
00249     ##############################################################
00250     irunlsdict={}
00251     rruns=[]    
00252     session.transaction().start(True)
00253     filerunlist=None
00254     if options.inputfile:
00255         (irunlsdict,iresults)=parseInputFiles(options.inputfile)
00256         filerunlist=irunlsdict.keys()
00257     ##############################################################
00258     # check datatag
00259     # #############################################################
00260     datatagname=options.datatag
00261     if not datatagname:
00262         (datatagid,datatagname)=revisionDML.currentDataTag(session.nominalSchema())
00263     else:
00264         datatagid=revisionDML.getDataTagId(session.nominalSchema(),datatagname)
00265 
00266     dataidmap=lumiCalcAPI.runList(session.nominalSchema(),datatagid,runmin=reqrunmin,runmax=reqrunmax,fillmin=reqfillmin,fillmax=reqfillmax,startT=reqtimemin,stopT=reqtimemax,l1keyPattern=None,hltkeyPattern=None,amodetag=None,nominalEnergy=None,energyFlut=None,requiretrg=reqTrg,requirehlt=reqHlt,preselectedruns=filerunlist,lumitype='PIXEL')
00267     if not dataidmap:
00268         print '[INFO] No qualified run found, do nothing'
00269         sys.exit(14)
00270     rruns=[]
00271     for irun,(lid,tid,hid) in dataidmap.items():
00272         if not lid:
00273             print '[INFO] No qualified lumi data found for run, ',irun
00274         if reqTrg and not tid:
00275             print '[INFO] No qualified trg data found for run ',irun
00276             continue
00277         if reqHlt and not hid:
00278             print '[INFO] No qualified hlt data found for run ',irun
00279             continue
00280         rruns.append(irun)
00281     if not irunlsdict: #no file
00282         irunlsdict=dict(zip(rruns,[None]*len(rruns)))
00283     else:
00284         for selectedrun in irunlsdict.keys():#if there's further filter on the runlist,clean input dict
00285             if selectedrun not in rruns:
00286                 del irunlsdict[selectedrun]
00287     if not irunlsdict:
00288         print '[INFO] No qualified run found, do nothing'
00289         sys.exit(13)
00290     ###############################################################
00291     # check normtag and get norm values if required
00292     ###############################################################
00293     normname='NONE'
00294     normid=0
00295     normvalueDict={}
00296     if not options.withoutNorm:
00297         normname=options.normtag
00298         if not normname:
00299             normmap=normDML.normIdByType(session.nominalSchema(),lumitype='PIXEL',defaultonly=True)
00300             if len(normmap):
00301                 normname=normmap.keys()[0]
00302                 normid=normmap[normname]
00303         else:
00304             normid=normDML.normIdByName(session.nominalSchema(),normname)
00305         if not normid:
00306             raise RuntimeError('[ERROR] cannot resolve norm/correction')
00307             sys.exit(12)
00308         normvalueDict=normDML.normValueById(session.nominalSchema(),normid) #{since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]}
00309     session.transaction().commit()
00310     lumiReport.toScreenHeader(thiscmmd,datatagname,normname,workingversion,updateversion,'PIXEL',toFile=options.headerfile)
00311 
00312     ##################
00313     # ls level       #
00314     ##################
00315     session.transaction().start(True)
00316     GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict,dataidmap,lumitype='PIXEL')
00317     if options.action == 'overview':
00318        result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=None,timeFilter=timeFilter,normmap=normvalueDict,lumitype='PIXEL')
00319        lumiReport.toScreenOverview(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00320     if options.action == 'lumibyls':
00321        if not options.hltpath:
00322            result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=None,timeFilter=timeFilter,normmap=normvalueDict,lumitype='PIXEL',minbiasXsec=options.minbiasxsec)
00323            lumiReport.toScreenLumiByLS(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00324        else:
00325            hltname=options.hltpath
00326            hltpat=None
00327            if hltname=='*' or hltname=='all':
00328                hltname=None
00329            elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
00330               hltpat=hltname
00331               hltname=None
00332            result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=None,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,withBeamIntensity=False,lumitype='PIXEL')
00333            lumiReport.toScreenLSEffective(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile,)
00334     if options.action == 'recorded':#recorded actually means effective because it needs to show all the hltpaths...
00335        hltname=options.hltpath
00336        hltpat=None
00337        if hltname is not None:
00338           if hltname=='*' or hltname=='all':
00339               hltname=None
00340           elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
00341               hltpat=hltname
00342               hltname=None
00343        result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=None,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,withBeamIntensity=False,lumitype='PIXEL')
00344        lumiReport.toScreenTotEffective(result,iresults,options.scalefactor,irunlsdict=irunlsdict,noWarning=noWarning,toFile=options.outputfile)
00345     session.transaction().commit()
00346     del session
00347     del svc
/data/refman/pasoursint/CMSSW_5_3_9_patch3/src/RecoLuminosity/LumiDB/scripts/pixelLumiCalc.py
