9 from __future__
import print_function
10 import os,os.path,sys,datetime,time,csv,re
11 from RecoLuminosity.LumiDB
import argparse,lumiTime,matplotRender,sessionManager,lumiParameters,RegexValidator,revisionDML,lumiCalcAPI,normDML
13 from matplotlib.figure
import Figure
17 output ({run:[cmsls,cmsls,...]},[[resultlines]]) 22 runlsbyfile=p.runsandls()
23 selectedProcessedRuns=p.selectedRunsWithresult()
24 selectedNonProcessedRuns=p.selectedRunsWithoutresult()
25 resultlines=p.resultlines()
26 for runinfile
in selectedNonProcessedRuns:
27 selectedrunlsInDB[runinfile]=runlsbyfile[runinfile]
28 return (selectedrunlsInDB,resultlines)
37 if __name__==
'__main__':
38 referenceLabel=
'Delivered' 39 labels=[
'Delivered',
'Recorded']
40 allowedActions = [
'run',
'time',
'fill',
'perday',
'instpeakperday']
41 beamChoices=[
'PROTPHYS',
'IONPHYS',
'PAPHYS']
42 allowedscales=[
'linear',
'log',
'both']
43 beamModeChoices = [
"stable",
"quiet",
"either"]
44 amodetagChoices = [
"PROTPHYS",
"IONPHYS",
'PAPHYS' ]
45 actiontofilebasemap={
'time':
'lumivstime',
'run':
'lumivsrun',
'fill':
'lumivsfill',
'perday':
'lumiperday',
'instpeakperday':
'lumipeak'}
49 currentdir=os.getcwd()
50 rcparamfile=
'.lumiplotrc' 51 mplrcdir=matplotlib.get_configdir()
52 mpllumiconfig=os.path.join(mplrcdir,rcparamfile)
53 locallumiconfig=os.path.join(currentdir,rcparamfile)
54 figureparams={
'sizex':7.5,
'sizey':5.7,
'dpi':135}
55 if os.path.exists(locallumiconfig):
58 config=ConfigParser.RawConfigParser()
59 config.read(locallumiconfig)
60 figureparams[
'sizex']=config.getfloat(
'sizex')
61 figureparams[
'sizey']=config.getfloat(
'sizey')
62 figureparams[
'dpi']=config.getint(
'dpi')
63 except ConfigParser.NoOptionError:
65 elif os.path.exists(mpllumiconfig):
68 config=ConfigParser.RawConfigParser()
69 config.read(mpllumiconfig)
70 figureparams[
'sizex']=config.getfloat(
'sizex')
71 figureparams[
'sizey']=config.getfloat(
'sizey')
72 figureparams[
'dpi']=config.getint(
'dpi')
73 except ConfigParser.NoOptionError:
81 parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
82 description=
"Plot luminosity as function of the time variable of choice",
83 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
84 parser.add_argument(
'-c',
87 help=
'connect string to lumiDB',
88 default=
'frontier://LumiCalc/CMS_LUMI_PROD')
89 parser.add_argument(
'-P',
92 help=
'path to authentication file')
93 parser.add_argument(
'--datatag',
98 parser.add_argument(
'--normtag',
106 parser.add_argument(
'-i',
109 help=
'run/ls selection file')
113 parser.add_argument(
'--hltpath',
118 help=
'specific hltpath or hltpath pattern to calculate the effectived luminosity (optional)')
122 parser.add_argument(
'-b',
125 help=
'selection criteria beam status')
129 parser.add_argument(
'-f',
'--fill',
133 help=
'specific fill',
135 parser.add_argument(
'--amodetag',
138 help=
'accelerator mode')
139 parser.add_argument(
'--beamenergy',
144 help=
'beamenergy (in GeV) selection criteria')
145 parser.add_argument(
'--beamfluctuation',
146 dest=
'beamfluctuation',
150 help=
'allowed fraction of beamenergy to fluctuate')
151 parser.add_argument(
'--begin',
154 default=
'03/01/10 00:00:00',
156 help=
'min run start time,mm/dd/yy hh:mm:ss')
157 parser.add_argument(
'--end',
161 help=
'max run start time,mm/dd/yy hh:mm:ss')
165 parser.add_argument(
'--siteconfpath',
168 help=
'specific path to site-local-config.xml file, default to $CMS_PATH/SITECONF/local/JobConfig, if path undefined, fallback to cern proxy&server')
173 parser.add_argument(
'--inplotdata',
176 help=
'existing base plot(s) in text format')
177 parser.add_argument(
'--outplotdata',
180 help=
'output plot. By default, a text dump of the plot is produced.')
182 parser.add_argument(
'--with-annotation',
183 dest=
'withannotation',
185 help=
'annotate boundary run numbers')
186 parser.add_argument(
'--interactive',
189 help=
'graphical mode to draw plot in a QT pannel.')
190 parser.add_argument(
'--without-textoutput',
191 dest=
'withoutTextoutput',
193 help=
'not to write out text output file')
194 parser.add_argument(
'--without-png',
197 help=
'do not produce')
198 parser.add_argument(
'--yscale',
202 help=
'y_scale[linear,log,both]')
203 parser.add_argument(
'--lastpointfromdb',
204 dest=
'lastpointfromdb',
206 help=
'the last point in the plot is always recaculated from current values in LumiDB')
209 parser.add_argument(
'--without-correction',
212 help=
'without any correction/calibration')
215 parser.add_argument(
'--verbose',
218 help=
'verbose mode, print result also to screen')
219 parser.add_argument(
'--debug',
223 parser.add_argument(
'action',
224 choices=allowedActions,
225 help=
'type of plots')
226 options=parser.parse_args()
227 if options.yscale==
'both' and options.interactive:
228 print(
'--interactive mode can draw either yscale log or linear at a time')
230 outplotfilename = options.outplot
231 if not outplotfilename:
232 outplotfilename=actiontofilebasemap[options.action]
233 outtextfilename = outplotfilename+
'.csv' 234 if options.withoutTextoutput:
238 authpath=options.authpath,
239 siteconfpath=options.siteconfpath,
240 debugON=options.debug)
241 session=svc.openSession(isReadOnly=
True,cpp2sqltype=[(
'unsigned int',
'NUMBER(10)'),(
'unsigned long long',
'NUMBER(20)')])
242 lslength=lumip.lslengthsec()
243 begtime=options.begintime
244 endtime=options.endtime
247 endtime=lut.DatetimeToStr(datetime.datetime.utcnow(),customfm=
'%m/%d/%y %H:%M:%S')
249 if options.beamstatus==
'stable':
250 pbeammode=
'STABLE BEAMS' 256 inplot=options.inplot
258 inplots=inplot.split(
'+')
261 reader=csv.reader(f,delimiter=',')
263 if '#' in row[0]:
continue 264 resultlines.append(row)
281 session.transaction().
start(
True)
282 if options.action==
'perday' or options.action==
'instpeakperday':
283 maxDrawnDay=
int(lut.StrToDatetime(begtime,customfm=
'%m/%d/%y %H:%M:%S').
date().toordinal())
285 for drawnDay
in [
int(t[0])
for t
in resultlines]:
286 if drawnDay>maxDrawnDay:
288 newFirstDay=maxDrawnDay
289 midnight=datetime.time()
290 begT=datetime.datetime.combine(datetime.date.fromordinal(newFirstDay),midnight)
291 begTStr=lut.DatetimeToStr(begT,customfm=
'%m/%d/%y %H:%M:%S')
293 runlist=
lumiCalcAPI.runList(session.nominalSchema(),options.fillnum,runmin=
None,runmax=
None,startT=begTStr,stopT=endtime,l1keyPattern=
None,hltkeyPattern=
None,amodetag=options.amodetag,nominalEnergy=options.beamenergy,energyFlut=options.beamfluctuation,requiretrg=reqTrg,requirehlt=reqHlt)
294 if options.action==
'run' or options.action==
'time':
297 lastDrawnRun=
max([
int(t[0])
for t
in resultlines])
298 newFirstRun=lastDrawnRun+1
299 if options.lastpointfromdb:
300 newFirstRun=lastDrawnRun
301 runlist=
lumiCalcAPI.runList(session.nominalSchema(),options.fillnum,runmin=newFirstRun,runmax=
None,startT=begtime,stopT=endtime,l1keyPattern=
None,hltkeyPattern=
None,amodetag=options.amodetag,nominalEnergy=options.beamenergy,energyFlut=options.beamfluctuation,requiretrg=reqTrg,requirehlt=reqHlt)
302 if options.action==
'fill':
306 lastDrawnFill=
max([
int(t[0])
for t
in resultlines])
307 lastDrawnRun=
min([
int(t[1])
for t
in resultlines
if int(t[0])==lastDrawnFill])
308 newFirstRun=lastDrawnRun+1
309 if options.lastpointfromdb:
310 newFirstRun=lastDrawnRun
311 runlist=
lumiCalcAPI.runList(session.nominalSchema(),runmin=newFirstRun,runmax=
None,startT=begtime,stopT=endtime,l1keyPattern=
None,hltkeyPattern=
None,amodetag=options.amodetag,nominalEnergy=options.beamenergy,energyFlut=options.beamfluctuation,requiretrg=reqTrg,requirehlt=reqHlt)
312 fillrunMap=
lumiCalcAPI.fillrunMap(session.nominalSchema(),runmin=newFirstRun,runmax=
None,startT=begtime,stopT=endtime)
314 if options.inputfile:
317 if options.fillnum
or options.begin
or options.end
or options.amodetag
or options.beamenergy:
318 rruns=[val
for val
in runlist
if val
in irunlsdict.keys()]
319 for selectedrun
in irunlsdict.keys():
320 if selectedrun
not in rruns:
321 del irunlsdict[selectedrun]
323 rruns=irunlsdict.keys()
327 rruns=irunlsdict.keys()
330 print(
'[INFO] runs from db: ',irunlsdict)
332 print(
'[INFO] last run in old plot: ',lastDrawnRun)
333 print(
'[INFO] first run from DB in fresh plot: ',newFirstRun)
335 print(
'[INFO] last day in old plot: ',maxDrawnDay)
336 print(
'[INFO] first day from DB in fresh plot: ',newFirstDay)
339 if len(resultlines)!=0:
340 print(
'[INFO] drawing all from old plot data')
342 print(
'[INFO] found no old nor new data, do nothing')
349 datatagname=options.datatag
362 if not options.withoutNorm:
363 normname=options.normtag
367 normname=normmap.keys()[0]
368 normid=normmap[normname]
372 raise RuntimeError(
'[ERROR] cannot resolve norm/correction')
376 fig=Figure(figsize=(figureparams[
'sizex'],figureparams[
'sizey']),dpi=figureparams[
'dpi'])
378 logfig=Figure(figsize=(figureparams[
'sizex'],figureparams[
'sizey']),dpi=figureparams[
'dpi'])
381 if not options.hltpath:
382 lumibyls=
lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,normmap=normvalueDict,lumitype=
'HF')
384 referenceLabel=
'Recorded' 385 hltname=options.hltpath
387 if hltname==
'*' or hltname==
'all':
389 elif 1
in [c
in hltname
for c
in '*?[]']:
392 lumibyls=
lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat)
393 session.transaction().commit()
398 if options.action==
'run':
399 for run
in sorted(lumibyls):
400 rundata=lumibyls[run]
401 if not options.hltpath:
403 rawdata.setdefault(
'Delivered',[]).
append((run,sum([t[5]
for t
in rundata
if t[5]])))
404 rawdata.setdefault(
'Recorded',[]).
append((run,sum([t[6]
for t
in rundata
if t[6]])))
409 rawdata.setdefault(
'Recorded',[]).
append((run,sum([t[6]
for t
in rundata
if t[6]])))
410 for perlsdata
in rundata:
412 pathnames=effdict.keys()
413 for thispath
in pathnames:
414 pathdict.setdefault(thispath,[]).
append(effdict[thispath][3])
415 for thispath
in pathdict.keys():
416 labels.append(thispath)
417 rawdata.setdefault(thispath,[]).
append((run,sum([t
for t
in pathdict[thispath]
if t])))
418 if options.yscale==
'linear':
419 m.plotSumX_Run(rawdata,resultlines,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel,labels=labels)
420 elif options.yscale==
'log':
421 m.plotSumX_Run(rawdata,resultlines,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel,labels=labels)
423 m.plotSumX_Run(rawdata,resultlines,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel,labels=labels)
424 m.plotSumX_Run(rawdata,resultlines,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel,labels=labels)
425 if options.action==
'fill':
426 for fill
in sorted(fillrunMap):
427 for run
in fillrunMap[fill]:
428 if run
not in lumibyls:
continue 429 rundata=lumibyls[run]
430 if not options.hltpath:
432 rawdata.setdefault(
'Delivered',[]).
append((fill,run,sum([t[5]
for t
in rundata
if t[5]])))
433 rawdata.setdefault(
'Recorded',[]).
append((fill,run,sum([t[6]
for t
in rundata
if t[6]])))
438 rawdata.setdefault(
'Recorded',[]).
append((fill,run,sum([t[6]
for t
in rundata
if t[6]])))
439 for perlsdata
in rundata:
441 pathnames=effdict.keys()
442 for thispath
in pathnames:
443 pathdict.setdefault(thispath,[]).
append(effdict[thispath][3])
444 for thispath
in pathdict.keys():
445 labels.append(thispath)
446 rawdata.setdefault(thispath,[]).
append((fill,run,sum([t
for t
in pathdict[thispath]
if t])))
447 if options.yscale==
'linear':
448 m.plotSumX_Fill(rawdata,resultlines,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
449 elif options.yscale==
'log':
450 m.plotSumX_Fill(rawdata,resultlines,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
452 m.plotSumX_Fill(rawdata,resultlines,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
453 m.plotSumX_Fill(rawdata,resultlines,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
454 if options.action==
'time':
455 for run
in sorted(lumibyls):
456 rundata=lumibyls[run]
457 if not options.hltpath:
459 rawdata.setdefault(
'Delivered',[]).
append((run,rundata[0][2],rundata[-1][2],sum([t[5]
for t
in rundata
if t[5]])))
460 rawdata.setdefault(
'Recorded',[]).
append((run,rundata[0][2],rundata[-1][2],sum([t[6]
for t
in rundata
if t[6]])))
465 rawdata.setdefault(
'Recorded',[]).
append((run,rundata[0][2],rundata[-1][2],sum([t[6]
for t
in rundata
if t[6]])))
466 for perlsdata
in rundata:
468 pathnames=effdict.keys()
469 for thispath
in pathnames:
470 pathdict.setdefault(thispath,[]).
append(effdict[thispath][3])
471 for thispath
in pathdict.keys():
472 labels.append(thispath)
473 rawdata.setdefault(thispath,[]).
append((run,rundata[0][2],rundata[-1][2],sum([t
for t
in pathdict[thispath]
if t])))
474 if options.yscale==
'linear':
475 m.plotSumX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
476 elif options.yscale==
'log':
477 mlog.plotSumX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
479 m.plotSumX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
480 mlog.plotSumX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
481 if options.action==
'perday':
483 for run
in sorted(lumibyls):
484 rundata=lumibyls[run]
485 for lsdata
in rundata:
490 daynumber=lsTS.date().toordinal()
491 daydict.setdefault(daynumber,[]).
append((run,lumilsnum,dellum,reclum))
492 for day
in sorted(daydict):
494 daybeg=
str(daydata[0][0])+
':'+
str(daydata[0][1])
495 dayend=
str(daydata[-1][0])+
':'+
str(daydata[-1][1])
496 daydel=sum([t[2]
for t
in daydata
if t[2]])
497 dayrec=sum([t[3]
for t
in daydata
if t[3]])
498 rawdata.setdefault(
'Delivered',[]).
append((day,daybeg,dayend,daydel))
499 rawdata.setdefault(
'Recorded',[]).
append((day,daybeg,dayend,dayrec))
501 if options.yscale==
'linear':
502 m.plotPerdayX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
503 elif options.yscale==
'log':
504 mlog.plotPerdayX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
506 m.plotPerdayX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
507 mlog.plotPerdayX_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
508 if options.action==
'instpeakperday':
510 for run
in sorted(lumibyls):
511 rundata=lumibyls[run]
512 for lsdata
in rundata:
515 instlum=lsdata[5]/lslength
516 daynumber=lsTS.date().toordinal()
517 daydict.setdefault(daynumber,[]).
append((run,lumilsnum,instlum))
518 for day
in sorted(daydict):
523 for datatp
in daydata:
524 if datatp[2]>daymax_val:
528 rawdata.setdefault(
'Delivered',[]).
append((day,daymax_run,daymax_ls,daymax_val))
529 if options.yscale==
'linear':
530 m.plotPeakPerday_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
531 elif options.yscale==
'log':
532 mlog.plotPeakPerday_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
534 m.plotPeakPerday_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'linear',referenceLabel=referenceLabel)
535 mlog.plotPeakPerday_Time(rawdata,resultlines,minTime=begtime,maxTime=endtime,textoutput=outtextfilename,yscale=
'log',referenceLabel=referenceLabel)
536 if options.action==
'inst':
537 thisfillnumber=fillrunMap.keys()[0]
542 for run,rundata
in lumibyls.items():
546 starttime=rundata[0][2]
547 stoptime=rundata[-1][2]
548 for lsdata
in rundata:
557 rawydata.setdefault(
'Delivered',[]).
append(dellumi)
558 rawydata.setdefault(
'Recorded',[]).
append(reclumi)
559 rawxdata=[run,thisfillnumber,starttime,stoptime,totlumils,totcmsls]
560 m.plotInst_RunLS(rawxdata,rawydata,textoutput=
None)
562 if options.yscale==
'linear':
563 if options.interactive:
567 if not options.withoutpng:
568 m.drawPNG(outplotfilename+
'.png')
570 elif options.yscale==
'log':
571 if options.interactive:
572 mlog.drawInteractive()
575 if not options.withoutpng:
576 mlog.drawPNG(outplotfilename+
'_log.png')
579 if options.interactive:
580 print(
'cannot draw both log and linear from interactive')
582 if not options.withoutpng:
583 m.drawPNG(outplotfilename+
'.png')
584 mlog.drawPNG(outplotfilename+
'_log.png')
def effectiveLumiForIds(schema, irunlsdict, dataidmap, runsummaryMap=None, beamstatusfilter=None, timeFilter=None, normmap=None, hltpathname=None, hltpathpattern=None, withBXInfo=False, bxAlgo=None, xingMinLum=None, withBeamIntensity=False, lumitype='HF', minbiasXsec=None)
def normIdByName(schema, normname)
def dataIdsByTagName(schema, tagname, runlist=None, withcomment=False, lumitype='HF')
def currentDataTag(schema, lumitype='HF')
S & print(S &os, JobReport::InputFile const &f)
def normIdByType(schema, lumitype='HF', defaultonly=True)
def runsummaryMap(schema, irunlsdict, dataidmap, lumitype='HF')
def parseInputFiles(inputfilename)
def lumiForIds(schema, irunlsdict, dataidmap, runsummaryMap, beamstatusfilter=None, timeFilter=None, normmap=None, withBXInfo=False, bxAlgo=None, xingMinLum=None, withBeamIntensity=False, lumitype='HF', minbiasXsec=None)
def fillrunMap(schema, fillnum=None, runmin=None, runmax=None, startT=None, stopT=None, l1keyPattern=None, hltkeyPattern=None, amodetag=None)
def normValueById(schema, normid)
def dataIdsByTagId(schema, tagid, runlist=None, withcomment=False, lumitype='HF')
def runList(schema, datatagid, runmin=None, runmax=None, fillmin=None, fillmax=None, startT=None, stopT=None, l1keyPattern=None, hltkeyPattern=None, amodetag=None, nominalEnergy=None, energyFlut=0.2, requiretrg=True, requirehlt=True, preselectedruns=None, lumitype='HF')