specificLumi Namespace Reference

Functions
def	calculateSpecificLumi (lumi, lumierr, beam1intensity, beam1intensityerr, beam2intensity, beam2intensityerr)
def	filltofiles (allfills, runsperfill, runtimes, dirname)
	output methods#### More...
def	getFillFromDB (schema, fillnum)
def	getFillFromFile (fillnum, inputdir)
def	getSpecificLumi (schema, fillnum, inputdir, dataidmap, normmap, xingMinLum=0.0, amodetag='PROTPHYS', bxAlgo='OCC1')
def	lastcompleteFill (infile)
def	listfilldir (indir)
def	specificlumiTofile (fillnum, filldata, outdir)

Variables
	action
	allfillname
	allfillsFromDB
	amodetag
	amodetagChoices
	authpath
	bxAlgo
	choices
	connect
	cpp2sqltype
	dataidmap
	datatagid
	datatagname
	debugON
	default
	defaultonly
	description
	dest
	energyFlut
	False
	filldata
	fillmax
	fillmin
	fillnum
	fillstoprocess
	formatter_class
	help
	hltkeyPattern
	inputdir
	isReadOnly
	l1keyPattern
	lastcompletedFill
	lumitype
	MAXFILL
	maxfillnum
	MINFILL
	minfillnum
	nominalEnergy
	None
	normid
	normmap
	normname
	normvalueDict
	options
	parser
	## Main Program More...
	processedfills
	prog
	reqfillmax
	reqfillmin
	required
	requirehlt
	requiretrg
	runlist
	runmax
	runmin
	schema
	session
	startT
	stopT
	svc
	query DB for all fills and compare with allfills.txt if found newer fills, store in mem fill number reprocess anyway the last 1 fill in the dir redo specific lumi for all marked fills More...
	True
	type
	withcomment
	xingAlgoChoices
	xingMinLum

Function Documentation

def specificLumi.calculateSpecificLumi	(		lumi,
			lumierr,
			beam1intensity,
			beam1intensityerr,
			beam2intensity,
			beam2intensityerr
	)

calculate specific lumi
input: instlumi, instlumierror,beam1intensity,beam1intensityerror,beam2intensity,beam2intensityerror
output (specific lumi value,specific lumi error)

Definition at line 75 of file specificLumi.py.

References objects.autophobj.float.

Referenced by getSpecificLumi().

def calculateSpecificLumi(lumi,lumierr,beam1intensity,beam1intensityerr,beam2intensity,beam2intensityerr):
    '''
    calculate specific lumi
    input: instlumi, instlumierror,beam1intensity,beam1intensityerror,beam2intensity,beam2intensityerror
    output (specific lumi value,specific lumi error)
    '''
    specificlumi=0.0
    specificlumierr=0.0
    if beam1intensity<0: beam1intensity=0
    if beam2intensity<0: beam2intensity=0
    if beam1intensity>0.0 and  beam2intensity>0.0:
        specificlumi=float(lumi)/(float(beam1intensity)*float(beam2intensity))
        specificlumierr=specificlumi*math.sqrt(lumierr**2/lumi**2+beam1intensityerr**2/beam1intensity**2+beam2intensityerr**2/beam2intensity**2)
    return (specificlumi,specificlumierr)

def specificLumi.filltofiles	(		allfills,
			runsperfill,
			runtimes,
			dirname
	)

output methods####

write runnumber:starttime map per fill to files

Definition at line 122 of file specificLumi.py.

References edm.print(), and str.

def filltofiles(allfills,runsperfill,runtimes,dirname):
    '''
    write runnumber:starttime map per fill to files
    '''
    f=open(os.path.join(dirname,allfillname),'w')
    for fill in allfills:
        print('%d'%(fill), file=f)
    f.close()
    for fill,runs in runsperfill.items():
        filename='fill_'+str(fill)+'.txt'
        if len(runs)!=0:
            f=open(os.path.join(dirname,filename),'w')
            for run in runs:
                print('%d,%s'%(run,runtimes[run]), file=f)
            f.close()
            

def specificLumi.getFillFromDB	(		schema,
			fillnum
	)

output: {run:starttime}

Definition at line 23 of file specificLumi.py.

References cmsPerfStripChart.dict, lumiCalcAPI.fillrunMap(), list(), lumiCalcAPI.runsummary(), and ComparisonHelper.zip().

Referenced by getSpecificLumi().

def getFillFromDB(schema,fillnum):
    '''
    output: {run:starttime}
    '''
    runtimesInFill={}
    fillrundict=lumiCalcAPI.fillrunMap(schema,fillnum)
    if len(fillrundict)>0:
        runs=fillrundict.values()[0]
        runlsdict=dict(list(zip(runs,[None]*len(runs))))
        runresult=lumiCalcAPI.runsummary(schema,runlsdict)    
        for perrundata in runresult:
            runtimesInFill[perrundata[0]]=perrundata[7]
    return runtimesInFill

def specificLumi.getFillFromFile	(		fillnum,
			inputdir
	)

parse fill_xxx.txt files in the input directory for runs, starttime in the fill
input: fillnumber, input dir
output: {run:tarttime}

Definition at line 90 of file specificLumi.py.

References createfilelist.int.

def getFillFromFile(fillnum,inputdir):
    '''
    parse fill_xxx.txt files in the input directory for runs, starttime in the fill
    input: fillnumber, input dir
    output: {run:tarttime}
    '''
    runtimesInFill={}
    #look for files 'fill_num.txt' in inputdir
    for filename in os.listdir(inputdir):
        mpat=r'^fill_[0-9]{4}.txt$'
        m=re.compile(mpat)
        if m.match(filename) is None:
            continue
        filename=filename.strip()
        if filename.find('.')==-1: continue            
        basename,extension=filename.split('.')        
        if not extension or extension!='txt':
            continue
        if basename.find('_')==-1: continue
        prefix,number=basename.split('_')
        if not number : continue
        if fillnum!=int(number):continue
        f=open(os.path.join(inputdir,'fill_'+number+'.txt'),'r')
        for line in f:
            l=line.strip()
            fields=l.split(',')
            if len(fields)<2 : continue
            runtimesInFill[int(fields[0])]=fields[1]
        f.close()
    return runtimesInFill

def specificLumi.getSpecificLumi	(		schema,
			fillnum,
			inputdir,
			dataidmap,
			normmap,
			xingMinLum = 0.0,
			amodetag = 'PROTPHYS',
			bxAlgo = 'OCC1'
	)

specific lumi in 1e-30 (ub-1s-1) unit
lumidetail occlumi in 1e-27
1309_lumi_401_CMS.txt
time(in seconds since January 1,2011,00:00:00 UTC) stab(fraction of time spent in stable beams for this time bin) l(lumi in Hz/ub) dl(point-to-point error on lumi in Hz/ub) sl(specific lumi in Hz/ub) dsl(error on specific lumi)
20800119.0 1 -0.889948 0.00475996848729 0.249009 0.005583287562 -0.68359 6.24140208607 0.0 0.0 0.0 0.0 0.0 0.0 0.0383576 0.00430892097862 0.0479095 0.00430892097862 66.6447 4.41269758764 0.0 0.0 0.0
result [(time,beamstatusfrac,lumi,lumierror,speclumi,speclumierror)]

Definition at line 240 of file specificLumi.py.

References mps_setup.append, calculateSpecificLumi(), lumiCalcAPI.deliveredLumiForIds(), cmsPerfStripChart.dict, getFillFromDB(), list(), SiStripPI.max, edm.print(), lumiCalcAPI.runsummaryMap(), and ComparisonHelper.zip().

def getSpecificLumi(schema,fillnum,inputdir,dataidmap,normmap,xingMinLum=0.0,amodetag='PROTPHYS',bxAlgo='OCC1'):
    '''
    specific lumi in 1e-30 (ub-1s-1) unit
    lumidetail occlumi in 1e-27
    1309_lumi_401_CMS.txt
    time(in seconds since January 1,2011,00:00:00 UTC) stab(fraction of time spent in stable beams for this time bin) l(lumi in Hz/ub) dl(point-to-point error on lumi in Hz/ub) sl(specific lumi in Hz/ub) dsl(error on specific lumi)
    20800119.0 1 -0.889948 0.00475996848729 0.249009 0.005583287562 -0.68359 6.24140208607 0.0 0.0 0.0 0.0 0.0 0.0 0.0383576 0.00430892097862 0.0479095 0.00430892097862 66.6447 4.41269758764 0.0 0.0 0.0
    result [(time,beamstatusfrac,lumi,lumierror,speclumi,speclumierror)]
    '''
    t=lumiTime.lumiTime()
    fillbypos={}#{bxidx:[[ts,beamstatusfrac,lumi,lumierror,spec1,specerror],[]]}
    runtimesInFill=getFillFromDB(schema,fillnum)#{runnum:starttimestr}
    runlist=runtimesInFill.keys()
    if not runlist: return fillbypos
    irunlsdict=dict(list(zip(runlist,[None]*len(runlist))))
    #prirunlsdict
    GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict)
    lumidetails=lumiCalcAPI.deliveredLumiForIds(schema,irunlsdict,dataidmap,GrunsummaryData,beamstatusfilter=None,normmap=normmap,withBXInfo=True,bxAlgo=bxAlgo,xingMinLum=xingMinLum,withBeamIntensity=True,lumitype='HF')
    #
    #output: {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),calibratedlumierr(6),(bxvalues,bxerrs)(7),(bxidx,b1intensities,b2intensities)(8),fillnum(9)]}
    #
    totalstablebeamls=0
    orderedrunlist=sorted(lumidetails)
    for run in orderedrunlist:
        perrundata=lumidetails[run]
        for perlsdata in perrundata:
            beamstatus=perlsdata[3]
            if beamstatus=='STABLE BEAMS':
                totalstablebeamls+=1
    #print 'totalstablebeamls in fill ',totalstablebeamls
    if totalstablebeamls<10:#less than 10 LS in a fill has 'stable beam', it's no a good fill
        print('fill ',fillnum,' , having less than 10 stable beam lS, is not good, skip')
        return fillbypos
    lumiparam=lumiParameters.ParametersObject()
    for run in orderedrunlist:
        perrundata=lumidetails[run]
        for perlsdata in perrundata:
            beamstatusfrac=0.0
            tsdatetime=perlsdata[2]
            ts=calendar.timegm(tsdatetime.utctimetuple())
            beamstatus=perlsdata[3]
            if beamstatus=='STABLE BEAMS':
                beamstatusfrac=1.0
            (bxidxlist,bxvaluelist,bxerrolist)=perlsdata[7]
            #instbxvaluelist=[x/lumiparam.lslengthsec() for x in bxvaluelist if x]
            instbxvaluelist=[x for x in bxvaluelist if x]
            maxlumi=0.0
            if len(instbxvaluelist)!=0:
                maxlumi=max(instbxvaluelist)
            avginstlumi=0.0
            if len(instbxvaluelist)!=0:
                avginstlumi=sum(instbxvaluelist)
            (intbxidxlist,b1intensities,b2intensities)=perlsdata[8]#contains only non-zero bx
            for bxidx in bxidxlist:
                idx=bxidxlist.index(bxidx)
                instbxvalue=bxvaluelist[idx]
                bxerror=bxerrolist[idx]
                if instbxvalue<max(xingMinLum,maxlumi*0.2):
                    continue
                bintensityPos=-1
                try:
                    bintensityPos=intbxidxlist.index(bxidx)
                except ValueError:
                    pass
                if bintensityPos<=0:
                    fillbypos.setdefault(bxidx,[]).append([ts,beamstatusfrac,instbxvalue,bxerror,0.0,0.0])
                    continue
                b1intensity=b1intensities[bintensityPos]
                b2intensity=b2intensities[bintensityPos]
                speclumi=calculateSpecificLumi(instbxvalue,bxerror,b1intensity,0.0,b2intensity,0.0)
                fillbypos.setdefault(bxidx,[]).append([ts,beamstatusfrac,instbxvalue,bxerror,speclumi[0],speclumi[1]])
    return fillbypos

def specificLumi.lastcompleteFill

(

infile

)

parse infile to find LASTCOMPLETEFILL
input: input file name
output: last completed fill number

Definition at line 58 of file specificLumi.py.

References createfilelist.int.

def lastcompleteFill(infile):
    '''
    parse infile to find LASTCOMPLETEFILL
    input: input file name
    output: last completed fill number
    '''
    lastfill=None
    hlinepat=r'(LASTCOMPLETEFILL )([0-9]{4})'
    h=re.compile(hlinepat)
    dqmfile=open(infile,'r')
    for line in dqmfile:
        result=h.match(line)
        if result:
            lastfill=result.group(2)
            break
    return int(lastfill)

def specificLumi.listfilldir

(

indir

)

list all fills contained in the given dir
input: indir
output: [fill]

Definition at line 37 of file specificLumi.py.

References createfilelist.int.

def listfilldir(indir):
    '''
    list all fills contained in the given dir
    input: indir
    output: [fill]
    '''
    fillnamepat=r'^[0-9]{4}$'
    p=re.compile(fillnamepat)
    processedfills=[]
    dirList=os.listdir(indir)
    for fname in dirList:
        if p.match(fname) and os.path.isdir(os.path.join(indir,fname)):#found fill dir
            allfs=os.listdir(os.path.join(indir,fname))
            for myfile in allfs:
                sumfilenamepat=r'^[0-9]{4}_bxsum_CMS.txt$'
                s=re.compile(sumfilenamepat)
                if s.match(myfile):
                    #only if fill_summary_CMS.txt file exists
                    processedfills.append(int(fname))
    return processedfills

def specificLumi.specificlumiTofile	(		fillnum,
			filldata,
			outdir
	)

Definition at line 138 of file specificLumi.py.

References mps_setup.append, objects.autophobj.float, createfilelist.int, genParticles_cff.map, SiStripPI.max, edm.print(), str, and CommonUtil.transposed().

def specificlumiTofile(fillnum,filldata,outdir):
    #
    #input : fillnum
    #        filldata: {bxidx:[[lstime,beamstatusfrac,lumivalue,lumierror,speclumi,speclumierr]],[]}
    #sorted by bxidx, sorted by lstime inside list
    #check outdir/fillnum subdir exists; if not, create it; else outdir=outdir/fillnum
    #
    if not filldata:
        print('empty input data, do nothing for fill ',fillnum)
        return
    timedict={}#{lstime:[[stablebeamfrac,lumi,lumierr,speclumi,speclumierr]]}
    filloutdir=os.path.join(outdir,str(fillnum))
    if not os.path.exists(filloutdir):
        os.mkdir(filloutdir)
    for cmsbxidx,perbxdata in filldata.items():
        lhcbucket=0
        if cmsbxidx!=0:
            lhcbucket=(cmsbxidx-1)*10+1
        a=sorted(perbxdata,key=lambda x:x[0])
        filename=str(fillnum)+'_lumi_'+str(lhcbucket)+'_CMS.txt'
        linedata=[]
        for perlsdata in a:
            ts=int(perlsdata[0])
            beamstatusfrac=perlsdata[1]
            lumi=perlsdata[2]
            lumierror=perlsdata[3]
            #beam1intensity=perlsdata[4]
            #beam2intensity=perlsdata[5]
            speclumi=perlsdata[4]
            speclumierror= perlsdata[5]
            if lumi>0:
                linedata.append([ts,beamstatusfrac,lumi,lumierror,speclumi,speclumierror])
            if ts not in timedict:
                timedict[ts]=[]
            timedict[ts].append([beamstatusfrac,lumi,lumierror,speclumi,speclumierror])
        if len(linedata)>10:#at least 10 good ls
            f=open(os.path.join(filloutdir,filename),'w')
            for line in linedata:
                print('%d\t%e\t%e\t%e\t%e\t%e'%(line[0],line[1],line[2],line[3],line[4],line[5]), file=f)
            f.close()
    #print 'writing avg file'
    summaryfilename=str(fillnum)+'_lumi_CMS.txt'
    f=None
    lstimes=sorted(timedict.keys())
    fillseg=[]
    lscounter=0
    for lstime in lstimes:
        allvalues=timedict[lstime]
        transposedvalues=CommonUtil.transposed(allvalues,0.0)
        bstatfrac=transposedvalues[0][0]#beamstatus does not change with bx position
        lumivals=transposedvalues[1]
        lumitot=sum(lumivals)
        if bstatfrac==1.0 :
            fillseg.append([lstime,lumitot])
        lumierrs=transposedvalues[2]
        lumierrortot=math.sqrt(sum(map(lambda x:x**2,lumierrs)))
        specificvals=transposedvalues[3]
        specificavg=sum(specificvals)/float(len(specificvals))#avg spec lumi
        specificerrs=transposedvalues[4]
        specifictoterr=math.sqrt(sum(map(lambda x:x**2,specificerrs)))
        specificerravg=specifictoterr/float(len(specificvals))
        if lscounter==0:
            f=open(os.path.join(filloutdir,summaryfilename),'w')
        lscounter+=1
        print('%d\t%e\t%e\t%e\t%e\t%e'%(lstime,bstatfrac,lumitot,lumierrortot,specificavg,specificerravg), file=f)
    if f is not None:
        f.close()
    #print 'writing summary file'
    fillsummaryfilename=str(fillnum)+'_bxsum_CMS.txt'
    f=open(os.path.join(filloutdir,fillsummaryfilename),'w')    
    if len(fillseg)==0:
        print('%s'%('#no stable beams'), file=f)
        f.close()
        return
    previoustime=fillseg[0][0]
    boundarytime=fillseg[0][0]
    #print 'boundary time ',boundarytime
    summaryls={}
    summaryls[boundarytime]=[]
    for [lstime,lumitot] in fillseg:#fillseg is everything with stable beam flag
        if lstime-previoustime>50.0:
            boundarytime=lstime
            #print 'found new boundary ',boundarytime
            summaryls[boundarytime]=[]
     #   print 'appending ',boundarytime,lstime,lumitot
        summaryls[boundarytime].append([lstime,lumitot])
        previoustime=lstime
    #print summaryls
   
    summarylstimes=summaryls.keys()
    summarylstimes.sort()
    lumip=lumiParameters.ParametersObject()
    for bts in summarylstimes:
        startts=bts
        tsdatainseg=summaryls[bts]
        #print 'tsdatainseg ',tsdatainseg
        stopts=tsdatainseg[-1][0]
        plu=max(CommonUtil.transposed(tsdatainseg,0.0)[1])
        lui=sum(CommonUtil.transposed(tsdatainseg,0.0)[1])*lumip.lslengthsec()
        print('%d\t%d\t%e\t%e'%(startts,stopts,plu,lui), file=f)
    f.close()

Variable Documentation

specificLumi.action

Definition at line 328 of file specificLumi.py.

specificLumi.allfillname

Definition at line 21 of file specificLumi.py.

specificLumi.allfillsFromDB

Definition at line 417 of file specificLumi.py.

specificLumi.amodetag

Definition at line 417 of file specificLumi.py.

specificLumi.amodetagChoices

Definition at line 324 of file specificLumi.py.

specificLumi.authpath

Definition at line 406 of file specificLumi.py.

specificLumi.bxAlgo

Definition at line 479 of file specificLumi.py.

specificLumi.choices

Definition at line 365 of file specificLumi.py.

specificLumi.connect

Definition at line 406 of file specificLumi.py.

specificLumi.cpp2sqltype

Definition at line 407 of file specificLumi.py.

specificLumi.dataidmap

Definition at line 452 of file specificLumi.py.

specificLumi.datatagid

Definition at line 451 of file specificLumi.py.

specificLumi.datatagname

Definition at line 449 of file specificLumi.py.

specificLumi.debugON

Definition at line 406 of file specificLumi.py.

specificLumi.default

Definition at line 331 of file specificLumi.py.

specificLumi.defaultonly

Definition at line 466 of file specificLumi.py.

specificLumi.description

Definition at line 323 of file specificLumi.py.

specificLumi.dest

Definition at line 327 of file specificLumi.py.

specificLumi.energyFlut

Definition at line 447 of file specificLumi.py.

specificLumi.False

Definition at line 447 of file specificLumi.py.

specificLumi.filldata

Definition at line 440 of file specificLumi.py.

specificLumi.fillmax

Definition at line 417 of file specificLumi.py.

specificLumi.fillmin

Definition at line 417 of file specificLumi.py.

specificLumi.fillnum

Definition at line 447 of file specificLumi.py.

specificLumi.fillstoprocess

Definition at line 409 of file specificLumi.py.

specificLumi.formatter_class

Definition at line 323 of file specificLumi.py.

specificLumi.help

Definition at line 330 of file specificLumi.py.

specificLumi.hltkeyPattern

Definition at line 447 of file specificLumi.py.

specificLumi.inputdir

Definition at line 479 of file specificLumi.py.

Referenced by EfficiencyPlotter.dqmEndJob().

specificLumi.isReadOnly

Definition at line 407 of file specificLumi.py.

specificLumi.l1keyPattern

Definition at line 447 of file specificLumi.py.

specificLumi.lastcompletedFill

Definition at line 419 of file specificLumi.py.

specificLumi.lumitype

Definition at line 466 of file specificLumi.py.

specificLumi.MAXFILL

Definition at line 20 of file specificLumi.py.

specificLumi.maxfillnum

Definition at line 410 of file specificLumi.py.

specificLumi.MINFILL

Definition at line 19 of file specificLumi.py.

specificLumi.minfillnum

Definition at line 411 of file specificLumi.py.

specificLumi.nominalEnergy

Definition at line 447 of file specificLumi.py.

specificLumi.None

Definition at line 447 of file specificLumi.py.

specificLumi.normid

Definition at line 461 of file specificLumi.py.

specificLumi.normmap

Definition at line 466 of file specificLumi.py.

specificLumi.normname

Definition at line 460 of file specificLumi.py.

specificLumi.normvalueDict

Definition at line 462 of file specificLumi.py.

specificLumi.options

Definition at line 397 of file specificLumi.py.

specificLumi.parser

## Main Program

Definition at line 323 of file specificLumi.py.

specificLumi.processedfills

Definition at line 418 of file specificLumi.py.

specificLumi.prog

Definition at line 323 of file specificLumi.py.

specificLumi.reqfillmax

Definition at line 445 of file specificLumi.py.

specificLumi.reqfillmin

Definition at line 444 of file specificLumi.py.

specificLumi.required

Definition at line 329 of file specificLumi.py.

specificLumi.requirehlt

Definition at line 447 of file specificLumi.py.

specificLumi.requiretrg

Definition at line 447 of file specificLumi.py.

specificLumi.runlist

Definition at line 447 of file specificLumi.py.

specificLumi.runmax

Definition at line 447 of file specificLumi.py.

specificLumi.runmin

Definition at line 447 of file specificLumi.py.

specificLumi.schema

Definition at line 416 of file specificLumi.py.

specificLumi.session

Definition at line 407 of file specificLumi.py.

specificLumi.startT

Definition at line 447 of file specificLumi.py.

specificLumi.stopT

Definition at line 447 of file specificLumi.py.

specificLumi.svc

query DB for all fills and compare with allfills.txt if found newer fills, store in mem fill number reprocess anyway the last 1 fill in the dir redo specific lumi for all marked fills

Definition at line 406 of file specificLumi.py.

specificLumi.True

Definition at line 407 of file specificLumi.py.

specificLumi.type

Definition at line 351 of file specificLumi.py.

specificLumi.withcomment

Definition at line 452 of file specificLumi.py.

specificLumi.xingAlgoChoices

Definition at line 325 of file specificLumi.py.

specificLumi.xingMinLum

Definition at line 479 of file specificLumi.py.