createIOVlist Namespace Reference

Functions
def	fillJson (runJson, listIOV, filelist, nMax, outDir)
def	getFileInfo (filename)
	Called in fillJson function in parallel. More...
def	getFileList (dataset)
	Called in paralell in the main function. More...
def	main ()
def	parser ()

Variables
	dataset
	dbs
	Find files in dataset. More...
	detail
	emptyfiles
	filelist
	File list which will contain skimmed file names. More...
	filename
	nEvents
	nevents
	sites

Function Documentation

fillJson()

def createIOVlist.fillJson	(		runJson,
			listIOV,
			filelist,
			nMax,
			outDir
	)

Definition at line 76 of file createIOVlist.py.

References PVValHelper.add(), createfilelist.int, FastTimerService_cff.range, emtf.shuffle(), jetUpdater_cfi.sort, str, and ComparisonHelper.zip().

Referenced by main().

def fillJson(runJson, listIOV, filelist, nMax, outDir):
    
    sort = lambda lower, run, upper: lower < int(run) < upper

    
    if listIOV:
        lowerBoundaries = [int(run) for run in listIOV[:-1]]
        upperBoundaries = [int(run)-1 for run in listIOV[1:]]

    else:
        lowerBoundaries = [0.]
        upperBoundaries = [1e20]
   
    
    pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
    results = [pool.apply_async(getFileInfo, (filename,)) for filename in filelist]
    output = [result.get() for result in results]

    fileInfo = [result for result in output if type(result) == list]
    notAtCern = [result for result in output if type(result) == str]

    
    with open("{}/filesNotAtCern.txt".format(outDir), "w") as filesNotCern:
        for filename in notAtCern:
            filesNotCern.write(filename)
            filesNotCern.write("\n")

    runDic = {}

    
    for (runInfo, filename) in zip(fileInfo, filelist):
        for (run, lumi, events) in runInfo:
            if events not in ["Events", "Lumi"]:
                try:
                    runDic[int(run)][int(lumi)] = (int(events), filename)

                except KeyError:
                    runDic[int(run)] = {int(lumi): (int(events), filename)}

    
    jsonAlign = [{} for index in lowerBoundaries]
    jsonVali = [{} for index in lowerBoundaries]
    eventsInTotal = [0 for index in lowerBoundaries]
    eventsInAlign = [0 for index in lowerBoundaries]
    eventsInVali = [0 for index in lowerBoundaries]

    
    if runJson:
        runJson = runJson.items()
        shuffle(runJson)
        filelist = {}

    else:
        return jsonAlign, jsonVali, set(filelist)

    
    for (run, value) in runJson:
        try:
            
            index = [sort(lower, run, upper) for (lower, upper) in zip(lowerBoundaries, upperBoundaries)].index(True)
        
            
            if int(run) in runDic:
                alignLumi = [[]]
                valiLumi = [[]]

                
                for (lumi, lumiInfo) in runDic[int(run)].iteritems():
                    eventsInTotal[index] += lumiInfo[0]

                    
                    if eventsInAlign[index] < nMax:
                        if not True in [sort(lower, lumi, upper) for lower, upper in value]:
                            if len(alignLumi[-1]) != 0:
                                alignLumi.append([])
                            continue

                        eventsInAlign[index] += lumiInfo[0]
                        filelist.setdefault(index, set()).add(lumiInfo[1])

                        if len(alignLumi[-1]) == 0:
                            alignLumi[-1] = [lumi, lumi]

                        else:
                            alignLumi[-1][1] = lumi

                    else:
                        if not True in [sort(lower, lumi, upper) for lower, upper in value]:
                            if len(valiLumi[-1]) != 0:
                                valiLumi.append([])
                            continue

                        eventsInVali[index] += lumiInfo[0]
                        if len(valiLumi[-1]) == 0:
                            valiLumi[-1] = [lumi, lumi]

                        else:
                            valiLumi[-1][1] = lumi

                alignLumi = [element for element in alignLumi if len(element) != 0]
                valiLumi = [element for element in valiLumi if len(element) != 0]

                if len(alignLumi) != 0:
                    jsonAlign[index][str(run)] = alignLumi

                if len(valiLumi) != 0:
                    jsonVali[index][str(run)] = valiLumi
                        

        except ValueError:
            
            pass

    
    
    with open("{}/eventsUsed.txt".format(outDir), "w") as eventsUsed:
        for index in range(len(eventsInTotal)):
            eventsUsed.write("Events used in Total for IOV {}: {}".format(lowerBoundaries[index], eventsInTotal[index]) + "\n")
            eventsUsed.write("Events used for Alignment for IOV {}: {}".format(lowerBoundaries[index], eventsInAlign[index]) + "\n")
            eventsUsed.write("Events used for Validation for IOV {}: {}".format(lowerBoundaries[index], eventsInVali[index]) + "\n")

    return jsonAlign, jsonVali, filelist
        

getFileInfo()

def createIOVlist.getFileInfo

(

filename

)

Called in fillJson function in parallel.

Definition at line 29 of file createIOVlist.py.

References FastTimerService_cff.range, submitPVValidationJobs.split(), and mkLumiAveragedPlots.tuple.

def getFileInfo(filename):
    print "Processing: {}".format(filename)

    
    try:
        edmFileUtilArgs = ['edmFileUtil', '-f', filename, '--eventsInLumis']
        fullRunInfo = subprocess.check_output(edmFileUtilArgs).split()[14:]
        runInfo = [tuple(fullRunInfo[index:index+3]) for index in range(0, len(fullRunInfo), 3)]

    
    except:
        print "Not at CERN {}".format(filename)
        runInfo = filename

    return runInfo

getFileList()

def createIOVlist.getFileList

(

dataset

)

Called in paralell in the main function.

Definition at line 46 of file createIOVlist.py.

def getFileList(dataset):

main()

def createIOVlist.main

(

)

Definition at line 200 of file createIOVlist.py.

References fillJson(), createfilelist.int, join(), parser(), and ComparisonHelper.zip().

def main():
    
    args = parser()

    
    os.system("mkdir -p {}".format(args.out_dir))

    
    filelist = []
    emptyfiles = []
    nEvents = []
    pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())

    with open(args.data_txt, "r") as datasets:        results = [pool.apply_async(getFileList, (dataset.replace("\n", ""),)) for dataset in datasets.readlines()]
    
    for result in results:
        files, empties, events = result.get()
        filelist.extend(files)
        emptyfiles.extend(empties)
        nEvents.append(events)

    with open("{}/emptyFiles.txt".format(args.out_dir), "w") as empty:
        for emptyFile in emptyfiles:
            empty.write(emptyFile + '\n')

    
    if args.iov_txt:
        with open(args.iov_txt) as fIOV:
            listIOV = [line.strip() for line in fIOV]

    else:
        listIOV = args.iov_txt

    
    if args.json_input:
        with open(args.json_input) as fJson:
            runJson = json.load(fJson)

    else:
        runJson = args.json_input

    
    jsonAlign, jsonVali, filelist = fillJson(runJson, listIOV, filelist, args.N_max_IOV, args.out_dir)

    
    if args.rm_bad_runs != None:
        with open(args.rm_bad_runs, "r") as badRuns:            for badRun in badRuns:
                for dic in jsonAlign:
                    dic.pop(int(badRun), None)

                for dic in jsonVali:
                    dic.pop(int(badRun), None)


    
    pyTempl = """import FWCore.ParameterSet.Config as cms
import FWCore.PythonUtilities.LumiList as LumiList

lumiSecs = cms.untracked.VLuminosityBlockRange()
goodLumiSecs = LumiList.LumiList(filename = '{json}').getCMSSWString().split(',')
readFiles = cms.untracked.vstring()
source = cms.Source("PoolSource",
                            lumisToProcess = lumiSecs,
                            fileNames = readFiles)
readFiles.extend([
    {filenames}
])
lumiSecs.extend(goodLumiSecs)
maxEvents = cms.untracked.PSet(input = cms.untracked.int32(-1))
    """

    
    if not args.iov_txt:
        with open("{}/{}.txt".format(args.out_dir, args.out_data), "w") as outData:
            for filename in filelist:
                outData.write(filename + '\n')
    
    
    if args.iov_txt and args.json_input:
        for index, (jsonContent, runNumber) in enumerate(zip(jsonAlign, [int(run) for run in listIOV[:-1]])):
            with open("{}/IOV_Align_{}.json".format(args.out_dir, runNumber), "w") as fAlignJson:
                json.dump(jsonContent, fAlignJson, sort_keys=True, indent=4, separators=(',', ': '))

        for (jsonContent, runNumber) in zip(jsonVali, [int(run) for run in listIOV[:-1]]):
            with open("{}/IOV_Vali_{}.json".format(args.out_dir, runNumber), "w") as fValiJson:
                json.dump(jsonContent, fValiJson, sort_keys=True, indent=4, separators=(',', ': '))

            with open("{}/{}_since{}_cff.py".format(args.out_dir, args.out_data, runNumber), "w") as outData:
                outData.write(pyTempl.format(json=os.path.abspath("{}/IOV_Vali_{}.json".format(args.out_dir, runNumber)), filenames=",\n".join(["'{}'".format(filename) for filename in filelist[index]])))

    if args.json_input:
        mergeJsonAlign = {}
        [mergeJsonAlign.update(jsonDic) for jsonDic in jsonAlign]

        mergeJsonVali = {}
        [mergeJsonVali.update(jsonDic) for jsonDic in jsonVali]

        with open("{}/Align.json".format(args.out_dir, runNumber), "w") as fAlignJson:
            json.dump(mergeJsonAlign, fAlignJson, sort_keys=True, indent=4, separators=(',', ': '))

        with open("{}/Vali.json".format(args.out_dir, runNumber), "w") as fValiJson:
            json.dump(mergeJsonVali, fValiJson, sort_keys=True, indent=4, separators=(',', ': '))

    if not os.path.exists("{}/eventsUsed.txt".format(args.out_dir)):
        with open("{}/eventsUsed.txt".format(args.out_dir), "w") as eventsUsed:
            eventsUsed.write("Events used for Alignment: {}".format(sum(nEvents)) + "\n")
            eventsUsed.write("Events used for Validation: {}".format(0) + "\n")

parser()

def createIOVlist.parser

(

)

Definition at line 13 of file createIOVlist.py.

References join(), FastTimerService_cff.range, and str.

Referenced by main().

def parser():
    parser = argparse.ArgumentParser(description='Create json config files for your defined IOV')
    
    parser.add_argument("--json-input", type = str, help = "Input json file", default = {})
    parser.add_argument("--data-txt", type = str, help = "Txt file with data set names", required = True)
    parser.add_argument("--N-max-IOV", type = int, help = "Maximum number of events per IOV", default = 1e20) 
    parser.add_argument("--rm-bad-runs", type = str, help = "Remove bad runs from json config")
    parser.add_argument("--iov-txt", type = str, help = "Txt file with IOV boundaries", default = [])
    parser.add_argument("--out-data", type = str, help = "Name of skimmed file with list of data file names", default = "skimmed_dataset")
    parser.add_argument("--out-dir", type = str, help = "Output dir name", default = "configs_" + "_".join([str(time.localtime()[i]) for i in range(6)]))


    return parser.parse_args()

Variable Documentation

dataset

createIOVlist.dataset

Definition at line 59 of file createIOVlist.py.

dbs

createIOVlist.dbs

Find files in dataset.

Definition at line 53 of file createIOVlist.py.

Referenced by hcaldqm::DQHarvester.beginRun(), hcaldqm::DQClient.beginRun(), NoCQTask.bookHistograms(), PedestalTask.bookHistograms(), QIE11Task.bookHistograms(), QIE10Task.bookHistograms(), HcalGPUComparisonTask.bookHistograms(), TPTask.bookHistograms(), RawTask.bookHistograms(), RecHitTask.bookHistograms(), and DigiTask.bookHistograms().

detail

createIOVlist.detail

Definition at line 59 of file createIOVlist.py.

emptyfiles

createIOVlist.emptyfiles

Definition at line 49 of file createIOVlist.py.

filelist

createIOVlist.filelist

File list which will contain skimmed file names.

Definition at line 48 of file createIOVlist.py.

Referenced by main().

filename

createIOVlist.filename

Definition at line 60 of file createIOVlist.py.

nEvents

createIOVlist.nEvents

Definition at line 50 of file createIOVlist.py.

Referenced by L1RCTSaveInput.analyze(), MuonIsolationDQM.analyze(), edm::JobReport::JobReportImpl.associateLumiSection(), ExternalLHEProducer.beginRunProduce(), fwlite::ChainEvent.ChainEvent(), HltDiff.compare(), edm::IndexIntoFile.fillRunOrLumiIndexes(), edm::IndexIntoFile::IndexIntoFileItrImpl.firstEventEntryThisLumi(), edm::IndexIntoFile::IndexIntoFileItrImpl.firstEventEntryThisRun(), MatacqProducer::PosEstimator.init(), edm::IndexIntoFile::IndexIntoFileItrSorted.initializeLumi_(), ExternalLHEProducer.makeArgs(), MuonIsolationDQM.MuonIsolationDQM(), edm::IndexIntoFile::IndexIntoFileItrNoSort.peekAheadAtEventEntry(), edm::IndexIntoFile::IndexIntoFileItrSorted.peekAheadAtEventEntry(), edm::IndexIntoFile::IndexIntoFileItrEntryOrder.peekAheadAtEventEntry(), edm.postIndexIntoFilePrintEventsInLumis(), edm.preIndexIntoFilePrintEventsInLumis(), edm::IndexIntoFile::IndexIntoFileItrNoSort.previousEventRange(), edm::IndexIntoFile::IndexIntoFileItrEntryOrder.previousEventRange(), lhef::H5Handler.readBlock(), edm::JobReport.reportFastClonedBranches(), edm::JobReport.reportLumiSection(), edm::IndexIntoFile::IndexIntoFileItrNoSort.setToLastEventInRange(), edm::IndexIntoFile::IndexIntoFileItrSorted.setToLastEventInRange(), edm::IndexIntoFile::IndexIntoFileItrEntryOrder.setToLastEventInRange(), and DTTimeEvolutionHisto.updateTimeSlot().

nevents

createIOVlist.nevents

Definition at line 61 of file createIOVlist.py.

Referenced by SiStripHitEffFromCalibTree.algoAnalyze(), DTLocalTriggerBaseTest.beginRun(), ZDCDigiStudy.dqmEndRun(), DTRunConditionVarClient.DTRunConditionVarClient(), SiStripHitEffData.fillMapFromTkMap(), gen::BaseHadronizer.generateLHE(), popcon::EcalPedestalsHandler.readPedestalTimestamp(), popcon::EcalPedestalsHandler.readPedestalTree(), DTLocalTriggerBaseTest.setConfig(), edm::IndexIntoFile.setNumberOfEvents(), TagProbeFitter.setSplitMode(), StatisticsPlots(), SummaryHisto(), and DTLocalTriggerBaseTest.~DTLocalTriggerBaseTest().

sites

createIOVlist.sites

Definition at line 56 of file createIOVlist.py.

Referenced by l1t::Stage2Layer2TauAlgorithmFirmwareImp1.makeSecClusters(), and l1t::Stage2Layer2TauAlgorithmFirmwareImp1.merging().

---

Generated for CMSSW Reference Manual by  1.8.14