cmsswPreprocessor.CmsswPreprocessor Class Reference

Public Member Functions
def	__init__ (self, configFile, command="cmsRun", addOrigAsSecondary=True, prefetch=False, options={})
def	endLoop (self, component)
def	maybePrefetchFiles (self, component)
def	prefetchOneXrootdFile (self, fname)
def	run (self, component, wd, firstEvent, nEvents)

Public Attributes
	addOrigAsSecondary
	command
	configFile
	garbageFiles
	options
	prefetch

Detailed Description

Definition at line 11 of file cmsswPreprocessor.py.

Constructor & Destructor Documentation

__init__()

def cmsswPreprocessor.CmsswPreprocessor.__init__	(		self,
			configFile,
			command = "cmsRun",
			addOrigAsSecondary = True,
			prefetch = False,
			options = {}
	)

Definition at line 12 of file cmsswPreprocessor.py.

    def __init__(self,configFile,command="cmsRun", addOrigAsSecondary=True, prefetch=False, options={}) :
        self.configFile=configFile
        self.command=command
        self.addOrigAsSecondary=addOrigAsSecondary
        self.prefetch=prefetch
        self.garbageFiles=[]
        self.options=options

Member Function Documentation

endLoop()

def cmsswPreprocessor.CmsswPreprocessor.endLoop	(		self,
			component
	)

Definition at line 46 of file cmsswPreprocessor.py.

References print().

    def endLoop(self,component):
        for fname in component._preprocessor_tempFiles:
            print("Removing local cache file ",fname)
            os.remove(fname)
        component._preprocessor_tempFiles = []

maybePrefetchFiles()

def cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles	(		self,
			component
	)

Definition at line 34 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.prefetch, and cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile().

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

    def maybePrefetchFiles(self,component):
        newfiles = []
        component._preprocessor_tempFiles = []
        for fn in component.files:
            if self.prefetch and fn.startswith("root://"):
                (newfile,istemp) = self.prefetchOneXrootdFile(fn)
                newfiles.append(newfile)
                if istemp: 
                    component._preprocessor_tempFiles.append(newfile)
            else:
                newfiles.append(fn)
        component.files = newfiles

prefetchOneXrootdFile()

def cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile	(		self,
			fname
	)

Definition at line 20 of file cmsswPreprocessor.py.

References join(), print(), and python.rootplot.root2matplotlib.replace().

Referenced by cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles().

    def prefetchOneXrootdFile(self,fname):
        tmpdir = os.environ['TMPDIR'] if 'TMPDIR' in os.environ else "/tmp"
        rndchars  = "".join([hex(ord(i))[2:] for i in os.urandom(8)])
        localfile = "%s/%s-%s.root" % (tmpdir, os.path.basename(fname).replace(".root",""), rndchars)
        try:
            print("Fetching %s to local path %s " % (fname,localfile))
            start = timeit.default_timer()
            subprocess.check_output(["xrdcp","-f","-N",fname,localfile])
            print("Time used for transferring the file locally: %s s" % (timeit.default_timer() - start))
            return (localfile,True)
        except:
            print("Could not save file locally, will run from remote")
            if os.path.exists(localfile): os.remove(localfile) # delete in case of incomplete transfer
            return (fname,False)

run()

def cmsswPreprocessor.CmsswPreprocessor.run	(		self,
			component,
			wd,
			firstEvent,
			nEvents
	)

Definition at line 51 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.addOrigAsSecondary, reco.ceil(), runall.testit.command, cmsswPreprocessor.CmsswPreprocessor.command, WorkFlow.WorkFlowBlock.command, TritonService::FallbackOpts.command, cmsswPreprocessor.CmsswPreprocessor.configFile, XMLConfigReader.configFile, edmUtils.edmFileLs(), beamvalidation.exit(), dqmMemoryStats.float, createfilelist.int, cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles(), cmsswPreprocessor.CmsswPreprocessor.options, DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, TestProcess.TestProcess.options, confdb.HLTProcess.options, Config.Process.options, and print().

    def run(self,component,wd,firstEvent,nEvents):
        if firstEvent != 0: raise RuntimeError("The preprocessor can't skip events at the moment")
        fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit', (1,1))
        if fineSplitFactor > 1:
            if len(component.files) != 1:
                raise RuntimeError("Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (component.name, component.files))
            evtsInFile = edmFileLs(component.files[0])['events']
            if nEvents in (None, -1) or nEvents > evtsInFile: nEvents =  evtsInFile
            nEvents = int(ceil(nEvents/float(fineSplitFactor)))
            firstEvent = fineSplitIndex * nEvents
            # Now we will run on these events, and the output will contain only those
            # Thus, we switch off fine-split in the component
            component.fineSplit = (1,1)
        if nEvents is None:
            nEvents = -1
        self.maybePrefetchFiles(component)
        cmsswConfig = imp.load_source("cmsRunProcess",os.path.expandvars(self.configFile))
        inputfiles= []
        for fn in component.files :
            if not re.match("file:.*",fn) and not re.match("root:.*",fn) :
                fn="file:"+fn
            inputfiles.append(fn)

        # Four cases: 
        # - no options, cmsswConfig with initialize function
        #     run initialize with default parameters
        # - filled options, cmsswConfig with initialize function
        #     pass on options to initialize
        # - no options, classic cmsswConfig
        #     legacy mode
        # - filled options, classic cmsswConfig
        #     legacy mode but warn that options are not passed on

        if hasattr(cmsswConfig, "initialize"):
            if len(self.options) == 0:                                
                cmsswConfig.process = cmsswConfig.initialize()                                 
            else:                                
                cmsswConfig.process = cmsswConfig.initialize(**self.options)                                 
        else:
            if len(self.options) == 0:                             
                pass
            else:
                print("WARNING: cmsswPreprocessor received options but can't pass on to cmsswConfig")

        cmsswConfig.process.source.fileNames = inputfiles
        # cmsRun will not create the output file if maxEvents==0, leading to crash of the analysis downstream.
        # Thus, we set nEvents = 1 if the input file is empty (the output file will be empty as well).
        cmsswConfig.process.maxEvents.input = 1 if (fineSplitFactor>1 and nEvents==0) else nEvents
        cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32(0 if (fineSplitFactor>1 and nEvents==0) else firstEvent)
        #fixme: implement skipEvent / firstevent

        outfilename=wd+"/cmsswPreProcessing.root"
        # for outName in cmsswConfig.process.endpath.moduleNames():
        for module in cmsswConfig.process.endpaths.viewvalues():
            for outName in module.moduleNames():
                out = getattr(cmsswConfig.process,outName)
            if not hasattr(out,"fileName"): continue
            out.fileName = outfilename

        if not hasattr(component,"options"):
            component.options = CFG(name="postCmsrunOptions")
        #use original as primary and new as secondary 
        #component.options.inputFiles= component.files
        #component.options.secondaryInputFiles=[outfilename]

        #use new as primary and original as secondary
        if self.addOrigAsSecondary:
            component.options.secondaryInputFiles= component.files
        component.options.inputFiles=[outfilename]
        component.files=[outfilename]

        configfile=wd+"/cmsRun_config.py"
        f = open(configfile, 'w')
        f.write(cmsswConfig.process.dumpPython())
        f.close()
        runstring="%s %s >& %s/cmsRun.log" % (self.command,configfile,wd)
        print("Running pre-processor: %s " %runstring)
        ret=os.system(runstring)
        if ret != 0:
            print("CMSRUN failed")
            exit(ret)
        return component

Member Data Documentation

addOrigAsSecondary

cmsswPreprocessor.CmsswPreprocessor.addOrigAsSecondary

Definition at line 15 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

command

cmsswPreprocessor.CmsswPreprocessor.command

Definition at line 14 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

configFile

cmsswPreprocessor.CmsswPreprocessor.configFile

Definition at line 13 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

garbageFiles

cmsswPreprocessor.CmsswPreprocessor.garbageFiles

Definition at line 17 of file cmsswPreprocessor.py.

options

cmsswPreprocessor.CmsswPreprocessor.options

Definition at line 18 of file cmsswPreprocessor.py.

Referenced by betterConfigParser.BetterConfigParser.__updateDict(), submitPVValidationJobs.BetterConfigParser.__updateDict(), CrabHelper.CrabHelper.check_crabtask(), betterConfigParser.BetterConfigParser.checkInput(), CrabHelper.CrabHelper.crab_config_filename(), CrabHelper.CrabHelper.crab_config_filepath(), CrabHelper.CrabHelper.crab_taskname(), CrabHelper.CrabHelper.fill_options_from_crab_config(), edmIntegrityCheck.IntegrityCheck.listFiles(), production_tasks.MonitorJobs.monitor(), edmIntegrityCheck.IntegrityCheck.query(), production_tasks.BaseDataset.query(), edmIntegrityCheck.IntegrityCheck.report(), cmsswPreprocessor.CmsswPreprocessor.run(), production_tasks.CheckDatasetExists.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.CreateJobDirectory.run(), production_tasks.SourceCFG.run(), production_tasks.FullCFG.run(), production_tasks.WriteToDatasets.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), production_tasks.WriteJobReport.run(), production_tasks.CleanJobFiles.run(), edmIntegrityCheck.IntegrityCheck.structured(), CrabHelper.CrabHelper.submit_crab_task(), edmIntegrityCheck.IntegrityCheck.test(), and CrabHelper.CrabHelper.write_crabConfig().

prefetch

cmsswPreprocessor.CmsswPreprocessor.prefetch

Definition at line 16 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles().

---

Generated for CMSSW Reference Manual by  1.8.14