Public Member Functions
def	__init__ (self, configFile, command="cmsRun", addOrigAsSecondary=True, prefetch=False, options={})

def	endLoop (self, component)

def	maybePrefetchFiles (self, component)

def	prefetchOneXrootdFile (self, fname)

def	run (self, component, wd, firstEvent, nEvents)

Public Attributes
	addOrigAsSecondary

	command

	configFile

	garbageFiles

	options

	prefetch

Detailed Description

Definition at line 10 of file cmsswPreprocessor.py.

Constructor & Destructor Documentation

def cmsswPreprocessor.CmsswPreprocessor.__init__	(	self,
		configFile,
		command = `"cmsRun"`,
		addOrigAsSecondary = `True`,
		prefetch = `False`,
		options = `{}`
	)

Definition at line 11 of file cmsswPreprocessor.py.

     def __init__(self,configFile,command="cmsRun", addOrigAsSecondary=True, prefetch=False, options={}) :
         self.configFile=configFile
         self.command=command
         self.addOrigAsSecondary=addOrigAsSecondary
         self.prefetch=prefetch
         self.garbageFiles=[]
                 self.options=options
     

Member Function Documentation

def cmsswPreprocessor.CmsswPreprocessor.endLoop	(	self,
		component
	)

Definition at line 45 of file cmsswPreprocessor.py.

     def endLoop(self,component):
         for fname in component._preprocessor_tempFiles:
             print "Removing local cache file ",fname
             os.remove(fname)
         component._preprocessor_tempFiles = []

def cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles	(	self,
		component
	)

Definition at line 33 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.prefetch, and cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile().

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

     def maybePrefetchFiles(self,component):
         newfiles = []
         component._preprocessor_tempFiles = []
         for fn in component.files:
             if self.prefetch and fn.startswith("root://"):
                 (newfile,istemp) = self.prefetchOneXrootdFile(fn)
                 newfiles.append(newfile)
                 if istemp: 
                     component._preprocessor_tempFiles.append(newfile)
             else:
                 newfiles.append(fn)
         component.files = newfiles

def cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile	(	self,
		fname
	)

Definition at line 19 of file cmsswPreprocessor.py.

References join(), and python.rootplot.root2matplotlib.replace().

Referenced by cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles().

     def prefetchOneXrootdFile(self,fname):
         tmpdir = os.environ['TMPDIR'] if 'TMPDIR' in os.environ else "/tmp"
         rndchars  = "".join([hex(ord(i))[2:] for i in os.urandom(8)])
         localfile = "%s/%s-%s.root" % (tmpdir, os.path.basename(fname).replace(".root",""), rndchars)
         try:
             print "Fetching %s to local path %s " % (fname,localfile)
             start = timeit.default_timer()
             subprocess.check_output(["xrdcp","-f","-N",fname,localfile])
             print "Time used for transferring the file locally: %s s" % (timeit.default_timer() - start)
             return (localfile,True)
         except:
             print "Could not save file locally, will run from remote"
             if os.path.exists(localfile): os.remove(localfile) # delete in case of incomplete transfer
             return (fname,False)

def cmsswPreprocessor.CmsswPreprocessor.run	(	self,
		component,
		wd,
		firstEvent,
		nEvents
	)

Definition at line 50 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.addOrigAsSecondary, runall.testit.command, cmsswPreprocessor.CmsswPreprocessor.command, WorkFlow.WorkFlowBlock.command, cmsswPreprocessor.CmsswPreprocessor.configFile, XMLConfigReader.configFile, edmUtils.edmFileLs(), cmsRelvalreport.exit, objects.autophobj.float, createfilelist.int, cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles(), FileExportPlugin.FileExportPlugin.options, cmsswPreprocessor.CmsswPreprocessor.options, DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, DOTExport.DotProducer.options, TestProcess.TestProcess.options, and confdb.HLTProcess.options.

     def run(self,component,wd,firstEvent,nEvents):
         if firstEvent != 0: raise RuntimeError("The preprocessor can't skip events at the moment")
                 fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit', (1,1))
                 if fineSplitFactor > 1:
                     if len(component.files) != 1:
                         raise RuntimeError("Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (component.name, component.files))
                     evtsInFile = edmFileLs(component.files[0])['events']
                     if nEvents in (None, -1) or nEvents > evtsInFile: nEvents =  evtsInFile
                     nEvents = int(ceil(nEvents/float(fineSplitFactor)))
                     firstEvent = fineSplitIndex * nEvents
                     # Now we will run on these events, and the output will contain only those
                     # Thus, we switch off fine-split in the component
                     component.fineSplit = (1,1)
         if nEvents is None:
             nEvents = -1
         self.maybePrefetchFiles(component)
         cmsswConfig = imp.load_source("cmsRunProcess",os.path.expandvars(self.configFile))
         inputfiles= []
         for fn in component.files :
             if not re.match("file:.*",fn) and not re.match("root:.*",fn) :
                 fn="file:"+fn
             inputfiles.append(fn)
 
                 # Four cases: 
                 # - no options, cmsswConfig with initialize function
                 #     run initialize with default parameters
                 # - filled options, cmsswConfig with initialize function
                 #     pass on options to initialize
                 # - no options, classic cmsswConfig
                 #     legacy mode
                 # - filled options, classic cmsswConfig
                 #     legacy mode but warn that options are not passed on
 
                 if hasattr(cmsswConfig, "initialize"):
                         if len(self.options) == 0:                                
                                 cmsswConfig.process = cmsswConfig.initialize()                                 
                         else:                                
                                 cmsswConfig.process = cmsswConfig.initialize(**self.options)                                 
                 else:
                         if len(self.options) == 0:                             
                                 pass
                         else:
                                 print "WARNING: cmsswPreprocessor received options but can't pass on to cmsswConfig"
                 
         cmsswConfig.process.source.fileNames = inputfiles
         # cmsRun will not create the output file if maxEvents==0, leading to crash of the analysis downstream.
         # Thus, we set nEvents = 1 if the input file is empty (the output file will be empty as well).
         cmsswConfig.process.maxEvents.input = 1 if (fineSplitFactor>1 and nEvents==0) else nEvents
         cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32(0 if (fineSplitFactor>1 and nEvents==0) else firstEvent)
         #fixme: implement skipEvent / firstevent
 
         outfilename=wd+"/cmsswPreProcessing.root"
         # for outName in cmsswConfig.process.endpath.moduleNames():
         for module in cmsswConfig.process.endpaths.viewvalues():
             for outName in module.moduleNames():
                 out = getattr(cmsswConfig.process,outName)
             if not hasattr(out,"fileName"): continue
                 out.fileName = outfilename
 
         if not hasattr(component,"options"):
             component.options = CFG(name="postCmsrunOptions")
                 #use original as primary and new as secondary 
                 #component.options.inputFiles= component.files
         #component.options.secondaryInputFiles=[outfilename]
 
                 #use new as primary and original as secondary
                 if self.addOrigAsSecondary:
                     component.options.secondaryInputFiles= component.files
         component.options.inputFiles=[outfilename]
                 component.files=[outfilename]
 
         configfile=wd+"/cmsRun_config.py"
         f = open(configfile, 'w')
         f.write(cmsswConfig.process.dumpPython())
         f.close()
         runstring="%s %s >& %s/cmsRun.log" % (self.command,configfile,wd)
         print "Running pre-processor: %s " %runstring
                 ret=os.system(runstring)
                 if ret != 0:
                      print "CMSRUN failed"
                      exit(ret)
         return component