CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes
cmsswPreprocessor.CmsswPreprocessor Class Reference

Public Member Functions

def __init__
 
def endLoop
 
def maybePrefetchFiles
 
def prefetchOneXrootdFile
 
def run
 

Public Attributes

 addOrigAsSecondary
 
 command
 
 configFile
 
 garbageFiles
 
 options
 
 prefetch
 

Detailed Description

Definition at line 11 of file cmsswPreprocessor.py.

Constructor & Destructor Documentation

def cmsswPreprocessor.CmsswPreprocessor.__init__ (   self,
  configFile,
  command = "cmsRun",
  addOrigAsSecondary = True,
  prefetch = False,
  options = {} 
)

Definition at line 12 of file cmsswPreprocessor.py.

12 
13  def __init__(self,configFile,command="cmsRun", addOrigAsSecondary=True, prefetch=False, options={}) :
14  self.configFile=configFile
15  self.command=command
16  self.addOrigAsSecondary=addOrigAsSecondary
17  self.prefetch=prefetch
18  self.garbageFiles=[]
19  self.options=options

Member Function Documentation

def cmsswPreprocessor.CmsswPreprocessor.endLoop (   self,
  component 
)

Definition at line 46 of file cmsswPreprocessor.py.

References print().

46 
47  def endLoop(self,component):
48  for fname in component._preprocessor_tempFiles:
49  print("Removing local cache file ",fname)
50  os.remove(fname)
component._preprocessor_tempFiles = []
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles (   self,
  component 
)

Definition at line 34 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.prefetch, and cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile().

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

34 
35  def maybePrefetchFiles(self,component):
36  newfiles = []
37  component._preprocessor_tempFiles = []
38  for fn in component.files:
39  if self.prefetch and fn.startswith("root://"):
40  (newfile,istemp) = self.prefetchOneXrootdFile(fn)
41  newfiles.append(newfile)
42  if istemp:
43  component._preprocessor_tempFiles.append(newfile)
44  else:
45  newfiles.append(fn)
component.files = newfiles
def cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile (   self,
  fname 
)

Definition at line 20 of file cmsswPreprocessor.py.

References join(), print(), and python.rootplot.root2matplotlib.replace().

Referenced by cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles().

20 
21  def prefetchOneXrootdFile(self,fname):
22  tmpdir = os.environ['TMPDIR'] if 'TMPDIR' in os.environ else "/tmp"
23  rndchars = "".join([hex(ord(i))[2:] for i in os.urandom(8)])
24  localfile = "%s/%s-%s.root" % (tmpdir, os.path.basename(fname).replace(".root",""), rndchars)
25  try:
26  print("Fetching %s to local path %s " % (fname,localfile))
27  start = timeit.default_timer()
28  subprocess.check_output(["xrdcp","-f","-N",fname,localfile])
29  print("Time used for transferring the file locally: %s s" % (timeit.default_timer() - start))
30  return (localfile,True)
31  except:
32  print("Could not save file locally, will run from remote")
33  if os.path.exists(localfile): os.remove(localfile) # delete in case of incomplete transfer
return (fname,False)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def cmsswPreprocessor.CmsswPreprocessor.run (   self,
  component,
  wd,
  firstEvent,
  nEvents 
)

Definition at line 51 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.addOrigAsSecondary, reco.ceil(), runall.testit.command, cmsswPreprocessor.CmsswPreprocessor.command, WorkFlow.WorkFlowBlock.command, cmsswPreprocessor.CmsswPreprocessor.configFile, XMLConfigReader.configFile, edmUtils.edmFileLs(), beamvalidation.exit(), cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles(), cmsswPreprocessor.CmsswPreprocessor.options, DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, TestProcess.TestProcess.options, confdb.HLTProcess.options, Config.Process.options, validateAlignments.ValidationJobMultiIOV.options, and print().

51 
52  def run(self,component,wd,firstEvent,nEvents):
53  if firstEvent != 0: raise RuntimeError("The preprocessor can't skip events at the moment")
54  fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit', (1,1))
55  if fineSplitFactor > 1:
56  if len(component.files) != 1:
57  raise RuntimeError("Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (component.name, component.files))
58  evtsInFile = edmFileLs(component.files[0])['events']
59  if nEvents in (None, -1) or nEvents > evtsInFile: nEvents = evtsInFile
60  nEvents = int(ceil(nEvents/float(fineSplitFactor)))
61  firstEvent = fineSplitIndex * nEvents
62  # Now we will run on these events, and the output will contain only those
63  # Thus, we switch off fine-split in the component
64  component.fineSplit = (1,1)
65  if nEvents is None:
66  nEvents = -1
67  self.maybePrefetchFiles(component)
68  cmsswConfig = imp.load_source("cmsRunProcess",os.path.expandvars(self.configFile))
69  inputfiles= []
70  for fn in component.files :
71  if not re.match("file:.*",fn) and not re.match("root:.*",fn) :
72  fn="file:"+fn
73  inputfiles.append(fn)
74 
75  # Four cases:
76  # - no options, cmsswConfig with initialize function
77  # run initialize with default parameters
78  # - filled options, cmsswConfig with initialize function
79  # pass on options to initialize
80  # - no options, classic cmsswConfig
81  # legacy mode
82  # - filled options, classic cmsswConfig
83  # legacy mode but warn that options are not passed on
84 
85  if hasattr(cmsswConfig, "initialize"):
86  if len(self.options) == 0:
87  cmsswConfig.process = cmsswConfig.initialize()
88  else:
89  cmsswConfig.process = cmsswConfig.initialize(**self.options)
90  else:
91  if len(self.options) == 0:
92  pass
93  else:
94  print("WARNING: cmsswPreprocessor received options but can't pass on to cmsswConfig")
95 
96  cmsswConfig.process.source.fileNames = inputfiles
97  # cmsRun will not create the output file if maxEvents==0, leading to crash of the analysis downstream.
98  # Thus, we set nEvents = 1 if the input file is empty (the output file will be empty as well).
99  cmsswConfig.process.maxEvents.input = 1 if (fineSplitFactor>1 and nEvents==0) else nEvents
100  cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32(0 if (fineSplitFactor>1 and nEvents==0) else firstEvent)
101  #fixme: implement skipEvent / firstevent
102 
103  outfilename=wd+"/cmsswPreProcessing.root"
104  # for outName in cmsswConfig.process.endpath.moduleNames():
105  for module in cmsswConfig.process.endpaths.viewvalues():
106  for outName in module.moduleNames():
107  out = getattr(cmsswConfig.process,outName)
108  if not hasattr(out,"fileName"): continue
109  out.fileName = outfilename
110 
111  if not hasattr(component,"options"):
112  component.options = CFG(name="postCmsrunOptions")
113  #use original as primary and new as secondary
114  #component.options.inputFiles= component.files
115  #component.options.secondaryInputFiles=[outfilename]
116 
117  #use new as primary and original as secondary
118  if self.addOrigAsSecondary:
119  component.options.secondaryInputFiles= component.files
120  component.options.inputFiles=[outfilename]
121  component.files=[outfilename]
122 
123  configfile=wd+"/cmsRun_config.py"
124  f = open(configfile, 'w')
125  f.write(cmsswConfig.process.dumpPython())
126  f.close()
127  runstring="%s %s >& %s/cmsRun.log" % (self.command,configfile,wd)
128  print("Running pre-processor: %s " %runstring)
129  ret=os.system(runstring)
130  if ret != 0:
131  print("CMSRUN failed")
132  exit(ret)
133  return component
reco::ceil
constexpr int32_t ceil(float num)
Definition: constexpr_cmath.h:7
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
edmUtils.edmFileLs
def edmFileLs
Definition: edmUtils.py:3

Member Data Documentation

cmsswPreprocessor.CmsswPreprocessor.addOrigAsSecondary

Definition at line 15 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

cmsswPreprocessor.CmsswPreprocessor.command

Definition at line 14 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

cmsswPreprocessor.CmsswPreprocessor.configFile

Definition at line 13 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

cmsswPreprocessor.CmsswPreprocessor.garbageFiles

Definition at line 17 of file cmsswPreprocessor.py.

cmsswPreprocessor.CmsswPreprocessor.options

Definition at line 18 of file cmsswPreprocessor.py.

Referenced by betterConfigParser.BetterConfigParser.__updateDict(), submitPVValidationJobs.BetterConfigParser.__updateDict(), betterConfigParser.BetterConfigParser.checkInput(), CrabHelper.CrabHelper.crab_config_filename(), CrabHelper.CrabHelper.crab_taskname(), CrabHelper.CrabHelper.fill_options_from_crab_config(), edmIntegrityCheck.IntegrityCheck.query(), cmsswPreprocessor.CmsswPreprocessor.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), and production_tasks.CleanJobFiles.run().

cmsswPreprocessor.CmsswPreprocessor.prefetch

Definition at line 16 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles().

Generated for CMSSW Reference Manual by doxygen 1.8.5