CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
cmsswPreprocessor.CmsswPreprocessor Class Reference

Public Member Functions

def __init__
 
def endLoop
 
def maybePrefetchFiles
 
def prefetchOneXrootdFile
 
def run
 

Public Attributes

 addOrigAsSecondary
 
 command
 
 configFile
 
 garbageFiles
 
 options
 
 prefetch
 

Detailed Description

Definition at line 10 of file cmsswPreprocessor.py.

Constructor & Destructor Documentation

def cmsswPreprocessor.CmsswPreprocessor.__init__ (   self,
  configFile,
  command = "cmsRun",
  addOrigAsSecondary = True,
  prefetch = False,
  options = {} 
)

Definition at line 11 of file cmsswPreprocessor.py.

11 
12  def __init__(self,configFile,command="cmsRun", addOrigAsSecondary=True, prefetch=False, options={}) :
13  self.configFile=configFile
14  self.command=command
15  self.addOrigAsSecondary=addOrigAsSecondary
16  self.prefetch=prefetch
17  self.garbageFiles=[]
18  self.options=options

Member Function Documentation

def cmsswPreprocessor.CmsswPreprocessor.endLoop (   self,
  component 
)

Definition at line 45 of file cmsswPreprocessor.py.

45 
46  def endLoop(self,component):
47  for fname in component._preprocessor_tempFiles:
48  print "Removing local cache file ",fname
49  os.remove(fname)
component._preprocessor_tempFiles = []
def cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles (   self,
  component 
)

Definition at line 33 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.prefetch, and cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile().

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

33 
34  def maybePrefetchFiles(self,component):
35  newfiles = []
36  component._preprocessor_tempFiles = []
37  for fn in component.files:
38  if self.prefetch and fn.startswith("root://"):
39  (newfile,istemp) = self.prefetchOneXrootdFile(fn)
40  newfiles.append(newfile)
41  if istemp:
42  component._preprocessor_tempFiles.append(newfile)
43  else:
44  newfiles.append(fn)
component.files = newfiles
def cmsswPreprocessor.CmsswPreprocessor.prefetchOneXrootdFile (   self,
  fname 
)

Definition at line 19 of file cmsswPreprocessor.py.

References join(), and python.rootplot.root2matplotlib.replace().

Referenced by cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles().

19 
20  def prefetchOneXrootdFile(self,fname):
21  tmpdir = os.environ['TMPDIR'] if 'TMPDIR' in os.environ else "/tmp"
22  rndchars = "".join([hex(ord(i))[2:] for i in os.urandom(8)])
23  localfile = "%s/%s-%s.root" % (tmpdir, os.path.basename(fname).replace(".root",""), rndchars)
24  try:
25  print "Fetching %s to local path %s " % (fname,localfile)
26  start = timeit.default_timer()
27  subprocess.check_output(["xrdcp","-f","-N",fname,localfile])
28  print "Time used for transferring the file locally: %s s" % (timeit.default_timer() - start)
29  return (localfile,True)
30  except:
31  print "Could not save file locally, will run from remote"
32  if os.path.exists(localfile): os.remove(localfile) # delete in case of incomplete transfer
return (fname,False)
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsswPreprocessor.CmsswPreprocessor.run (   self,
  component,
  wd,
  firstEvent,
  nEvents 
)

Definition at line 50 of file cmsswPreprocessor.py.

References cmsswPreprocessor.CmsswPreprocessor.addOrigAsSecondary, runall.testit.command, cmsswPreprocessor.CmsswPreprocessor.command, WorkFlow.WorkFlowBlock.command, ConfigBuilder.ConfigBuilder.command, cmsswPreprocessor.CmsswPreprocessor.configFile, edmUtils.edmFileLs(), cmsRelvalreport.exit, cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles(), FileExportPlugin.FileExportPlugin.options, cmsswPreprocessor.CmsswPreprocessor.options, DOTExport.DotProducer.options, and confdb.HLTProcess.options.

50 
51  def run(self,component,wd,firstEvent,nEvents):
52  if firstEvent != 0: raise RuntimeError, "The preprocessor can't skip events at the moment"
53  fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit', (1,1))
54  if fineSplitFactor > 1:
55  if len(component.files) != 1:
56  raise RuntimeError, "Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (component.name, component.files)
57  evtsInFile = edmFileLs(component.files[0])['events']
58  if nEvents in (None, -1) or nEvents > evtsInFile: nEvents = evtsInFile
59  nEvents = int(ceil(nEvents/float(fineSplitFactor)))
60  firstEvent = fineSplitIndex * nEvents
61  # Now we will run on these events, and the output will contain only those
62  # Thus, we switch off fine-split in the component
63  component.fineSplit = (1,1)
64  if nEvents is None:
65  nEvents = -1
66  self.maybePrefetchFiles(component)
67  cmsswConfig = imp.load_source("cmsRunProcess",os.path.expandvars(self.configFile))
68  inputfiles= []
69  for fn in component.files :
70  if not re.match("file:.*",fn) and not re.match("root:.*",fn) :
71  fn="file:"+fn
72  inputfiles.append(fn)
73 
74  # Four cases:
75  # - no options, cmsswConfig with initialize function
76  # run initialize with default parameters
77  # - filled options, cmsswConfig with initialize function
78  # pass on options to initialize
79  # - no options, classic cmsswConfig
80  # legacy mode
81  # - filled options, classic cmsswConfig
82  # legacy mode but warn that options are not passed on
83 
84  if hasattr(cmsswConfig, "initialize"):
85  if len(self.options) == 0:
86  cmsswConfig.process = cmsswConfig.initialize()
87  else:
88  cmsswConfig.process = cmsswConfig.initialize(**self.options)
89  else:
90  if len(self.options) == 0:
91  pass
92  else:
93  print "WARNING: cmsswPreprocessor received options but can't pass on to cmsswConfig"
94 
95  cmsswConfig.process.source.fileNames = inputfiles
96  # cmsRun will not create the output file if maxEvents==0, leading to crash of the analysis downstream.
97  # Thus, we set nEvents = 1 if the input file is empty (the output file will be empty as well).
98  cmsswConfig.process.maxEvents.input = 1 if (fineSplitFactor>1 and nEvents==0) else nEvents
99  cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32(0 if (fineSplitFactor>1 and nEvents==0) else firstEvent)
100  #fixme: implement skipEvent / firstevent
101 
102  outfilename=wd+"/cmsswPreProcessing.root"
103  # for outName in cmsswConfig.process.endpath.moduleNames():
104  for module in cmsswConfig.process.endpaths.viewvalues():
105  for outName in module.moduleNames():
106  out = getattr(cmsswConfig.process,outName)
107  if not hasattr(out,"fileName"): continue
108  out.fileName = outfilename
109 
110  if not hasattr(component,"options"):
111  component.options = CFG(name="postCmsrunOptions")
112  #use original as primary and new as secondary
113  #component.options.inputFiles= component.files
114  #component.options.secondaryInputFiles=[outfilename]
115 
116  #use new as primary and original as secondary
117  if self.addOrigAsSecondary:
118  component.options.secondaryInputFiles= component.files
119  component.options.inputFiles=[outfilename]
120  component.files=[outfilename]
121 
122  configfile=wd+"/cmsRun_config.py"
123  f = open(configfile, 'w')
124  f.write(cmsswConfig.process.dumpPython())
125  f.close()
126  runstring="%s %s >& %s/cmsRun.log" % (self.command,configfile,wd)
127  print "Running pre-processor: %s " %runstring
128  ret=os.system(runstring)
129  if ret != 0:
130  print "CMSRUN failed"
131  exit(ret)
132  return component
edmUtils.edmFileLs
def edmFileLs
Definition: edmUtils.py:3

Member Data Documentation

cmsswPreprocessor.CmsswPreprocessor.addOrigAsSecondary

Definition at line 14 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

cmsswPreprocessor.CmsswPreprocessor.command

Definition at line 13 of file cmsswPreprocessor.py.

Referenced by cmsRelvalreport.Profile._profile_edmsize(), cmsRelvalreport.Profile._profile_igprof(), cmsRelvalreport.Profile._profile_Memcheck_Valgrind(), cmsRelvalreport.Profile._profile_None(), cmsRelvalreport.Profile._profile_valgrindfce(), cmsRelvalreport.Profile._save_output(), and cmsswPreprocessor.CmsswPreprocessor.run().

cmsswPreprocessor.CmsswPreprocessor.configFile

Definition at line 12 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.run().

cmsswPreprocessor.CmsswPreprocessor.garbageFiles

Definition at line 16 of file cmsswPreprocessor.py.

cmsswPreprocessor.CmsswPreprocessor.options

Definition at line 17 of file cmsswPreprocessor.py.

Referenced by betterConfigParser.BetterConfigParser.__updateDict(), betterConfigParser.BetterConfigParser.checkInput(), DOTExport.DotExport.export(), edmIntegrityCheck.IntegrityCheck.query(), cmsswPreprocessor.CmsswPreprocessor.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), production_tasks.CleanJobFiles.run(), and DOTExport.DotExport.write_output().

cmsswPreprocessor.CmsswPreprocessor.prefetch

Definition at line 15 of file cmsswPreprocessor.py.

Referenced by cmsswPreprocessor.CmsswPreprocessor.maybePrefetchFiles().

Generated for CMSSW Reference Manual by doxygen 1.8.5