CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
heppy_loop Namespace Reference

Functions

def callBack
 
def chunks
 
def createOutputDir
 
def getHeppyOption
 
def main
 
def runLoop
 
def runLoopAsync
 
def setHeppyOption
 
def split
 

Variables

dictionary _heppyGlobalOptions = {}
 
string action = 'store_true'
 
 default = None)
 
string dest = "nevents"
 
string help = "number of events to process"
 
 loop = None
 
list oldv = sys.argv[:]
 
tuple parser = OptionParser()
 
string type = "int"
 

Function Documentation

def heppy_loop.callBack (   result)

Definition at line 30 of file heppy_loop.py.

References print(), and str.

Referenced by cond::persistency::CoralMsgReporter.subscribe().

30 
31 def callBack( result ):
32  pass
33  print('production done:', str(result))
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
heppy_loop.callBack
def callBack
Definition: heppy_loop.py:30
str
#define str(s)
Definition: TestProcessor.cc:53
def heppy_loop.chunks (   l,
  n 
)

Definition at line 95 of file heppy_loop.py.

References sistrip::SpyUtilities.range().

Referenced by split().

95 
96 def chunks(l, n):
97  return [l[i:i+n] for i in range(0, len(l), n)]
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
heppy_loop.chunks
def chunks
Definition: heppy_loop.py:95
def heppy_loop.createOutputDir (   dir,
  components,
  force 
) 
Creates the output dir, dealing with the case where dir exists.

Definition at line 68 of file heppy_loop.py.

References join(), and print().

Referenced by main().

68 
69 def createOutputDir(dir, components, force):
70  '''Creates the output dir, dealing with the case where dir exists.'''
71  answer = None
72  try:
73  os.mkdir(dir)
74  return True
75  except OSError:
76  print('directory %s already exists' % dir)
77  print('contents: ')
78  dirlist = [path for path in os.listdir(dir) if os.path.isdir( '/'.join([dir, path]) )]
79  pprint( dirlist )
80  print('component list: ')
81  print([comp.name for comp in components])
82  if force is True:
83  print('force mode, continue.')
84  return True
85  else:
86  while answer not in ['Y','y','yes','N','n','no']:
87  answer = raw_input('Continue? [y/n]')
88  if answer.lower().startswith('n'):
89  return False
90  elif answer.lower().startswith('y'):
91  return True
92  else:
93  raise ValueError( ' '.join(['answer can not have this value!',
94  answer]) )
heppy_loop.createOutputDir
def createOutputDir
Definition: heppy_loop.py:68
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def heppy_loop.getHeppyOption (   name,
  default = None 
)

Definition at line 129 of file heppy_loop.py.

129 
130 def getHeppyOption(name,default=None):
131  global _heppyGlobalOptions
return _heppyGlobalOptions[name] if name in _heppyGlobalOptions else default
heppy_loop.getHeppyOption
def getHeppyOption
Definition: heppy_loop.py:129
def heppy_loop.main (   options,
  args,
  parser 
)

Definition at line 136 of file heppy_loop.py.

References createOutputDir(), min(), print(), runLoop(), and split().

136 
137 def main( options, args, parser ):
138 
139  if len(args) != 2:
140  parser.print_help()
141  print('ERROR: please provide the processing name and the component list')
142  sys.exit(1)
143 
144  outDir = args[0]
145  if os.path.exists(outDir) and not os.path.isdir( outDir ):
146  parser.print_help()
147  print('ERROR: when it exists, first argument must be a directory.')
148  sys.exit(2)
149  cfgFileName = args[1]
150  if not os.path.isfile( cfgFileName ):
151  parser.print_help()
152  print('ERROR: second argument must be an existing file (your input cfg).')
153  sys.exit(3)
154 
155  if options.verbose:
156  import logging
157  logging.basicConfig(level=logging.INFO)
158 
159  # Propagate global options to _heppyGlobalOptions within this module
160  # I have to import it explicitly, 'global' does not work since the
161  # module is not set when executing the main
162  from PhysicsTools.HeppyCore.framework.heppy_loop import _heppyGlobalOptions
163  for opt in options.extraOptions:
164  if "=" in opt:
165  (key,val) = opt.split("=",1)
166  _heppyGlobalOptions[key] = val
167  else:
168  _heppyGlobalOptions[opt] = True
169 
170  file = open( cfgFileName, 'r' )
171  cfg = imp.load_source( 'PhysicsTools.HeppyCore.__cfg_to_run__', cfgFileName, file)
172 
173  selComps = [comp for comp in cfg.config.components if len(comp.files)>0]
174  selComps = split(selComps)
175  # for comp in selComps:
176  # print comp
177  if len(selComps)>options.ntasks:
178  print("WARNING: too many threads {tnum}, will just use a maximum of {jnum}.".format(tnum=len(selComps),jnum=options.ntasks))
179  if not createOutputDir(outDir, selComps, options.force):
180  print('exiting')
181  sys.exit(0)
182  if len(selComps)>1:
183  shutil.copy( cfgFileName, outDir )
184  pool = Pool(processes=min(len(selComps),options.ntasks))
185  ## workaround for a scoping problem in ipython+multiprocessing
186  import PhysicsTools.HeppyCore.framework.heppy_loop as ML
187  for comp in selComps:
188  print('submitting', comp.name)
189  pool.apply_async( ML.runLoopAsync, [comp, outDir, 'PhysicsTools.HeppyCore.__cfg_to_run__', options],
190  callback=ML.callBack)
191  pool.close()
192  pool.join()
193  else:
194  # when running only one loop, do not use multiprocessor module.
195  # then, the exceptions are visible -> use only one sample for testing
196  global loop
197  loop = runLoop( comp, outDir, cfg.config, options )
198  return loop
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:46
heppy_loop.createOutputDir
def createOutputDir
Definition: heppy_loop.py:68
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
min
T min(T a, T b)
Definition: MathUtil.h:58
heppy_loop.main
def main
Definition: heppy_loop.py:136
heppy_loop.split
def split
Definition: heppy_loop.py:98
def heppy_loop.runLoop (   comp,
  outDir,
  config,
  options 
)

Definition at line 46 of file heppy_loop.py.

References join().

Referenced by main(), and runLoopAsync().

46 
47 def runLoop( comp, outDir, config, options):
48  fullName = '/'.join( [outDir, comp.name ] )
49  # import pdb; pdb.set_trace()
50  config.components = [comp]
51  loop = Looper( fullName,
52  config,
53  options.nevents, 0,
54  nPrint = options.nprint,
55  timeReport = options.timeReport,
56  quiet = options.quiet)
57  # print loop
58  if options.iEvent is None:
59  loop.loop()
60  loop.write()
61  # print loop
62  else:
63  # loop.InitOutput()
64  iEvent = int(options.iEvent)
65  loop.process( iEvent )
66  return loop
67 
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:46
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def heppy_loop.runLoopAsync (   comp,
  outDir,
  configName,
  options 
)

Definition at line 34 of file heppy_loop.py.

References print(), and runLoop().

34 
35 def runLoopAsync(comp, outDir, configName, options):
36  try:
37  loop = runLoop( comp, outDir, copy.copy(sys.modules[configName].config), options)
38  return loop.name
39  except Exception:
40  import traceback
41  print("ERROR processing component %s" % comp.name)
42  print(comp)
43  print("STACK TRACE: ")
44  print(traceback.format_exc())
45  raise
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:46
heppy_loop.runLoopAsync
def runLoopAsync
Definition: heppy_loop.py:34
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def heppy_loop.setHeppyOption (   name,
  value = True 
)

Definition at line 132 of file heppy_loop.py.

132 
133 def setHeppyOption(name,value=True):
134  global _heppyGlobalOptions
135  _heppyGlobalOptions[name] = value
heppy_loop.setHeppyOption
def setHeppyOption
Definition: heppy_loop.py:132
def heppy_loop.split (   comps)

Definition at line 98 of file heppy_loop.py.

References chunks(), and sistrip::SpyUtilities.range().

Referenced by main().

98 
99 def split(comps):
100  # import pdb; pdb.set_trace()
101  splitComps = []
102  for comp in comps:
103  if hasattr( comp, 'fineSplitFactor') and comp.fineSplitFactor>1:
104  subchunks = list(range(comp.fineSplitFactor))
105  for ichunk, chunk in enumerate([(f,i) for f in comp.files for i in subchunks]):
106  newComp = copy.deepcopy(comp)
107  newComp.files = [chunk[0]]
108  newComp.fineSplit = ( chunk[1], comp.fineSplitFactor )
109  newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
110  index=ichunk)
111  splitComps.append( newComp )
112  elif hasattr( comp, 'splitFactor') and comp.splitFactor>1:
113  chunkSize = len(comp.files) / comp.splitFactor
114  if len(comp.files) % comp.splitFactor:
115  chunkSize += 1
116  # print 'chunk size',chunkSize, len(comp.files), comp.splitFactor
117  for ichunk, chunk in enumerate( chunks( comp.files, chunkSize)):
118  newComp = copy.deepcopy(comp)
119  newComp.files = chunk
120  newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
121  index=ichunk)
122  splitComps.append( newComp )
123  else:
124  splitComps.append( comp )
125  return splitComps
126 
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
heppy_loop.chunks
def chunks
Definition: heppy_loop.py:95
heppy_loop.split
def split
Definition: heppy_loop.py:98

Variable Documentation

dictionary heppy_loop._heppyGlobalOptions = {}

Definition at line 127 of file heppy_loop.py.

string heppy_loop.action = 'store_true'

Definition at line 27 of file heppy_loop.py.

int heppy_loop.default = None)

Definition at line 16 of file heppy_loop.py.

string heppy_loop.dest = "nevents"

Definition at line 13 of file heppy_loop.py.

string heppy_loop.help = "number of events to process"

Definition at line 15 of file heppy_loop.py.

tuple heppy_loop.loop = None

Definition at line 28 of file heppy_loop.py.

Referenced by gpuVertexFinder.__attribute__(), optutl::CommandLineParser._getSectionFiles(), CACell.checkAlignmentAndAct(), HGCDoublet.checkCompatibilityAndTag(), RawDataConverter.ClearData(), CmsShowMainFrame.CmsShowMainFrame(), DDCheckMaterial(), SiStripTFile.dirContent(), MillePedeAlignmentAlgorithm.doIO(), LaserAlignment.DumpPosFileSet(), LaserAlignment.DumpStripFileSet(), SETSeedFinder.estimateMomentum(), PhysicsTools::MVAComputer.evalInternal(), LMFDefFabric.getColor(), LMFDefFabric.getColorFromID(), RawDataConverter.GetDigis(), LMFDefFabric.getRunTag(), LMFDefFabric.getRunTagFromID(), LMFDefFabric.getTrigType(), LMFDefFabric.getTrigTypeFromID(), pat::EventHypothesis.loop(), output(), optutl::CommandLineParser.parseArguments(), EcalDigiSelector.produce(), SiStripTFile.readDQMFormat(), TrajectoryManager.reconstruct(), stdcomb.recursive_combination(), LMFColoredTable.setColor(), LMFColoredTable.setSystem(), HcalTestAnalysis.update(), DDI::Polycone.volume(), DDI::Polyhedra.volume(), and gpuVertexFinder.while().

list heppy_loop.oldv = sys.argv[:]

Definition at line 18 of file heppy_loop.py.

tuple heppy_loop.parser = OptionParser()

Definition at line 5 of file heppy_loop.py.

string heppy_loop.type = "int"

Definition at line 14 of file heppy_loop.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5