CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
heppy_loop Namespace Reference

Functions

def callBack
 
def chunks
 
def createOutputDir
 
def getHeppyOption
 
def main
 
def runLoop
 
def runLoopAsync
 
def split
 

Variables

dictionary _heppyGlobalOptions = {}
 
string action = 'store_true'
 
 default = None)
 
string dest = "nevents"
 
string help = "number of events to process"
 
 loop = None
 
list oldv = sys.argv[:]
 
tuple parser = OptionParser()
 
string type = "int"
 

Function Documentation

def heppy_loop.callBack (   result)

Definition at line 28 of file heppy_loop.py.

28 
29 def callBack( result ):
30  pass
31  print 'production done:', str(result)
heppy_loop.callBack
def callBack
Definition: heppy_loop.py:28
def heppy_loop.chunks (   l,
  n 
)

Definition at line 93 of file heppy_loop.py.

Referenced by split().

93 
94 def chunks(l, n):
95  return [l[i:i+n] for i in range(0, len(l), n)]
heppy_loop.chunks
def chunks
Definition: heppy_loop.py:93
def heppy_loop.createOutputDir (   dir,
  components,
  force 
) 
Creates the output dir, dealing with the case where dir exists.

Definition at line 66 of file heppy_loop.py.

References join().

Referenced by main().

66 
67 def createOutputDir(dir, components, force):
68  '''Creates the output dir, dealing with the case where dir exists.'''
69  answer = None
70  try:
71  os.mkdir(dir)
72  return True
73  except OSError:
74  print 'directory %s already exists' % dir
75  print 'contents: '
76  dirlist = [path for path in os.listdir(dir) if os.path.isdir( '/'.join([dir, path]) )]
77  pprint( dirlist )
78  print 'component list: '
79  print [comp.name for comp in components]
80  if force is True:
81  print 'force mode, continue.'
82  return True
83  else:
84  while answer not in ['Y','y','yes','N','n','no']:
85  answer = raw_input('Continue? [y/n]')
86  if answer.lower().startswith('n'):
87  return False
88  elif answer.lower().startswith('y'):
89  return True
90  else:
91  raise ValueError( ' '.join(['answer can not have this value!',
92  answer]) )
heppy_loop.createOutputDir
def createOutputDir
Definition: heppy_loop.py:66
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def heppy_loop.getHeppyOption (   name,
  default = None 
)

Definition at line 127 of file heppy_loop.py.

127 
128 def getHeppyOption(name,default=None):
129  global _heppyGlobalOptions
130  return _heppyGlobalOptions[name] if name in _heppyGlobalOptions else default
heppy_loop.getHeppyOption
def getHeppyOption
Definition: heppy_loop.py:127
def heppy_loop.main (   options,
  args,
  parser 
)

Definition at line 131 of file heppy_loop.py.

References createOutputDir(), min(), runLoop(), and split().

131 
132 def main( options, args, parser ):
133 
134  if len(args) != 2:
135  parser.print_help()
136  print 'ERROR: please provide the processing name and the component list'
137  sys.exit(1)
138 
139  outDir = args[0]
140  if os.path.exists(outDir) and not os.path.isdir( outDir ):
141  parser.print_help()
142  print 'ERROR: when it exists, first argument must be a directory.'
143  sys.exit(2)
144  cfgFileName = args[1]
145  if not os.path.isfile( cfgFileName ):
146  parser.print_help()
147  print 'ERROR: second argument must be an existing file (your input cfg).'
148  sys.exit(3)
149 
150  if options.verbose:
151  import logging
152  logging.basicConfig(level=logging.INFO)
153 
154  # Propagate global options to _heppyGlobalOptions within this module
155  # I have to import it explicitly, 'global' does not work since the
156  # module is not set when executing the main
157  from PhysicsTools.HeppyCore.framework.heppy_loop import _heppyGlobalOptions
158  for opt in options.extraOptions:
159  if "=" in opt:
160  (key,val) = opt.split("=",1)
161  _heppyGlobalOptions[key] = val
162  else:
163  _heppyGlobalOptions[opt] = True
164 
165  file = open( cfgFileName, 'r' )
166  cfg = imp.load_source( 'PhysicsTools.HeppyCore.__cfg_to_run__', cfgFileName, file)
167 
168  selComps = [comp for comp in cfg.config.components if len(comp.files)>0]
169  selComps = split(selComps)
170  # for comp in selComps:
171  # print comp
172  if len(selComps)>10:
173  print "WARNING: too many threads {tnum}, will just use a maximum of 10.".format(tnum=len(selComps))
174  if not createOutputDir(outDir, selComps, options.force):
175  print 'exiting'
176  sys.exit(0)
177  if len(selComps)>1:
178  shutil.copy( cfgFileName, outDir )
179  pool = Pool(processes=min(len(selComps),10))
180  ## workaround for a scoping problem in ipython+multiprocessing
181  import PhysicsTools.HeppyCore.framework.heppy_loop as ML
182  for comp in selComps:
183  print 'submitting', comp.name
184  pool.apply_async( ML.runLoopAsync, [comp, outDir, 'PhysicsTools.HeppyCore.__cfg_to_run__', options],
185  callback=ML.callBack)
186  pool.close()
187  pool.join()
188  else:
189  # when running only one loop, do not use multiprocessor module.
190  # then, the exceptions are visible -> use only one sample for testing
191  global loop
192  loop = runLoop( comp, outDir, cfg.config, options )
193  return loop
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:44
heppy_loop.createOutputDir
def createOutputDir
Definition: heppy_loop.py:66
min
T min(T a, T b)
Definition: MathUtil.h:58
heppy_loop.main
def main
Definition: heppy_loop.py:131
heppy_loop.split
def split
Definition: heppy_loop.py:96
def heppy_loop.runLoop (   comp,
  outDir,
  config,
  options 
)

Definition at line 44 of file heppy_loop.py.

References join().

Referenced by main(), and runLoopAsync().

44 
45 def runLoop( comp, outDir, config, options):
46  fullName = '/'.join( [outDir, comp.name ] )
47  # import pdb; pdb.set_trace()
48  config.components = [comp]
49  loop = Looper( fullName,
50  config,
51  options.nevents, 0,
52  nPrint = options.nprint,
53  timeReport = options.timeReport,
54  quiet = options.quiet)
55  # print loop
56  if options.iEvent is None:
57  loop.loop()
58  loop.write()
59  # print loop
60  else:
61  # loop.InitOutput()
62  iEvent = int(options.iEvent)
63  loop.process( iEvent )
64  return loop
65 
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:44
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def heppy_loop.runLoopAsync (   comp,
  outDir,
  configName,
  options 
)

Definition at line 32 of file heppy_loop.py.

References runLoop().

32 
33 def runLoopAsync(comp, outDir, configName, options):
34  try:
35  loop = runLoop( comp, outDir, copy.copy(sys.modules[configName].config), options)
36  return loop.name
37  except Exception:
38  import traceback
39  print "ERROR processing component %s" % comp.name
40  print comp
41  print "STACK TRACE: "
42  print traceback.format_exc()
43  raise
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:44
heppy_loop.runLoopAsync
def runLoopAsync
Definition: heppy_loop.py:32
def heppy_loop.split (   comps)

Definition at line 96 of file heppy_loop.py.

References chunks().

Referenced by main().

96 
97 def split(comps):
98  # import pdb; pdb.set_trace()
99  splitComps = []
100  for comp in comps:
101  if hasattr( comp, 'fineSplitFactor') and comp.fineSplitFactor>1:
102  subchunks = range(comp.fineSplitFactor)
103  for ichunk, chunk in enumerate([(f,i) for f in comp.files for i in subchunks]):
104  newComp = copy.deepcopy(comp)
105  newComp.files = [chunk[0]]
106  newComp.fineSplit = ( chunk[1], comp.fineSplitFactor )
107  newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
108  index=ichunk)
109  splitComps.append( newComp )
110  elif hasattr( comp, 'splitFactor') and comp.splitFactor>1:
111  chunkSize = len(comp.files) / comp.splitFactor
112  if len(comp.files) % comp.splitFactor:
113  chunkSize += 1
114  # print 'chunk size',chunkSize, len(comp.files), comp.splitFactor
115  for ichunk, chunk in enumerate( chunks( comp.files, chunkSize)):
116  newComp = copy.deepcopy(comp)
117  newComp.files = chunk
118  newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
119  index=ichunk)
120  splitComps.append( newComp )
121  else:
122  splitComps.append( comp )
123  return splitComps
124 
heppy_loop.chunks
def chunks
Definition: heppy_loop.py:93
heppy_loop.split
def split
Definition: heppy_loop.py:96

Variable Documentation

dictionary heppy_loop._heppyGlobalOptions = {}

Definition at line 125 of file heppy_loop.py.

string heppy_loop.action = 'store_true'

Definition at line 27 of file heppy_loop.py.

list heppy_loop.default = None)

Definition at line 16 of file heppy_loop.py.

string heppy_loop.dest = "nevents"

Definition at line 13 of file heppy_loop.py.

string heppy_loop.help = "number of events to process"

Definition at line 15 of file heppy_loop.py.

tuple heppy_loop.loop = None

Definition at line 26 of file heppy_loop.py.

list heppy_loop.oldv = sys.argv[:]

Definition at line 16 of file heppy_loop.py.

tuple heppy_loop.parser = OptionParser()

Definition at line 5 of file heppy_loop.py.

string heppy_loop.type = "int"

Definition at line 14 of file heppy_loop.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5