test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
heppy_loop Namespace Reference

Functions

def callBack
 
def chunks
 
def createOutputDir
 
def getHeppyOption
 
def main
 
def runLoop
 
def runLoopAsync
 
def setHeppyOption
 
def split
 

Variables

dictionary _heppyGlobalOptions = {}
 
string action = 'store_true'
 
 default = None)
 
string dest = "nevents"
 
string help = "number of events to process"
 
 loop = None
 
list oldv = sys.argv[:]
 
tuple parser = OptionParser()
 
string type = "int"
 

Function Documentation

def heppy_loop.callBack (   result)

Definition at line 28 of file heppy_loop.py.

28 
29 def callBack( result ):
30  pass
31  print 'production done:', str(result)
heppy_loop.callBack
def callBack
Definition: heppy_loop.py:28
def heppy_loop.chunks (   l,
  n 
)

Definition at line 93 of file heppy_loop.py.

Referenced by split().

93 
94 def chunks(l, n):
95  return [l[i:i+n] for i in range(0, len(l), n)]
heppy_loop.chunks
def chunks
Definition: heppy_loop.py:93
def heppy_loop.createOutputDir (   dir,
  components,
  force 
) 
Creates the output dir, dealing with the case where dir exists.

Definition at line 66 of file heppy_loop.py.

References join().

Referenced by main().

66 
67 def createOutputDir(dir, components, force):
68  '''Creates the output dir, dealing with the case where dir exists.'''
69  answer = None
70  try:
71  os.mkdir(dir)
72  return True
73  except OSError:
74  print 'directory %s already exists' % dir
75  print 'contents: '
76  dirlist = [path for path in os.listdir(dir) if os.path.isdir( '/'.join([dir, path]) )]
77  pprint( dirlist )
78  print 'component list: '
79  print [comp.name for comp in components]
80  if force is True:
81  print 'force mode, continue.'
82  return True
83  else:
84  while answer not in ['Y','y','yes','N','n','no']:
85  answer = raw_input('Continue? [y/n]')
86  if answer.lower().startswith('n'):
87  return False
88  elif answer.lower().startswith('y'):
89  return True
90  else:
91  raise ValueError( ' '.join(['answer can not have this value!',
92  answer]) )
heppy_loop.createOutputDir
def createOutputDir
Definition: heppy_loop.py:66
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def heppy_loop.getHeppyOption (   name,
  default = None 
)

Definition at line 127 of file heppy_loop.py.

127 
128 def getHeppyOption(name,default=None):
129  global _heppyGlobalOptions
return _heppyGlobalOptions[name] if name in _heppyGlobalOptions else default
heppy_loop.getHeppyOption
def getHeppyOption
Definition: heppy_loop.py:127
def heppy_loop.main (   options,
  args,
  parser 
)

Definition at line 134 of file heppy_loop.py.

References createOutputDir(), min(), runLoop(), and split().

134 
135 def main( options, args, parser ):
136 
137  if len(args) != 2:
138  parser.print_help()
139  print 'ERROR: please provide the processing name and the component list'
140  sys.exit(1)
141 
142  outDir = args[0]
143  if os.path.exists(outDir) and not os.path.isdir( outDir ):
144  parser.print_help()
145  print 'ERROR: when it exists, first argument must be a directory.'
146  sys.exit(2)
147  cfgFileName = args[1]
148  if not os.path.isfile( cfgFileName ):
149  parser.print_help()
150  print 'ERROR: second argument must be an existing file (your input cfg).'
151  sys.exit(3)
152 
153  if options.verbose:
154  import logging
155  logging.basicConfig(level=logging.INFO)
156 
157  # Propagate global options to _heppyGlobalOptions within this module
158  # I have to import it explicitly, 'global' does not work since the
159  # module is not set when executing the main
160  from PhysicsTools.HeppyCore.framework.heppy_loop import _heppyGlobalOptions
161  for opt in options.extraOptions:
162  if "=" in opt:
163  (key,val) = opt.split("=",1)
164  _heppyGlobalOptions[key] = val
165  else:
166  _heppyGlobalOptions[opt] = True
167 
168  file = open( cfgFileName, 'r' )
169  cfg = imp.load_source( 'PhysicsTools.HeppyCore.__cfg_to_run__', cfgFileName, file)
170 
171  selComps = [comp for comp in cfg.config.components if len(comp.files)>0]
172  selComps = split(selComps)
173  # for comp in selComps:
174  # print comp
175  if len(selComps)>options.ntasks:
176  print "WARNING: too many threads {tnum}, will just use a maximum of {jnum}.".format(tnum=len(selComps),jnum=options.ntasks)
177  if not createOutputDir(outDir, selComps, options.force):
178  print 'exiting'
179  sys.exit(0)
180  if len(selComps)>1:
181  shutil.copy( cfgFileName, outDir )
182  pool = Pool(processes=min(len(selComps),options.ntasks))
183  ## workaround for a scoping problem in ipython+multiprocessing
184  import PhysicsTools.HeppyCore.framework.heppy_loop as ML
185  for comp in selComps:
186  print 'submitting', comp.name
187  pool.apply_async( ML.runLoopAsync, [comp, outDir, 'PhysicsTools.HeppyCore.__cfg_to_run__', options],
188  callback=ML.callBack)
189  pool.close()
190  pool.join()
191  else:
192  # when running only one loop, do not use multiprocessor module.
193  # then, the exceptions are visible -> use only one sample for testing
194  global loop
195  loop = runLoop( comp, outDir, cfg.config, options )
196  return loop
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:44
heppy_loop.createOutputDir
def createOutputDir
Definition: heppy_loop.py:66
min
T min(T a, T b)
Definition: MathUtil.h:58
heppy_loop.main
def main
Definition: heppy_loop.py:134
heppy_loop.split
def split
Definition: heppy_loop.py:96
def heppy_loop.runLoop (   comp,
  outDir,
  config,
  options 
)

Definition at line 44 of file heppy_loop.py.

References join().

Referenced by main(), and runLoopAsync().

44 
45 def runLoop( comp, outDir, config, options):
46  fullName = '/'.join( [outDir, comp.name ] )
47  # import pdb; pdb.set_trace()
48  config.components = [comp]
49  loop = Looper( fullName,
50  config,
51  options.nevents, 0,
52  nPrint = options.nprint,
53  timeReport = options.timeReport,
54  quiet = options.quiet)
55  # print loop
56  if options.iEvent is None:
57  loop.loop()
58  loop.write()
59  # print loop
60  else:
61  # loop.InitOutput()
62  iEvent = int(options.iEvent)
63  loop.process( iEvent )
64  return loop
65 
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:44
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def heppy_loop.runLoopAsync (   comp,
  outDir,
  configName,
  options 
)

Definition at line 32 of file heppy_loop.py.

References runLoop().

32 
33 def runLoopAsync(comp, outDir, configName, options):
34  try:
35  loop = runLoop( comp, outDir, copy.copy(sys.modules[configName].config), options)
36  return loop.name
37  except Exception:
38  import traceback
39  print "ERROR processing component %s" % comp.name
40  print comp
41  print "STACK TRACE: "
42  print traceback.format_exc()
43  raise
heppy_loop.runLoop
def runLoop
Definition: heppy_loop.py:44
heppy_loop.runLoopAsync
def runLoopAsync
Definition: heppy_loop.py:32
def heppy_loop.setHeppyOption (   name,
  value = True 
)

Definition at line 130 of file heppy_loop.py.

130 
131 def setHeppyOption(name,value=True):
132  global _heppyGlobalOptions
133  _heppyGlobalOptions[name] = value
heppy_loop.setHeppyOption
def setHeppyOption
Definition: heppy_loop.py:130
def heppy_loop.split (   comps)

Definition at line 96 of file heppy_loop.py.

References chunks().

Referenced by main().

96 
97 def split(comps):
98  # import pdb; pdb.set_trace()
99  splitComps = []
100  for comp in comps:
101  if hasattr( comp, 'fineSplitFactor') and comp.fineSplitFactor>1:
102  subchunks = range(comp.fineSplitFactor)
103  for ichunk, chunk in enumerate([(f,i) for f in comp.files for i in subchunks]):
104  newComp = copy.deepcopy(comp)
105  newComp.files = [chunk[0]]
106  newComp.fineSplit = ( chunk[1], comp.fineSplitFactor )
107  newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
108  index=ichunk)
109  splitComps.append( newComp )
110  elif hasattr( comp, 'splitFactor') and comp.splitFactor>1:
111  chunkSize = len(comp.files) / comp.splitFactor
112  if len(comp.files) % comp.splitFactor:
113  chunkSize += 1
114  # print 'chunk size',chunkSize, len(comp.files), comp.splitFactor
115  for ichunk, chunk in enumerate( chunks( comp.files, chunkSize)):
116  newComp = copy.deepcopy(comp)
117  newComp.files = chunk
118  newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
119  index=ichunk)
120  splitComps.append( newComp )
121  else:
122  splitComps.append( comp )
123  return splitComps
124 
heppy_loop.chunks
def chunks
Definition: heppy_loop.py:93
heppy_loop.split
def split
Definition: heppy_loop.py:96

Variable Documentation

dictionary heppy_loop._heppyGlobalOptions = {}

Definition at line 125 of file heppy_loop.py.

string heppy_loop.action = 'store_true'

Definition at line 27 of file heppy_loop.py.

int heppy_loop.default = None)

Definition at line 16 of file heppy_loop.py.

string heppy_loop.dest = "nevents"

Definition at line 13 of file heppy_loop.py.

string heppy_loop.help = "number of events to process"

Definition at line 15 of file heppy_loop.py.

tuple heppy_loop.loop = None

Definition at line 26 of file heppy_loop.py.

list heppy_loop.oldv = sys.argv[:]

Definition at line 16 of file heppy_loop.py.

tuple heppy_loop.parser = OptionParser()

Definition at line 5 of file heppy_loop.py.

string heppy_loop.type = "int"

Definition at line 14 of file heppy_loop.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5