Functions
def	callBack (result)

def	chunks (l, n)

def	createOutputDir (dir, components, force)

def	getHeppyOption (name, default=None)

def	main (options, args, parser)

def	runLoop (comp, outDir, config, options)

def	runLoopAsync (comp, outDir, configName, options)

def	setHeppyOption (name, value=True)

def	split (comps)

Variables
	_heppyGlobalOptions

	action

	args

	argv

	default

	dest

	help

	loop

	oldv

	options

	parser

	type

	usage

Function Documentation

◆ callBack()

def heppy_loop.callBack ( result )

Definition at line 30 of file heppy_loop.py.

References print(), and str.

Referenced by cond::persistency::CoralMsgReporter.subscribe().

 def callBack( result ):
     pass
     print('production done:', str(result))
 

◆ chunks()

def heppy_loop.chunks	(	l,
		n
	)

Definition at line 95 of file heppy_loop.py.

References FastTimerService_cff.range.

Referenced by split().

 def chunks(l, n):
     return [l[i:i+n] for i in range(0, len(l), n)]
 

◆ createOutputDir()

def heppy_loop.createOutputDir	(	dir,
		components,
		force
	)

Creates the output dir, dealing with the case where dir exists.

Definition at line 68 of file heppy_loop.py.

References join(), and print().

Referenced by main().

 def createOutputDir(dir, components, force):
     '''Creates the output dir, dealing with the case where dir exists.'''
     answer = None
     try:
         os.mkdir(dir)
         return True
     except OSError:
         print('directory %s already exists' % dir)
         print('contents: ')
         dirlist = [path for path in os.listdir(dir) if os.path.isdir( '/'.join([dir, path]) )]
         pprint( dirlist )
         print('component list: ')
         print([comp.name for comp in components])
         if force is True:
             print('force mode, continue.')
             return True
         else:
             while answer not in ['Y','y','yes','N','n','no']:
                 answer = raw_input('Continue? [y/n]')
             if answer.lower().startswith('n'):
                 return False
             elif answer.lower().startswith('y'):
                 return True
             else:
                 raise ValueError( ' '.join(['answer can not have this value!',
                                             answer]) )
 

◆ getHeppyOption()

def heppy_loop.getHeppyOption	(	name,
		default = `None`
	)

Definition at line 129 of file heppy_loop.py.

 def getHeppyOption(name,default=None):
     global _heppyGlobalOptions
     return _heppyGlobalOptions[name] if name in _heppyGlobalOptions else default

◆ main()

def heppy_loop.main	(	options,
		args,
		parser
	)

Definition at line 136 of file heppy_loop.py.

References createOutputDir(), SiStripPI.min, print(), runLoop(), and split().

 def main( options, args, parser ):
 
     if len(args) != 2:
         parser.print_help()
         print('ERROR: please provide the processing name and the component list')
         sys.exit(1)
 
     outDir = args[0]
     if os.path.exists(outDir) and not os.path.isdir( outDir ):
         parser.print_help()
         print('ERROR: when it exists, first argument must be a directory.')
         sys.exit(2)
     cfgFileName = args[1]
     if not os.path.isfile( cfgFileName ):
         parser.print_help()
         print('ERROR: second argument must be an existing file (your input cfg).')
         sys.exit(3)
 
     if options.verbose:
         import logging
         logging.basicConfig(level=logging.INFO)
 
     # Propagate global options to _heppyGlobalOptions within this module
     # I have to import it explicitly, 'global' does not work since the
     # module is not set when executing the main
     from PhysicsTools.HeppyCore.framework.heppy_loop import _heppyGlobalOptions
     for opt in options.extraOptions:
         if "=" in opt:
             (key,val) = opt.split("=",1)
             _heppyGlobalOptions[key] = val
         else:
             _heppyGlobalOptions[opt] = True
 
     file = open( cfgFileName, 'r' )
     cfg = imp.load_source( 'PhysicsTools.HeppyCore.__cfg_to_run__', cfgFileName, file)
 
     selComps = [comp for comp in cfg.config.components if len(comp.files)>0]
     selComps = split(selComps)
     # for comp in selComps:
     #    print comp
     if len(selComps)>options.ntasks:
         print("WARNING: too many threads {tnum}, will just use a maximum of {jnum}.".format(tnum=len(selComps),jnum=options.ntasks))
     if not createOutputDir(outDir, selComps, options.force):
         print('exiting')
         sys.exit(0)
     if len(selComps)>1:
         shutil.copy( cfgFileName, outDir )
         pool = Pool(processes=min(len(selComps),options.ntasks))
         
         import PhysicsTools.HeppyCore.framework.heppy_loop as ML 
         for comp in selComps:
             print('submitting', comp.name)
             pool.apply_async( ML.runLoopAsync, [comp, outDir, 'PhysicsTools.HeppyCore.__cfg_to_run__', options],
                               callback=ML.callBack)
         pool.close()
         pool.join()
     else:
         # when running only one loop, do not use multiprocessor module.
         # then, the exceptions are visible -> use only one sample for testing
         global loop
         loop = runLoop( comp, outDir, cfg.config, options )
     return loop

◆ runLoop()

def heppy_loop.runLoop	(	comp,
		outDir,
		config,
		options
	)

Definition at line 46 of file heppy_loop.py.

References createfilelist.int, and join().

Referenced by main(), and runLoopAsync().

 def runLoop( comp, outDir, config, options):
     fullName = '/'.join( [outDir, comp.name ] )
     # import pdb; pdb.set_trace()
     config.components = [comp]
     loop = Looper( fullName,
                    config,
                    options.nevents, 0,
                    nPrint = options.nprint,
                    timeReport = options.timeReport,
                    quiet = options.quiet)
     # print loop
     if options.iEvent is None:
         loop.loop()
         loop.write()
         # print loop
     else:
         # loop.InitOutput()
         iEvent = int(options.iEvent)
         loop.process( iEvent )
     return loop
 
 

◆ runLoopAsync()

def heppy_loop.runLoopAsync	(	comp,
		outDir,
		configName,
		options
	)

Definition at line 34 of file heppy_loop.py.

References print(), and runLoop().

 def runLoopAsync(comp, outDir, configName, options):
     try:
         loop = runLoop( comp, outDir, copy.copy(sys.modules[configName].config), options)
         return loop.name
     except Exception:
         import traceback
         print("ERROR processing component %s" % comp.name)
         print(comp)
         print("STACK TRACE: ")
         print(traceback.format_exc())
         raise
 

◆ setHeppyOption()

def heppy_loop.setHeppyOption	(	name,
		value = `True`
	)

Definition at line 132 of file heppy_loop.py.

 def setHeppyOption(name,value=True):
     global _heppyGlobalOptions
     _heppyGlobalOptions[name] = value
 

◆ split()

def heppy_loop.split ( comps )

Definition at line 98 of file heppy_loop.py.

References chunks(), and FastTimerService_cff.range.

Referenced by main().

 def split(comps):
     # import pdb; pdb.set_trace()
     splitComps = []
     for comp in comps:
         if hasattr( comp, 'fineSplitFactor') and comp.fineSplitFactor>1:
             subchunks = list(range(comp.fineSplitFactor))
             for ichunk, chunk in enumerate([(f,i) for f in comp.files for i in subchunks]):
                 newComp = copy.deepcopy(comp)
                 newComp.files = [chunk[0]]
                 newComp.fineSplit = ( chunk[1], comp.fineSplitFactor )
                 newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
                                                        index=ichunk)
                 splitComps.append( newComp )
         elif hasattr( comp, 'splitFactor') and comp.splitFactor>1:
             chunkSize = len(comp.files) / comp.splitFactor
             if len(comp.files) % comp.splitFactor:
                 chunkSize += 1
             # print 'chunk size',chunkSize, len(comp.files), comp.splitFactor
             for ichunk, chunk in enumerate( chunks( comp.files, chunkSize)):
                 newComp = copy.deepcopy(comp)
                 newComp.files = chunk
                 newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
                                                        index=ichunk)
                 splitComps.append( newComp )
         else:
             splitComps.append( comp )
     return splitComps
 
 

Variable Documentation

◆ _heppyGlobalOptions

heppy_loop._heppyGlobalOptions

private

Definition at line 127 of file heppy_loop.py.

◆ action

heppy_loop.action

Definition at line 27 of file heppy_loop.py.

◆ args

heppy_loop.args

Definition at line 62 of file heppy_loop.py.

◆ argv

heppy_loop.argv

Definition at line 19 of file heppy_loop.py.

◆ default

heppy_loop.default

Definition at line 16 of file heppy_loop.py.

◆ dest

heppy_loop.dest

Definition at line 13 of file heppy_loop.py.

◆ help

heppy_loop.help

Definition at line 15 of file heppy_loop.py.

◆ loop

heppy_loop.loop

Definition at line 28 of file heppy_loop.py.

Referenced by optutl::CommandLineParser._getSectionFiles(), CACell.checkAlignmentAndAct(), HGCDoublet.checkCompatibilityAndTag(), RawDataConverter.ClearData(), CmsShowMainFrame.CmsShowMainFrame(), DDCheckMaterial(), SiStripTFile.dirContent(), MillePedeAlignmentAlgorithm.doIO(), LaserAlignment.DumpPosFileSet(), LaserAlignment.DumpStripFileSet(), SETSeedFinder.estimateMomentum(), PhysicsTools::MVAComputer::EvalContext.eval(), PhysicsTools::MVAComputer::DerivContext.eval(), PhysicsTools::MVAComputer.evalInternal(), LMFDefFabric.getColor(), LMFDefFabric.getColorFromID(), RawDataConverter.GetDigis(), LMFDefFabric.getRunTag(), LMFDefFabric.getRunTagFromID(), LMFDefFabric.getTrigType(), LMFDefFabric.getTrigTypeFromID(), cms::alpakatools::elements_with_stride_nd< TAcc, typename >::iterator.iterator(), pat::EventHypothesis.loop(), output(), optutl::CommandLineParser.parseArguments(), Pi0CalibXMLwriter.Pi0CalibXMLwriter(), EcalDigiSelector.produce(), SiStripTFile.readDQMFormat(), TrajectoryManager.reconstruct(), stdcomb.recursive_combination(), LMFColoredTable.setColor(), LMFColoredTable.setSystem(), HcalTestAnalysis.update(), DDI::Polycone.volume(), DDI::Polyhedra.volume(), and gpuVertexFinder.while().

◆ oldv

heppy_loop.oldv

Definition at line 18 of file heppy_loop.py.

◆ options

heppy_loop.options

Definition at line 62 of file heppy_loop.py.

◆ parser

heppy_loop.parser

Definition at line 5 of file heppy_loop.py.

◆ type

heppy_loop.type

Definition at line 14 of file heppy_loop.py.

◆ usage

heppy_loop.usage

Definition at line 6 of file heppy_loop.py.

Functions

Variables