heppy_loop Namespace Reference

Functions
def	callBack (result)
def	chunks (l, n)
def	createOutputDir (dir, components, force)
def	getHeppyOption (name, default=None)
def	main (options, args, parser)
def	runLoop (comp, outDir, config, options)
def	runLoopAsync (comp, outDir, configName, options)
def	setHeppyOption (name, value=True)
def	split (comps)

Variables
	_heppyGlobalOptions
	action
	args
	argv
	default
	dest
	help
	loop
	oldv
	options
	parser
	type
	usage

Function Documentation

def heppy_loop.callBack

(

result

)

Definition at line 30 of file heppy_loop.py.

References edm.print(), and str.

def callBack( result ):
    pass
    print('production done:', str(result))

def heppy_loop.chunks	(		l,
			n
	)

Definition at line 95 of file heppy_loop.py.

Referenced by split().

def chunks(l, n):
    return [l[i:i+n] for i in range(0, len(l), n)]

def heppy_loop.createOutputDir	(		dir,
			components,
			force
	)

Creates the output dir, dealing with the case where dir exists.

Definition at line 68 of file heppy_loop.py.

References join(), and edm.print().

Referenced by main().

def createOutputDir(dir, components, force):
    '''Creates the output dir, dealing with the case where dir exists.'''
    answer = None
    try:
        os.mkdir(dir)
        return True
    except OSError:
        print('directory %s already exists' % dir)
        print('contents: ')
        dirlist = [path for path in os.listdir(dir) if os.path.isdir( '/'.join([dir, path]) )]
        pprint( dirlist )
        print('component list: ')
        print([comp.name for comp in components])
        if force is True:
            print('force mode, continue.')
            return True
        else:
            while answer not in ['Y','y','yes','N','n','no']:
                answer = raw_input('Continue? [y/n]')
            if answer.lower().startswith('n'):
                return False
            elif answer.lower().startswith('y'):
                return True
            else:
                raise ValueError( ' '.join(['answer can not have this value!',
                                            answer]) )

def heppy_loop.getHeppyOption	(		name,
			default = None
	)

Definition at line 129 of file heppy_loop.py.

def getHeppyOption(name,default=None):
    global _heppyGlobalOptions
    return _heppyGlobalOptions[name] if name in _heppyGlobalOptions else default

def heppy_loop.main	(		options,
			args,
			parser
	)

Definition at line 136 of file heppy_loop.py.

References createOutputDir(), min(), edm.print(), runLoop(), and split().

def main( options, args, parser ):

    if len(args) != 2:
        parser.print_help()
        print('ERROR: please provide the processing name and the component list')
        sys.exit(1)

    outDir = args[0]
    if os.path.exists(outDir) and not os.path.isdir( outDir ):
        parser.print_help()
        print('ERROR: when it exists, first argument must be a directory.')
        sys.exit(2)
    cfgFileName = args[1]
    if not os.path.isfile( cfgFileName ):
        parser.print_help()
        print('ERROR: second argument must be an existing file (your input cfg).')
        sys.exit(3)

    if options.verbose:
        import logging
        logging.basicConfig(level=logging.INFO)

    # Propagate global options to _heppyGlobalOptions within this module
    # I have to import it explicitly, 'global' does not work since the
    # module is not set when executing the main
    from PhysicsTools.HeppyCore.framework.heppy_loop import _heppyGlobalOptions
    for opt in options.extraOptions:
        if "=" in opt:
            (key,val) = opt.split("=",1)
            _heppyGlobalOptions[key] = val
        else:
            _heppyGlobalOptions[opt] = True

    file = open( cfgFileName, 'r' )
    cfg = imp.load_source( 'PhysicsTools.HeppyCore.__cfg_to_run__', cfgFileName, file)

    selComps = [comp for comp in cfg.config.components if len(comp.files)>0]
    selComps = split(selComps)
    # for comp in selComps:
    #    print comp
    if len(selComps)>options.ntasks:
        print("WARNING: too many threads {tnum}, will just use a maximum of {jnum}.".format(tnum=len(selComps),jnum=options.ntasks))
    if not createOutputDir(outDir, selComps, options.force):
        print('exiting')
        sys.exit(0)
    if len(selComps)>1:
        shutil.copy( cfgFileName, outDir )
        pool = Pool(processes=min(len(selComps),options.ntasks))
        ## workaround for a scoping problem in ipython+multiprocessing
        import PhysicsTools.HeppyCore.framework.heppy_loop as ML 
        for comp in selComps:
            print('submitting', comp.name)
            pool.apply_async( ML.runLoopAsync, [comp, outDir, 'PhysicsTools.HeppyCore.__cfg_to_run__', options],
                              callback=ML.callBack)
        pool.close()
        pool.join()
    else:
        # when running only one loop, do not use multiprocessor module.
        # then, the exceptions are visible -> use only one sample for testing
        global loop
        loop = runLoop( comp, outDir, cfg.config, options )
    return loop

def heppy_loop.runLoop	(		comp,
			outDir,
			config,
			options
	)

Definition at line 46 of file heppy_loop.py.

References createfilelist.int, and join().

Referenced by main(), and runLoopAsync().

def runLoop( comp, outDir, config, options):
    fullName = '/'.join( [outDir, comp.name ] )
    # import pdb; pdb.set_trace()
    config.components = [comp]
    loop = Looper( fullName,
                   config,
                   options.nevents, 0,
                   nPrint = options.nprint,
                   timeReport = options.timeReport,
                   quiet = options.quiet)
    # print loop
    if options.iEvent is None:
        loop.loop()
        loop.write()
        # print loop
    else:
        # loop.InitOutput()
        iEvent = int(options.iEvent)
        loop.process( iEvent )
    return loop

def heppy_loop.runLoopAsync	(		comp,
			outDir,
			configName,
			options
	)

Definition at line 34 of file heppy_loop.py.

References edm.print(), and runLoop().

def runLoopAsync(comp, outDir, configName, options):
    try:
        loop = runLoop( comp, outDir, copy.copy(sys.modules[configName].config), options)
        return loop.name
    except Exception:
        import traceback
        print("ERROR processing component %s" % comp.name)
        print(comp)
        print("STACK TRACE: ")
        print(traceback.format_exc())
        raise

def heppy_loop.setHeppyOption	(		name,
			value = True
	)

Definition at line 132 of file heppy_loop.py.

def setHeppyOption(name,value=True):
    global _heppyGlobalOptions
    _heppyGlobalOptions[name] = value

def heppy_loop.split

(

comps

)

Definition at line 98 of file heppy_loop.py.

References chunks(), and list().

Referenced by main().

def split(comps):
    # import pdb; pdb.set_trace()
    splitComps = []
    for comp in comps:
        if hasattr( comp, 'fineSplitFactor') and comp.fineSplitFactor>1:
            subchunks = list(range(comp.fineSplitFactor))
            for ichunk, chunk in enumerate([(f,i) for f in comp.files for i in subchunks]):
                newComp = copy.deepcopy(comp)
                newComp.files = [chunk[0]]
                newComp.fineSplit = ( chunk[1], comp.fineSplitFactor )
                newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
                                                       index=ichunk)
                splitComps.append( newComp )
        elif hasattr( comp, 'splitFactor') and comp.splitFactor>1:
            chunkSize = len(comp.files) / comp.splitFactor
            if len(comp.files) % comp.splitFactor:
                chunkSize += 1
            # print 'chunk size',chunkSize, len(comp.files), comp.splitFactor
            for ichunk, chunk in enumerate( chunks( comp.files, chunkSize)):
                newComp = copy.deepcopy(comp)
                newComp.files = chunk
                newComp.name = '{name}_Chunk{index}'.format(name=newComp.name,
                                                       index=ichunk)
                splitComps.append( newComp )
        else:
            splitComps.append( comp )
    return splitComps

Variable Documentation

heppy_loop._heppyGlobalOptions

private

Definition at line 127 of file heppy_loop.py.

heppy_loop.action

Definition at line 27 of file heppy_loop.py.

heppy_loop.args

Definition at line 62 of file heppy_loop.py.

heppy_loop.argv

Definition at line 19 of file heppy_loop.py.

heppy_loop.default

Definition at line 16 of file heppy_loop.py.

heppy_loop.dest

Definition at line 13 of file heppy_loop.py.

heppy_loop.help

Definition at line 15 of file heppy_loop.py.

heppy_loop.loop

Definition at line 28 of file heppy_loop.py.

Referenced by optutl::CommandLineParser._getSectionFiles(), FWFFLooper.autoLoadNewEvent(), CACell.checkAlignmentAndAct(), FWFFService.checkPosition(), FWFFLooper.checkPosition(), RawDataConverter.ClearData(), CmsShowMainFrame.CmsShowMainFrame(), pat::EventHypothesis.count(), DDCheckMaterial(), PhysicsTools::MVAComputer::DerivContext.DerivContext(), SiStripTFile.dirContent(), MillePedeAlignmentAlgorithm.doIO(), LaserAlignment.DumpPosFileSet(), LaserAlignment.DumpStripFileSet(), SETSeedFinder.estimateMomentum(), PhysicsTools::MVAComputer.evalInternal(), LMFDefFabric.getColor(), LMFDefFabric.getColorFromID(), RawDataConverter.GetDigis(), LMFDefFabric.getRunTag(), LMFDefFabric.getRunTagFromID(), LMFDefFabric.getTrigType(), LMFDefFabric.getTrigTypeFromID(), output(), optutl::CommandLineParser.parseArguments(), EcalDigiSelector.produce(), SiStripTFile.readDQMFormat(), TrajectoryManager.reconstruct(), stdcomb.recursive_combination(), LMFColoredTable.setColor(), LMFColoredTable.setSystem(), HcalTestAnalysis.update(), DDI::Polyhedra.volume(), and DDI::Polycone.volume().

heppy_loop.oldv

Definition at line 18 of file heppy_loop.py.

heppy_loop.options

Definition at line 62 of file heppy_loop.py.

heppy_loop.parser

Definition at line 5 of file heppy_loop.py.

heppy_loop.type

Definition at line 14 of file heppy_loop.py.

heppy_loop.usage

Definition at line 6 of file heppy_loop.py.

---

Generated for CMSSW Reference Manual by  1.8.11