Classes
class	Activity

class	AllocInfo

class	DummyFile

class	EDModuleTransitionParser

class	ESModuleTransitionParser

class	FrameworkTransitionParser

class	ModuleAllocCompactFileParser

class	ModuleCentricContainers

class	ModuleCentricModuleData

class	ModuleCentricVisualizationContainers

class	ModuleData

class	Phase

class	PostEDModuleAcquireParser

class	PostEDModuleEventDelayedGetParser

class	PostEDModuleTransitionParser

class	PostESModuleAcquireParser

class	PostESModuleTransitionParser

class	PostEventReadFromSourceParser

class	PostFrameworkTransitionParser

class	PostSourceTransitionParser

class	PreEDModuleAcquireParser

class	PreEDModuleEventDelayedGetParser

class	PreEDModuleTransitionParser

class	PreESModuleAcquireParser

class	PreESModuleTransitionParser

class	PreEventReadFromSourceParser

class	PreFrameworkTransitionParser

class	PreSourceTransitionParser

class	SourceTransitionParser

class	SyncValues

class	TempModuleTransitionInfos

class	TemporalModuleData

class	TestModuleCommand

class	VisualizationContainers

Functions
def	activityName (activity)

def	findMatchingTransition (sync, containers)

def	jsonInfo (parser, temporal)

def	jsonModuleTransition (type, id, modID, callID, activity, start, finish=0)

def	jsonTransition (type, id, sync, start, finish, isSrc=False)

def	jsonVisualizationInfo (parser)

def	lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, moduleCentric)

def	printHelp ()

def	processingStepsFromFile (f, moduleNames, esModuleNames, recordNames, moduleCentric)

def	runTests ()

def	sortByAttribute (parser, attribute)

def	startTime (x)

def	textOutput (parser)

def	textPrefix_ (time, indentLevel)

def	transitionIndentLevel (transition)

def	transitionIsGlobal (transition)

def	transitionName (transition)

Function Documentation

◆ activityName()

def edmModuleAllocMonitorAnalyze.activityName ( activity )

Definition at line 99 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.ModuleData.toSimpleDict().

 def activityName(activity):
     return activityNames_[activity]
   

◆ findMatchingTransition()

def edmModuleAllocMonitorAnalyze.findMatchingTransition	(	sync,
		containers
	)

Definition at line 412 of file edmModuleAllocMonitorAnalyze.py.

References print(), and FastTimerService_cff.range.

Referenced by edmModuleAllocMonitorAnalyze.PreFrameworkTransitionParser.jsonVisInfo(), and edmModuleAllocMonitorAnalyze.PostFrameworkTransitionParser.jsonVisInfo().

 def findMatchingTransition(sync, containers):
     for i in range(len(containers)):
         if containers[i][-1]["sync"] == sync:
             return i
     #need more exhausting search
     for i in range(len(containers)):
         for t in containers[i]:
             if t["sync"] == sync:
                 return i
 
     print("find failed",sync, containers)
     return None
 

◆ jsonInfo()

def edmModuleAllocMonitorAnalyze.jsonInfo	(	parser,
		temporal
	)

Definition at line 1198 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.TestModuleCommand.testJson(), and edmModuleAllocMonitorAnalyze.TestModuleCommand.testJsonTemporal().

 def jsonInfo(parser, temporal):
     sync = SyncValues()
     #used to keep track of outstanding module transitions
     temp = TempModuleTransitionInfos()
     if temporal:
         data = TemporalModuleData()
     else:
         data = ModuleCentricModuleData()
     for p in parser.processingSteps():
         if hasattr(p, "jsonInfo"):
             p.jsonInfo(sync,temp, data)
     return data
 

◆ jsonModuleTransition()

def edmModuleAllocMonitorAnalyze.jsonModuleTransition	(	type,
		id,
		modID,
		callID,
		activity,
		start,
		finish = `0`
	)

Definition at line 1191 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.EDModuleTransitionParser._preJsonVis(), edmModuleAllocMonitorAnalyze.ESModuleTransitionParser._preJsonVis(), and edmModuleAllocMonitorAnalyze.PostEDModuleAcquireParser.jsonVisInfo().

 def jsonModuleTransition(type, id, modID, callID, activity, start, finish=0):
     return {"type": type, "id": id, "mod": modID, "call": callID, "act": activity, "start": start*kMicroToSec, "finish": finish*kMicroToSec}

◆ jsonTransition()

def edmModuleAllocMonitorAnalyze.jsonTransition	(	type,
		id,
		sync,
		start,
		finish,
		isSrc = `False`
	)

Definition at line 1188 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.PreFrameworkTransitionParser.jsonVisInfo().

 def jsonTransition(type, id, sync, start, finish, isSrc=False):
     return {"type": type, "id": id, "sync": sync, "start": start*kMicroToSec, "finish": finish*kMicroToSec, "isSrc":isSrc}
 

◆ jsonVisualizationInfo()

def edmModuleAllocMonitorAnalyze.jsonVisualizationInfo ( parser )

Definition at line 1221 of file edmModuleAllocMonitorAnalyze.py.

References mps_setup.append.

Referenced by edmModuleAllocMonitorAnalyze.TestModuleCommand.testFullVisualization(), and edmModuleAllocMonitorAnalyze.TestModuleCommand.testModuleCentricVisualization().

 def jsonVisualizationInfo(parser):
     if parser._moduleCentric:
         data = ModuleCentricContainers()
     else:
         data = VisualizationContainers()
     for p in parser.processingSteps():
         p.jsonVisInfo( data)
     #make sure everything is sorted
     for g in data.allGlobals():
         g.sort(key=startTime)
     final = {"transitions" : [] , "modules": [], "esModules": []}
     final["transitions"].append({ "name":"Global", "slots": []})
     globals = final["transitions"][-1]["slots"]
     for i, g in enumerate(data.allGlobals()):
         globals.append(g)
         if not parser._moduleCentric:
             if len(data._modGlobals) < i+1:
                 break
             for mod in data._modGlobals[i]:
                 globals.append(mod)
     for i,s in enumerate(data.allStreams()):
         final["transitions"].append({"name": f"Stream {i}", "slots":[]})
         stream = final["transitions"][-1]["slots"]
         stream.append(s)
         if not parser._moduleCentric:
             for mod in data._modStreams[i]:
                 stream.append(mod)
     if parser._moduleCentric:
         sourceSlot = data._modules[data._moduleID2Index(0)]
         modules = []
         for i,m in parser._moduleNames.items():
             modules.append({"name": f"{m}", "slots":[]})
             slots = modules[-1]["slots"]
             foundSlots = data._modules[data._moduleID2Index(i)]
             time = 0
             for s in foundSlots:
                 slots.append(s)
                 for t in s:
                     if t["act"] !=Activity.prefetch:
                         time += t["finish"]-t["start"]
             modules[-1]['time']=time
         for i,m in parser._esModuleNames.items():
             modules.append({"name": f"{m}", "slots":[]})
             slots = modules[-1]["slots"]
             foundSlots = data._modules[data._moduleID2Index(-1*i)]
             time = 0
             for s in foundSlots:
                 slots.append(s)
                 for t in s:
                     if t["act"] !=Activity.prefetch:
                         time += t["finish"]-t["start"]
             modules[-1]['time']=time
         modules.sort(key= lambda x : x['time'], reverse=True)
         final['transitions'].append({"name": "source", "slots":sourceSlot})
         for m in modules:
             final['transitions'].append(m)
 
     max = 0
     for k in parser._moduleNames.keys():
         if k > max:
             max = k
         
     final["modules"] =['']*(max+1)
     final["modules"][0] = 'source'
     for k,v in parser._moduleNames.items():
         final["modules"][k]=v
         
     max = 0
     for k in parser._esModuleNames.keys():
         if k > max:
             max = k
     final["esModules"] = ['']*(max+1)
     for k,v in parser._esModuleNames.items():
         final["esModules"][k] = v
     return final
 
 
 
 #=======================================

◆ lineParserFactory()

def edmModuleAllocMonitorAnalyze.lineParserFactory	(	step,
		payload,
		moduleNames,
		esModuleNames,
		recordNames,
		moduleCentric
	)

Definition at line 893 of file edmModuleAllocMonitorAnalyze.py.

References edm::errors.LogicError.

Referenced by processingStepsFromFile().

 def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, moduleCentric):
     if step == 'F':
         parser = PreFrameworkTransitionParser(payload)
         return parser
     if step == 'f':
         return PostFrameworkTransitionParser(payload)
     if step == 'S':
         return PreSourceTransitionParser(payload, moduleCentric)
     if step == 's':
         return PostSourceTransitionParser(payload, moduleCentric)
     if step == 'M':
         return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
     if step == 'm':
         return PostEDModuleTransitionParser(payload, moduleNames)
     if step == 'A':
         return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
     if step == 'a':
         return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
     if step == 'D':
         return PreEDModuleEventDelayedGetParser(payload, moduleNames)
     if step == 'd':
         return PostEDModuleEventDelayedGetParser(payload, moduleNames)
     if step == 'R':        return PreEventReadFromSourceParser(payload, moduleNames)
     if step == 'r':
         return PostEventReadFromSourceParser(payload, moduleNames)
     if step == 'N':
         return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
     if step == 'n':
         return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
     if step == 'B':
         return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
     if step == 'b':
         return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
     raise LogicError("Unknown step '{}'".format(step))
     
 #----------------------------------------------

◆ printHelp()

def edmModuleAllocMonitorAnalyze.printHelp ( )

Definition at line 9 of file edmModuleAllocMonitorAnalyze.py.

 def printHelp():
     s = '''
 To Use: Add the ModuleAllocMonitor Service to the cmsRun job use something like this
   in the configuration:
 
   process.add_(cms.Service("ModuleAllocMonitor", fileName = cms.untracked.string("moduleAlloc.log")))
 
   After running the job, execute this script and pass the name of the
   ModuleAllocMonitor log file to the script.
 
   This script will output a more human readable form of the data in the log file.'''
     return s
 
 #these values come from moduleALloc_setupFile.cc
 #enum class Step : char {
 #  preSourceTransition = 'S',
 #  postSourceTransition = 's',
 #  preModulePrefetching = 'P',
 #  postModulePrefetching = 'p',
 #  preModuleEventAcquire = 'A',
 #  postModuleEventAcquire = 'a',
 #  preModuleTransition = 'M',
 #  preEventReadFromSource = 'R',
 #  postEventReadFromSource = 'r',
 #  preModuleEventDelayedGet = 'D',
 #  postModuleEventDelayedGet = 'd',
 #  postModuleTransition = 'm',
 #  preESModulePrefetching = 'Q',
 #  postESModulePrefetching = 'q',
 #  preESModule = 'N',
 #  postESModule = 'n',
 #  preESModuleAcquire = 'B',
 #  postESModuleAcquire = 'b',
 #  preFrameworkTransition = 'F',
 #  postFrameworkTransition = 'f'
 #};
 
 

◆ processingStepsFromFile()

def edmModuleAllocMonitorAnalyze.processingStepsFromFile	(	f,
		moduleNames,
		esModuleNames,
		recordNames,
		moduleCentric
	)

Definition at line 930 of file edmModuleAllocMonitorAnalyze.py.

References lineParserFactory(), and mkLumiAveragedPlots.tuple.

Referenced by edmModuleAllocMonitorAnalyze.ModuleAllocCompactFileParser.processingSteps().

 def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, moduleCentric):
     for rawl in f:
         l = rawl.strip()
         if not l or l[0] == '#':
             continue
         (step,payload) = tuple(l.split(None,1))
         payload=payload.split()
 
         parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, moduleCentric)
         if parser:
             yield parser
     return
 

◆ runTests()

def edmModuleAllocMonitorAnalyze.runTests ( )

Definition at line 1475 of file edmModuleAllocMonitorAnalyze.py.

 def runTests():
     return unittest.main(argv=sys.argv[:1])
 
 #=======================================

◆ sortByAttribute()

def edmModuleAllocMonitorAnalyze.sortByAttribute	(	parser,
		attribute
	)

Definition at line 1211 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.TestModuleCommand.testSortBy().

 def sortByAttribute(parser, attribute):
     sync = SyncValues()
     #used to keep track of outstanding module transitions
     temp = TempModuleTransitionInfos()
     data = ModuleCentricModuleData()
     for p in parser.processingSteps():
         if hasattr(p, "jsonInfo"):
             p.jsonInfo(sync,temp, data)
     return data.sortModulesBy(attribute)
     

◆ startTime()

def edmModuleAllocMonitorAnalyze.startTime ( x )

Definition at line 1195 of file edmModuleAllocMonitorAnalyze.py.

 def startTime(x):
     return x["start"]
 

◆ textOutput()

def edmModuleAllocMonitorAnalyze.textOutput ( parser )

Definition at line 999 of file edmModuleAllocMonitorAnalyze.py.

References print().

 def textOutput( parser ):
     context = {}
     for p in parser.processingSteps():
         print(p.text(context))
     

◆ textPrefix_()

def edmModuleAllocMonitorAnalyze.textPrefix_	(	time,
		indentLevel
	)

Definition at line 190 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.FrameworkTransitionParser.textPrefix(), edmModuleAllocMonitorAnalyze.SourceTransitionParser.textPrefix(), edmModuleAllocMonitorAnalyze.EDModuleTransitionParser.textPrefix(), and edmModuleAllocMonitorAnalyze.ESModuleTransitionParser.textPrefix().

 def textPrefix_(time, indentLevel):
     #using 11 spaces for time should accomodate a job that runs 24 hrs
     return f'{time:>11} '+"++"*indentLevel
 

◆ transitionIndentLevel()

def edmModuleAllocMonitorAnalyze.transitionIndentLevel ( transition )

Definition at line 163 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.EDModuleTransitionParser.baseIndentLevel(), edmModuleAllocMonitorAnalyze.ESModuleTransitionParser.baseIndentLevel(), and edmModuleAllocMonitorAnalyze.FrameworkTransitionParser.indentLevel().

 def transitionIndentLevel(transition):
     return transitionToIndent_[transition]
 

◆ transitionIsGlobal()

def edmModuleAllocMonitorAnalyze.transitionIsGlobal ( transition )

Definition at line 187 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.EDModuleTransitionParser._postJsonVis(), edmModuleAllocMonitorAnalyze.ESModuleTransitionParser._postJsonVis(), edmModuleAllocMonitorAnalyze.EDModuleTransitionParser._preJsonVis(), edmModuleAllocMonitorAnalyze.ESModuleTransitionParser._preJsonVis(), edmModuleAllocMonitorAnalyze.SyncValues.get(), edmModuleAllocMonitorAnalyze.PreFrameworkTransitionParser.jsonVisInfo(), and edmModuleAllocMonitorAnalyze.PostFrameworkTransitionParser.jsonVisInfo().

 def transitionIsGlobal(transition):
     return transition in globalTransitions_;
 

◆ transitionName()

def edmModuleAllocMonitorAnalyze.transitionName ( transition )

Definition at line 132 of file edmModuleAllocMonitorAnalyze.py.

Referenced by edmModuleAllocMonitorAnalyze.ModuleData.toSimpleDict().

 def transitionName(transition):
     return transitionToNames_[transition]
 

Variable Documentation

◆ action

edmModuleAllocMonitorAnalyze.action

Definition at line 1492 of file edmModuleAllocMonitorAnalyze.py.

◆ activityNames_

edmModuleAllocMonitorAnalyze.activityNames_

Definition at line 93 of file edmModuleAllocMonitorAnalyze.py.

◆ args

edmModuleAllocMonitorAnalyze.args

Definition at line 1508 of file edmModuleAllocMonitorAnalyze.py.

◆ default

edmModuleAllocMonitorAnalyze.default

Definition at line 1495 of file edmModuleAllocMonitorAnalyze.py.

◆ description

edmModuleAllocMonitorAnalyze.description

Definition at line 1485 of file edmModuleAllocMonitorAnalyze.py.

◆ epilog

edmModuleAllocMonitorAnalyze.epilog

Definition at line 1487 of file edmModuleAllocMonitorAnalyze.py.

◆ formatter_class

edmModuleAllocMonitorAnalyze.formatter_class

Definition at line 1486 of file edmModuleAllocMonitorAnalyze.py.

◆ globalTransitions_

edmModuleAllocMonitorAnalyze.globalTransitions_

Definition at line 166 of file edmModuleAllocMonitorAnalyze.py.

◆ help

edmModuleAllocMonitorAnalyze.help

Definition at line 1490 of file edmModuleAllocMonitorAnalyze.py.

◆ indent

edmModuleAllocMonitorAnalyze.indent

Definition at line 1515 of file edmModuleAllocMonitorAnalyze.py.

◆ kLargestLumiNumber

edmModuleAllocMonitorAnalyze.kLargestLumiNumber

Definition at line 52 of file edmModuleAllocMonitorAnalyze.py.

◆ kMicroToSec

edmModuleAllocMonitorAnalyze.kMicroToSec

Definition at line 47 of file edmModuleAllocMonitorAnalyze.py.

◆ kSourceDelayedRead

edmModuleAllocMonitorAnalyze.kSourceDelayedRead

Definition at line 50 of file edmModuleAllocMonitorAnalyze.py.

◆ kSourceFindEvent

edmModuleAllocMonitorAnalyze.kSourceFindEvent

Definition at line 49 of file edmModuleAllocMonitorAnalyze.py.

◆ parser

edmModuleAllocMonitorAnalyze.parser