edmTracerCompactLogViewer Namespace Reference

Classes
class	Activity
class	Containers
class	Counter
class	DummyFile
class	EDModuleTransitionParser
class	ESModuleTransitionParser
class	FrameworkTransitionParser
class	ModuleCentricContainers
class	Phase
class	PostEDModuleAcquireParser
class	PostEDModuleEventDelayedGetParser
class	PostEDModulePrefetchingParser
class	PostEDModuleTransitionParser
class	PostESModuleAcquireParser
class	PostESModulePrefetchingParser
class	PostESModuleTransitionParser
class	PostEventReadFromSourceParser
class	PostFrameworkTransitionParser
class	PostSourceTransitionParser
class	PreEDModuleAcquireParser
class	PreEDModuleEventDelayedGetParser
class	PreEDModulePrefetchingParser
class	PreEDModuleTransitionParser
class	PreESModuleAcquireParser
class	PreESModulePrefetchingParser
class	PreESModuleTransitionParser
class	PreEventReadFromSourceParser
class	PreFrameworkTransitionParser
class	PreSourceTransitionParser
class	QueuingFrameworkTransitionParser
class	SourceTransitionParser
class	TestModuleCommand
class	TracerCompactFileParser

Functions
def	findMatchingTransition (sync, containers)
def	jsonInfo (parser)
def	jsonModuleTransition (type, id, modID, callID, activity, start, finish=0)
def	jsonTransition (type, id, sync, start, finish, isSrc=False)
def	lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric)
def	popQueuedTransitions (sync, container)
def	printHelp ()
def	processingStepsFromFile (f, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric)
def	runTests ()
def	startTime (x)
def	textOutput (parser)
def	textPrefix_ (time, indentLevel)
def	transitionIndentLevel (transition)
def	transitionIsGlobal (transition)
def	transitionName (transition)

Variables
	action
	args
	description
	epilog
	f
	formatter_class
	globalTransitions_
	help
	j
	kLargestLumiNumber
	kMicroToSec
	kSourceDelayedRead
	kSourceFindEvent
	parser
	transitionsToFindMatch_
	transitionToIndent_
	transitionToNames_
	type

Function Documentation

findMatchingTransition()

def edmTracerCompactLogViewer.findMatchingTransition	(		sync,
			containers
	)

Definition at line 224 of file edmTracerCompactLogViewer.py.

References print(), and isotrackApplyRegressor.range.

Referenced by edmTracerCompactLogViewer.QueuingFrameworkTransitionParser.jsonInfo().

def findMatchingTransition(sync, containers):
    for i in range(len(containers)):
        if containers[i][-1]["sync"] == sync:
            return i
    #need more exhausting search
    for i in range(len(containers)):
        for t in containers[i]:
            if t["sync"] == sync:
                return i

    print("find failed",sync, containers)
    return None

jsonInfo()

def edmTracerCompactLogViewer.jsonInfo

(

parser

)

Definition at line 991 of file edmTracerCompactLogViewer.py.

References mps_setup.append.

Referenced by edmTracerCompactLogViewer.TestModuleCommand.testFrameworkOnly(), edmTracerCompactLogViewer.TestModuleCommand.testFull(), and edmTracerCompactLogViewer.TestModuleCommand.testModuleCentric().

def jsonInfo(parser):
    counter = Counter()
    if parser._moduleCentric:
        data = ModuleCentricContainers()
    else:
        data = Containers()
    for p in parser.processingSteps():
        p.jsonInfo(counter, data)
    #make sure everything is sorted
    for g in data.allGlobals():
        g.sort(key=startTime)
    final = {"transitions" : [] , "modules": [], "esModules": []}
    final["transitions"].append({ "name":"Global", "slots": []})
    globals = final["transitions"][-1]["slots"]
    for i, g in enumerate(data.allGlobals()):
        globals.append(g)
        if not parser._moduleCentric and not parser._frameworkOnly:
            if len(data._modGlobals) < i+1:
                break
            for mod in data._modGlobals[i]:
                globals.append(mod)
    for i,s in enumerate(data.allStreams()):
        final["transitions"].append({"name": f"Stream {i}", "slots":[]})
        stream = final["transitions"][-1]["slots"]
        stream.append(s)
        if not parser._moduleCentric and not parser._frameworkOnly:
            for mod in data._modStreams[i]:
                stream.append(mod)
    if parser._moduleCentric:
        sourceSlot = data._modules[data._moduleID2Index(0)]
        modules = []
        for i,m in parser._moduleNames.items():
            modules.append({"name": f"{m}", "slots":[]})
            slots = modules[-1]["slots"]
            foundSlots = data._modules[data._moduleID2Index(i)]
            time = 0
            for s in foundSlots:
                slots.append(s)
                for t in s:
                    if t["act"] !=Activity.prefetch:
                        time += t["finish"]-t["start"]
            modules[-1]['time']=time
        for i,m in parser._esModuleNames.items():
            modules.append({"name": f"{m}", "slots":[]})
            slots = modules[-1]["slots"]
            foundSlots = data._modules[data._moduleID2Index(-1*i)]
            time = 0
            for s in foundSlots:
                slots.append(s)
                for t in s:
                    if t["act"] !=Activity.prefetch:
                        time += t["finish"]-t["start"]
            modules[-1]['time']=time
        modules.sort(key= lambda x : x['time'], reverse=True)
        final['transitions'].append({"name": "source", "slots":sourceSlot})
        for m in modules:
            final['transitions'].append(m)

    if not parser._frameworkOnly:
        max = 0
        for k in parser._moduleNames.keys():
            if k > max:
                max = k
        
        final["modules"] =['']*(max+1)
        final["modules"][0] = 'source'
        for k,v in parser._moduleNames.items():
            final["modules"][k]=v
        
        max = 0
        for k in parser._esModuleNames.keys():
            if k > max:
                max = k
        final["esModules"] = ['']*(max+1)
        for k,v in parser._esModuleNames.items():
            final["esModules"][k] = v
    return final
    
#=======================================

jsonModuleTransition()

def edmTracerCompactLogViewer.jsonModuleTransition	(		type,
			id,
			modID,
			callID,
			activity,
			start,
			finish = 0
	)

Definition at line 986 of file edmTracerCompactLogViewer.py.

Referenced by edmTracerCompactLogViewer.EDModuleTransitionParser._preJson(), edmTracerCompactLogViewer.ESModuleTransitionParser._preJson(), edmTracerCompactLogViewer.PostEDModulePrefetchingParser.jsonInfo(), and edmTracerCompactLogViewer.PostEDModuleAcquireParser.jsonInfo().

def jsonModuleTransition(type, id, modID, callID, activity, start, finish=0):
    return {"type": type, "id": id, "mod": modID, "call": callID, "act": activity, "start": start*kMicroToSec, "finish": finish*kMicroToSec}

jsonTransition()

def edmTracerCompactLogViewer.jsonTransition	(		type,
			id,
			sync,
			start,
			finish,
			isSrc = False
	)

Definition at line 983 of file edmTracerCompactLogViewer.py.

Referenced by edmTracerCompactLogViewer.QueuingFrameworkTransitionParser.jsonInfo().

def jsonTransition(type, id, sync, start, finish, isSrc=False):
    return {"type": type, "id": id, "sync": sync, "start": start*kMicroToSec, "finish": finish*kMicroToSec, "isSrc":isSrc}

lineParserFactory()

def edmTracerCompactLogViewer.lineParserFactory	(		step,
			payload,
			moduleNames,
			esModuleNames,
			recordNames,
			frameworkOnly,
			moduleCentric
	)

Definition at line 724 of file edmTracerCompactLogViewer.py.

Referenced by processingStepsFromFile().

def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
    if step == 'F':
        parser = PreFrameworkTransitionParser(payload)
        if parser.transition == Phase.esSyncEnqueue:
            return QueuingFrameworkTransitionParser(payload)
        return parser
    if step == 'f':
        return PostFrameworkTransitionParser(payload)
    if step == 'S':
        return PreSourceTransitionParser(payload, moduleCentric)
    if step == 's':
        return PostSourceTransitionParser(payload, moduleCentric)
    if frameworkOnly:
        return None
    if step == 'M':
        return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
    if step == 'm':
        return PostEDModuleTransitionParser(payload, moduleNames)
    if step == 'P':
        return PreEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
    if step == 'p':
        return PostEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
    if step == 'A':
        return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
    if step == 'a':
        return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
    if step == 'D':
        return PreEDModuleEventDelayedGetParser(payload, moduleNames)
    if step == 'd':
        return PostEDModuleEventDelayedGetParser(payload, moduleNames)
    if step == 'R':
        return PreEventReadFromSourceParser(payload, moduleNames)
    if step == 'r':
        return PostEventReadFromSourceParser(payload, moduleNames)
    if step == 'N':
        return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
    if step == 'n':
        return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
    if step == 'Q':
        return PreESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
    if step == 'q':
        return PostESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
    if step == 'B':
        return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
    if step == 'b':
        return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)

    
#----------------------------------------------

popQueuedTransitions()

def edmTracerCompactLogViewer.popQueuedTransitions	(		sync,
			container
	)

Definition at line 237 of file edmTracerCompactLogViewer.py.

References isotrackApplyRegressor.range.

def popQueuedTransitions(sync, container):
    results = []
    for i in range(len(container)):
        if sync == container[i]["sync"]:
            results.append(container[i])
            results.append(container[i+1])
            del container[i]
            del container[i]
            break
    return results
        

printHelp()

def edmTracerCompactLogViewer.printHelp

(

)

Definition at line 10 of file edmTracerCompactLogViewer.py.

def printHelp():
    s = '''
To Use: Add the Tracer Service to the cmsRun job use something like this
  in the configuration:

  process.add_(cms.Service("Tracer", fileName = cms.untracked.string("tracer.log")))

  After running the job, execute this script and pass the name of the
  Tracer log file to the script.

  This script will output a more human readable form of the data in the Tracer log file.'''
    return s

#these values come from tracer_setupFile.cc
#enum class Step : char {
#  preSourceTransition = 'S',
#  postSourceTransition = 's',
#  preModulePrefetching = 'P',
#  postModulePrefetching = 'p',
#  preModuleEventAcquire = 'A',
#  postModuleEventAcquire = 'a',
#  preModuleTransition = 'M',
#  preEventReadFromSource = 'R',
#  postEventReadFromSource = 'r',
#  preModuleEventDelayedGet = 'D',
#  postModuleEventDelayedGet = 'd',
#  postModuleTransition = 'm',
#  preESModulePrefetching = 'Q',
#  postESModulePrefetching = 'q',
#  preESModule = 'N',
#  postESModule = 'n',
#  preESModuleAcquire = 'B',
#  postESModuleAcquire = 'b',
#  preFrameworkTransition = 'F',
#  postFrameworkTransition = 'f'
#};

processingStepsFromFile()

def edmTracerCompactLogViewer.processingStepsFromFile	(		f,
			moduleNames,
			esModuleNames,
			recordNames,
			frameworkOnly,
			moduleCentric
	)

Definition at line 773 of file edmTracerCompactLogViewer.py.

References lineParserFactory(), and mkLumiAveragedPlots.tuple.

Referenced by edmTracerCompactLogViewer.TracerCompactFileParser.processingSteps().

def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
    for rawl in f:
        l = rawl.strip()
        if not l or l[0] == '#':
            continue
        (step,payload) = tuple(l.split(None,1))
        payload=payload.split()

        parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric)
        if parser:
            yield parser
    return

runTests()

def edmTracerCompactLogViewer.runTests

(

)

Definition at line 1250 of file edmTracerCompactLogViewer.py.

def runTests():
    return unittest.main(argv=sys.argv[:1])

#=======================================

startTime()

def edmTracerCompactLogViewer.startTime

(

x

)

Definition at line 989 of file edmTracerCompactLogViewer.py.

def startTime(x):
    return x["start"]

textOutput()

def edmTracerCompactLogViewer.textOutput

(

parser

)

Definition at line 843 of file edmTracerCompactLogViewer.py.

References print().

def textOutput( parser ):
    context = {}
    for p in parser.processingSteps():
        print(p.text(context))
    

textPrefix_()

def edmTracerCompactLogViewer.textPrefix_	(		time,
			indentLevel
	)

Definition at line 180 of file edmTracerCompactLogViewer.py.

Referenced by edmTracerCompactLogViewer.FrameworkTransitionParser.textPrefix(), edmTracerCompactLogViewer.SourceTransitionParser.textPrefix(), edmTracerCompactLogViewer.EDModuleTransitionParser.textPrefix(), and edmTracerCompactLogViewer.ESModuleTransitionParser.textPrefix().

def textPrefix_(time, indentLevel):
    #using 11 spaces for time should accomodate a job that runs 24 hrs
    return f'{time:>11} '+"++"*indentLevel

transitionIndentLevel()

def edmTracerCompactLogViewer.transitionIndentLevel

(

transition

)

Definition at line 154 of file edmTracerCompactLogViewer.py.

Referenced by edmTracerCompactLogViewer.EDModuleTransitionParser.baseIndentLevel(), edmTracerCompactLogViewer.ESModuleTransitionParser.baseIndentLevel(), and edmTracerCompactLogViewer.FrameworkTransitionParser.indentLevel().

def transitionIndentLevel(transition):
    return transitionToIndent_[transition]

transitionIsGlobal()

def edmTracerCompactLogViewer.transitionIsGlobal

(

transition

)

Definition at line 177 of file edmTracerCompactLogViewer.py.

Referenced by edmTracerCompactLogViewer.EDModuleTransitionParser._postJson(), edmTracerCompactLogViewer.ESModuleTransitionParser._postJson(), edmTracerCompactLogViewer.EDModuleTransitionParser._preJson(), edmTracerCompactLogViewer.ESModuleTransitionParser._preJson(), edmTracerCompactLogViewer.PreFrameworkTransitionParser.jsonInfo(), and edmTracerCompactLogViewer.PostFrameworkTransitionParser.jsonInfo().

def transitionIsGlobal(transition):
    return transition in globalTransitions_;

transitionName()

def edmTracerCompactLogViewer.transitionName

(

transition

)

Definition at line 123 of file edmTracerCompactLogViewer.py.

def transitionName(transition):
    return transitionToNames_[transition]

Variable Documentation

action

edmTracerCompactLogViewer.action

Definition at line 1267 of file edmTracerCompactLogViewer.py.

args

edmTracerCompactLogViewer.args

Definition at line 1282 of file edmTracerCompactLogViewer.py.

description

edmTracerCompactLogViewer.description

Definition at line 1260 of file edmTracerCompactLogViewer.py.

epilog

edmTracerCompactLogViewer.epilog

Definition at line 1262 of file edmTracerCompactLogViewer.py.

f

edmTracerCompactLogViewer.f

Definition at line 1292 of file edmTracerCompactLogViewer.py.

formatter_class

edmTracerCompactLogViewer.formatter_class

Definition at line 1261 of file edmTracerCompactLogViewer.py.

globalTransitions_

edmTracerCompactLogViewer.globalTransitions_

Definition at line 157 of file edmTracerCompactLogViewer.py.

help

edmTracerCompactLogViewer.help

Definition at line 1265 of file edmTracerCompactLogViewer.py.

j

edmTracerCompactLogViewer.j

Definition at line 1288 of file edmTracerCompactLogViewer.py.

kLargestLumiNumber

edmTracerCompactLogViewer.kLargestLumiNumber

Definition at line 53 of file edmTracerCompactLogViewer.py.

kMicroToSec

edmTracerCompactLogViewer.kMicroToSec

Definition at line 48 of file edmTracerCompactLogViewer.py.

kSourceDelayedRead

edmTracerCompactLogViewer.kSourceDelayedRead

Definition at line 51 of file edmTracerCompactLogViewer.py.

kSourceFindEvent

edmTracerCompactLogViewer.kSourceFindEvent

Definition at line 50 of file edmTracerCompactLogViewer.py.

parser

edmTracerCompactLogViewer.parser

Definition at line 1260 of file edmTracerCompactLogViewer.py.

transitionsToFindMatch_

edmTracerCompactLogViewer.transitionsToFindMatch_

Definition at line 248 of file edmTracerCompactLogViewer.py.

transitionToIndent_

edmTracerCompactLogViewer.transitionToIndent_

Definition at line 126 of file edmTracerCompactLogViewer.py.

transitionToNames_

edmTracerCompactLogViewer.transitionToNames_

Definition at line 94 of file edmTracerCompactLogViewer.py.

type

edmTracerCompactLogViewer.type

Definition at line 1264 of file edmTracerCompactLogViewer.py.