CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
edmStreamStallGrapher Namespace Reference

Functions

def createAsciiImage
 
def createPDFImage
 
def findStalledModules
 
def getTime
 
def printHelp
 
def printStalledModulesInOrder
 
def readLogFile
 

Variables

 doGraphic = False
 
list fileName = sys.argv[-1]
 
tuple stalledModules = findStalledModules(processingSteps, numStreams)
 

Function Documentation

def edmStreamStallGrapher.createAsciiImage (   processingSteps,
  numStreams,
  maxNameSize 
)

Definition at line 108 of file edmStreamStallGrapher.py.

108 
109 def createAsciiImage(processingSteps, numStreams, maxNameSize):
110  #print processingSteps
111  #exit(1)
112  streamState = [1]*(numStreams+1)
113  streamTime = [0]*(numStreams+1)
114  #lastTime = 0
115  seenInit = False
116  for n,trans,s,time,delayed in processingSteps:
117  if n == "FINISH INIT":
118  seenInit = True
119  continue
120  oldState = streamState[s]
121  streamState[s]=trans
122  waitTime = None
123  if delayed:
124  n = "source"
125  if trans == kStarted:
126  waitTime = time - streamTime[s]
127  streamState[s]=2
128  else:
129  if oldState != trans:
130  streamState[s]=3
131  streamTime[s] = time
132  states = "%-*s: " % (maxNameSize,n)
133  for state in streamState:
134  if state == 0:
135  states +=" "
136  elif state == 1:
137  states +="|"
138  elif state == 2:
139  states +="-"
140  elif state == 3:
141  states +="+"
142  if waitTime is not None:
143  states += " %.2f"% waitTime
144  if waitTime > 0.1 and seenInit:
145  states += " STALLED"
146 
147  print states
148  streamState[s]=trans
149  return stalledModules
150 
#----------------------------------------------
def edmStreamStallGrapher.createPDFImage (   processingSteps,
  numStreams,
  stalledModuleInfo 
)

Definition at line 178 of file edmStreamStallGrapher.py.

References funct.abs(), python.multivaluedict.append(), cmsRelvalreport.exit, and printHelp().

178 
179 def createPDFImage(processingSteps, numStreams, stalledModuleInfo):
180  import matplotlib.pyplot as plt
181 
182  streamStartDepth = [0]*(numStreams+1)
183  streamTime = [0]*(numStreams+1)
184  streamStartTimes = [ [] for x in xrange(numStreams+1)]
185  streamColors = [[] for x in xrange(numStreams+1)]
186 
187  stalledModuleNames = [ x for x in stalledModuleInfo.iterkeys()]
188 
189  streamStartTimes = [ [] for x in xrange(numStreams+1)]
190  streamColors = [[] for x in xrange(numStreams+1)]
191 
192  for n,trans,s,time,delayed in processingSteps:
193  if trans == kStarted:
194  streamStartDepth[s] +=1
195  streamTime[s] = time
196  else:
197  streamStartDepth[s] -=1
198  if 0 == streamStartDepth[s]:
199  streamStartTimes[s].append((streamTime[s],time-streamTime[s]))
200  c="green"
201  if delayed:
202  c="orange"
203  elif n in stalledModuleNames:
204  c="red"
205  #elif len(streamColors[s]) %2:
206  # c="blue"
207  streamColors[s].append(c)
208 
209  #consolodate contiguous blocks with the same color
210  # this drastically reduces the size of the pdf file
211  oldStreamTimes = streamStartTimes
212  oldStreamColors = streamColors
213 
214  streamStartTimes = [ [] for x in xrange(numStreams+1)]
215  streamColors = [[] for x in xrange(numStreams+1)]
216 
217  for s in xrange(numStreams+1):
218  streamStartTimes[s].append(oldStreamTimes[s][0])
219  streamColors[s].append(oldStreamColors[s][0])
220  lastStartTime,lastTimeLength = oldStreamTimes[s][0]
221  lastColor = oldStreamColors[s][0]
222  for i in xrange(1, len(oldStreamTimes[s])):
223  start,length = oldStreamTimes[s][i]
224  color = oldStreamColors[s][i]
225  #use a millisecond tolerance to avoid rounding
226  if color == lastColor and abs(lastStartTime+lastTimeLength-start)<0.001:
227  lastTimeLength += length
228  else:
229  streamStartTimes[s].append((lastStartTime,lastTimeLength))
230  streamColors[s].append(lastColor)
231  lastStartTime = start
232  lastTimeLength = length
233  lastColor = color
234  streamStartTimes[s].append((lastStartTime,lastTimeLength))
235  streamColors[s].append(lastColor)
236 
237  fig, ax = plt.subplots()
238  ax.set_xlabel("Time (sec)")
239  ax.set_ylabel("Stream ID")
240 
241  i=1
242  for s in xrange(numStreams+1):
243  t = streamStartTimes[s]
244  ax.broken_barh(t,(i-0.4,0.8),facecolors=streamColors[s],edgecolors=streamColors[s],linewidth=0)
245  i=i+1
246 
247  #add key .1, .3, .7
248  fig.text(0.1, 0.95, "modules running", color = "green", horizontalalignment = 'left')
249  fig.text(0.5, 0.95, "stalled module running", color = "red", horizontalalignment = 'center')
250  fig.text(0.9, 0.95, "read from input", color = "orange", horizontalalignment = 'right')
251  plt.savefig("stall.pdf")
252 
253 
254 
#=======================================
funct::abs
Abs< T >::type abs(const T &t)
Definition: Abs.h:22
def edmStreamStallGrapher.findStalledModules (   processingSteps,
  numStreams 
)

Definition at line 89 of file edmStreamStallGrapher.py.

89 
90 def findStalledModules(processingSteps, numStreams):
91  streamTime = [0]*(numStreams+1)
92  stalledModules = {}
93  for n,trans,s,time,delayed in processingSteps:
94  waitTime = None
95  if delayed:
96  n = "source"
97  if trans == kStarted:
98  waitTime = time - streamTime[s]
99  else:
100  streamTime[s] = time
101  if waitTime is not None:
102  if waitTime > 0.1:
103  t = stalledModules.setdefault(n,[])
104  t.append(waitTime)
105  return stalledModules
106 
107 
#----------------------------------------------
def edmStreamStallGrapher.getTime (   line)

Definition at line 40 of file edmStreamStallGrapher.py.

Referenced by EcalPerEvtLaserAnalyzer.endJob(), EcalLaserAnalyzer2.endJob(), EcalLaserAnalyzer.endJob(), readLogFile(), and pos::PixelFEDTestDAC.writeXMLHeader().

40 
41 def getTime(line):
42  time = line.split(" ")[1]
43  time = time.split(":")
44  time = int(time[0])*60*60+int(time[1])*60+float(time[2])
45  return time
46 
47 #Stream states
48 kStarted=0
49 kFinished=1
50 
#----------------------------------------------
def edmStreamStallGrapher.printHelp ( )

Definition at line 4 of file edmStreamStallGrapher.py.

Referenced by createPDFImage().

4 
5 def printHelp():
6  s = """Purpose: Convert a cmsRun log with Tracer info into a stream stall graph.
7 
8 edmStreamStallGrapher [-g] <log file name>
9 
10 Option: -g instead of ascii art, create a pdf file showing the work being done on each stream
11 
12 To Use: Add the Tracer Service to the cmsRun job you want to check for stream stalls.
13  Make sure to use the 'printTimstamps' option
14  cms.Service("Tracer", printTimestamps = cms.untracked.bool(True))
15  After running the job, execute this script and pass the name of the log file to the
16  script as the only command line argument.
17 
18 To Read: The script will then print an 'ASCII art' stall graph which consists of the name of
19  the module which either started or stopped running on a stream, and the state of each
20  stream at that moment in time and if the module just started, you will also see the
21  amount of time on that stream between the previous module finishing and this module starting.
22  The state of a stream is represented by a symbol:
23  blank (" ") the stream is currently running a module
24  line ("|") the stream is waiting to run a module
25  minus ("-") the stream has just finished waiting and is starting a module
26  plus ("+") the stream just finished running a module
27  If a module had to wait more than 0.1 seconds, the end of the line will have "STALLED".
28  Once the first 4 events have finished processing, the program prints "FINISH INIT".
29  This is useful if one wants to ignore stalled caused by startup actions, e.g. reading
30  conditions.
31 
32  Once the graph is completed, the program outputs the list of modules which had
33  the greatest total stall times. The list is sorted by total stall time and
34  written in descending order. In addition, the list of all stall times for the
35  module is given.
36 """
37  print s
38 
39 
#----------------------------------------------
def edmStreamStallGrapher.printStalledModulesInOrder (   stalledModules)

Definition at line 151 of file edmStreamStallGrapher.py.

References join(), and list().

151 
152 def printStalledModulesInOrder(stalledModules):
153  priorities = list()
154  maxNameSize = 0
155  for n,t in stalledModules.iteritems():
156  nameLength = len(n)
157  if nameLength > maxNameSize:
158  maxNameSize = nameLength
159  t.sort(reverse=True)
160  priorities.append((n,sum(t),t))
161 
162  def sumSort(i,j):
163  return cmp(i[1],j[1])
164  priorities.sort(cmp=sumSort, reverse=True)
165 
166  nameColumn = "Stalled Module"
167  if len(nameColumn) > maxNameSize:
168  maxNameSize = len(nameColumn)
169 
170  stallColumn = "Tot Stall Time"
171  stallColumnLength = len(stallColumn)
172 
173  print "%-*s" % (maxNameSize, nameColumn), "%-*s"%(stallColumnLength,stallColumn), " Stall Times"
174  for n,s,t in priorities:
175  paddedName = "%-*s:" % (maxNameSize,n)
176  print paddedName, "%-*.2f"%(stallColumnLength,s), ", ".join([ "%.2f"%x for x in t])
177 
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def edmStreamStallGrapher.readLogFile (   fileName)

Definition at line 51 of file edmStreamStallGrapher.py.

References getTime(), and list().

51 
52 def readLogFile(fileName):
53  f = open(fileName,"r")
54 
55  processingSteps = list()
56  numStreams = 0
57  maxNameSize = 0
58  startTime = 0
59  foundEventToStartFrom = False
60  for l in f:
61  if not foundEventToStartFrom:
62  if l.find("event = 5") != -1:
63  foundEventToStartFrom = True
64  stream = int( l[l.find("stream = ")+9])
65  processingSteps.append(("FINISH INIT",kFinished,stream,getTime(l)-startTime,False))
66  if l.find("processing event for module") != -1:
67  time = getTime(l)
68  if startTime == 0:
69  startTime = time
70  time = time - startTime
71  trans = kStarted
72  stream = 0
73  delayed = False
74  if l.find("finished:") != -1:
75  trans = kFinished
76  stream = int( l[l.find("stream = ")+9])
77  name = l.split("'")[1]
78  if l.find("delayed") != -1:
79  delayed = True
80  if len(name) > maxNameSize:
81  maxNameSize = len(name)
82  processingSteps.append((name,trans,stream,time,delayed))
83  if stream > numStreams:
84  numStreams = stream
85  f.close()
86  return (processingSteps,numStreams,maxNameSize)
87 
88 
#----------------------------------------------
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run

Variable Documentation

edmStreamStallGrapher.doGraphic = False

Definition at line 262 of file edmStreamStallGrapher.py.

list edmStreamStallGrapher.fileName = sys.argv[-1]

Definition at line 269 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.stalledModules = findStalledModules(processingSteps, numStreams)

Definition at line 272 of file edmStreamStallGrapher.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5