CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
edmStreamStallGrapher Namespace Reference

Functions

def createAsciiImage
 
def createPDFImage
 
def findStalledModules
 
def getTime
 
def printHelp
 
def printStalledModulesInOrder
 
def readLogFile
 

Variables

 doGraphic = False
 
list fileName = sys.argv[-1]
 
tuple stalledModules = findStalledModules(processingSteps, numStreams)
 

Function Documentation

def edmStreamStallGrapher.createAsciiImage (   processingSteps,
  numStreams,
  maxNameSize 
)

Definition at line 108 of file edmStreamStallGrapher.py.

108 
109 def createAsciiImage(processingSteps, numStreams, maxNameSize):
110  #print processingSteps
111  #exit(1)
112  streamState = [1]*(numStreams+1)
113  streamTime = [0]*(numStreams+1)
114  #lastTime = 0
115  seenInit = False
116  for n,trans,s,time,delayed in processingSteps:
117  if n == "FINISH INIT":
118  seenInit = True
119  continue
120  oldState = streamState[s]
121  streamState[s]=trans
122  waitTime = None
123  if delayed:
124  n = "source"
125  if trans == kStarted:
126  waitTime = time - streamTime[s]
127  streamState[s]=2
128  else:
129  if oldState != trans:
130  streamState[s]=3
131  streamTime[s] = time
132  states = "%-*s: " % (maxNameSize,n)
133  for state in streamState:
134  if state == 0:
135  states +=" "
136  elif state == 1:
137  states +="|"
138  elif state == 2:
139  states +="-"
140  elif state == 3:
141  states +="+"
142  if waitTime is not None:
143  states += " %.2f"% waitTime
144  if waitTime > 0.1 and seenInit:
145  states += " STALLED"
146 
147  print states
148  streamState[s]=trans
149  return stalledModules
150 
#----------------------------------------------
def edmStreamStallGrapher.createPDFImage (   processingSteps,
  numStreams,
  stalledModuleInfo 
)

Definition at line 178 of file edmStreamStallGrapher.py.

References funct.abs(), bitset_utilities.append(), cmsRelvalreport.exit, and printHelp().

178 
179 def createPDFImage(processingSteps, numStreams, stalledModuleInfo):
180  import matplotlib.pyplot as plt
181 
182  streamStartDepth = [0]*(numStreams+1)
183  streamTime = [0]*(numStreams+1)
184  streamStartTimes = [ [] for x in xrange(numStreams+1)]
185  streamColors = [[] for x in xrange(numStreams+1)]
186 
187  stalledModuleNames = [ x for x in stalledModuleInfo.iterkeys()]
188 
189  streamStartTimes = [ [] for x in xrange(numStreams+1)]
190  streamColors = [[] for x in xrange(numStreams+1)]
191 
192  for n,trans,s,time,delayed in processingSteps:
193  if n == "FINISH INIT":
194  continue
195  if trans == kStarted:
196  streamStartDepth[s] +=1
197  streamTime[s] = time
198  else:
199  streamStartDepth[s] -=1
200  if 0 == streamStartDepth[s]:
201  streamStartTimes[s].append((streamTime[s],time-streamTime[s]))
202  c="green"
203  if delayed:
204  c="orange"
205  elif n in stalledModuleNames:
206  c="red"
207  #elif len(streamColors[s]) %2:
208  # c="blue"
209  streamColors[s].append(c)
210 
211  #consolodate contiguous blocks with the same color
212  # this drastically reduces the size of the pdf file
213  oldStreamTimes = streamStartTimes
214  oldStreamColors = streamColors
215 
216  streamStartTimes = [ [] for x in xrange(numStreams+1)]
217  streamColors = [[] for x in xrange(numStreams+1)]
218 
219  for s in xrange(numStreams+1):
220  streamStartTimes[s].append(oldStreamTimes[s][0])
221  streamColors[s].append(oldStreamColors[s][0])
222  lastStartTime,lastTimeLength = oldStreamTimes[s][0]
223  lastColor = oldStreamColors[s][0]
224  for i in xrange(1, len(oldStreamTimes[s])):
225  start,length = oldStreamTimes[s][i]
226  color = oldStreamColors[s][i]
227  #use a millisecond tolerance to avoid rounding
228  if color == lastColor and abs(lastStartTime+lastTimeLength-start)<0.001:
229  lastTimeLength += length
230  else:
231  streamStartTimes[s].append((lastStartTime,lastTimeLength))
232  streamColors[s].append(lastColor)
233  lastStartTime = start
234  lastTimeLength = length
235  lastColor = color
236  streamStartTimes[s].append((lastStartTime,lastTimeLength))
237  streamColors[s].append(lastColor)
238 
239  fig, ax = plt.subplots()
240  ax.set_xlabel("Time (sec)")
241  ax.set_ylabel("Stream ID")
242 
243  i=1
244  for s in xrange(numStreams+1):
245  t = streamStartTimes[s]
246  ax.broken_barh(t,(i-0.4,0.8),facecolors=streamColors[s],edgecolors=streamColors[s],linewidth=0)
247  i=i+1
248 
249  #add key .1, .3, .7
250  fig.text(0.1, 0.95, "modules running", color = "green", horizontalalignment = 'left')
251  fig.text(0.5, 0.95, "stalled module running", color = "red", horizontalalignment = 'center')
252  fig.text(0.9, 0.95, "read from input", color = "orange", horizontalalignment = 'right')
253  plt.savefig("stall.pdf")
254 
255 
256 
#=======================================
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
funct::abs
Abs< T >::type abs(const T &t)
Definition: Abs.h:22
def edmStreamStallGrapher.findStalledModules (   processingSteps,
  numStreams 
)

Definition at line 89 of file edmStreamStallGrapher.py.

89 
90 def findStalledModules(processingSteps, numStreams):
91  streamTime = [0]*(numStreams+1)
92  stalledModules = {}
93  for n,trans,s,time,delayed in processingSteps:
94  waitTime = None
95  if delayed:
96  n = "source"
97  if trans == kStarted:
98  waitTime = time - streamTime[s]
99  else:
100  streamTime[s] = time
101  if waitTime is not None:
102  if waitTime > 0.1:
103  t = stalledModules.setdefault(n,[])
104  t.append(waitTime)
105  return stalledModules
106 
107 
#----------------------------------------------
def edmStreamStallGrapher.getTime (   line)

Definition at line 40 of file edmStreamStallGrapher.py.

Referenced by EcalPerEvtLaserAnalyzer.endJob(), EcalLaserAnalyzer2.endJob(), EcalLaserAnalyzer.endJob(), readLogFile(), and pos::PixelFEDTestDAC.writeXMLHeader().

40 
41 def getTime(line):
42  time = line.split(" ")[1]
43  time = time.split(":")
44  time = int(time[0])*60*60+int(time[1])*60+float(time[2])
45  return time
46 
47 #Stream states
48 kStarted=0
49 kFinished=1
50 
#----------------------------------------------
def edmStreamStallGrapher.printHelp ( )

Definition at line 4 of file edmStreamStallGrapher.py.

Referenced by createPDFImage().

4 
5 def printHelp():
6  s = """Purpose: Convert a cmsRun log with Tracer info into a stream stall graph.
7 
8 edmStreamStallGrapher [-g] <log file name>
9 
10 Option: -g instead of ascii art, create a pdf file showing the work being done on each stream
11 
12 To Use: Add the Tracer Service to the cmsRun job you want to check for stream stalls.
13  Make sure to use the 'printTimstamps' option
14  cms.Service("Tracer", printTimestamps = cms.untracked.bool(True))
15  After running the job, execute this script and pass the name of the log file to the
16  script as the only command line argument.
17 
18 To Read: The script will then print an 'ASCII art' stall graph which consists of the name of
19  the module which either started or stopped running on a stream, and the state of each
20  stream at that moment in time and if the module just started, you will also see the
21  amount of time on that stream between the previous module finishing and this module starting.
22  The state of a stream is represented by a symbol:
23  blank (" ") the stream is currently running a module
24  line ("|") the stream is waiting to run a module
25  minus ("-") the stream has just finished waiting and is starting a module
26  plus ("+") the stream just finished running a module
27  If a module had to wait more than 0.1 seconds, the end of the line will have "STALLED".
28  Once the first 4 events have finished processing, the program prints "FINISH INIT".
29  This is useful if one wants to ignore stalled caused by startup actions, e.g. reading
30  conditions.
31 
32  Once the graph is completed, the program outputs the list of modules which had
33  the greatest total stall times. The list is sorted by total stall time and
34  written in descending order. In addition, the list of all stall times for the
35  module is given.
36 """
37  print s
38 
39 
#----------------------------------------------
def edmStreamStallGrapher.printStalledModulesInOrder (   stalledModules)

Definition at line 151 of file edmStreamStallGrapher.py.

References join(), and list().

151 
152 def printStalledModulesInOrder(stalledModules):
153  priorities = list()
154  maxNameSize = 0
155  for n,t in stalledModules.iteritems():
156  nameLength = len(n)
157  if nameLength > maxNameSize:
158  maxNameSize = nameLength
159  t.sort(reverse=True)
160  priorities.append((n,sum(t),t))
161 
162  def sumSort(i,j):
163  return cmp(i[1],j[1])
164  priorities.sort(cmp=sumSort, reverse=True)
165 
166  nameColumn = "Stalled Module"
167  if len(nameColumn) > maxNameSize:
168  maxNameSize = len(nameColumn)
169 
170  stallColumn = "Tot Stall Time"
171  stallColumnLength = len(stallColumn)
172 
173  print "%-*s" % (maxNameSize, nameColumn), "%-*s"%(stallColumnLength,stallColumn), " Stall Times"
174  for n,s,t in priorities:
175  paddedName = "%-*s:" % (maxNameSize,n)
176  print paddedName, "%-*.2f"%(stallColumnLength,s), ", ".join([ "%.2f"%x for x in t])
177 
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def edmStreamStallGrapher.readLogFile (   fileName)

Definition at line 51 of file edmStreamStallGrapher.py.

References getTime(), and list().

51 
52 def readLogFile(fileName):
53  f = open(fileName,"r")
54 
55  processingSteps = list()
56  numStreams = 0
57  maxNameSize = 0
58  startTime = 0
59  foundEventToStartFrom = False
60  for l in f:
61  if not foundEventToStartFrom:
62  if l.find("event = 5") != -1:
63  foundEventToStartFrom = True
64  stream = int( l[l.find("stream = ")+9])
65  processingSteps.append(("FINISH INIT",kFinished,stream,getTime(l)-startTime,False))
66  if l.find("processing event for module") != -1:
67  time = getTime(l)
68  if startTime == 0:
69  startTime = time
70  time = time - startTime
71  trans = kStarted
72  stream = 0
73  delayed = False
74  if l.find("finished:") != -1:
75  trans = kFinished
76  stream = int( l[l.find("stream = ")+9])
77  name = l.split("'")[1]
78  if l.find("delayed") != -1:
79  delayed = True
80  if len(name) > maxNameSize:
81  maxNameSize = len(name)
82  processingSteps.append((name,trans,stream,time,delayed))
83  if stream > numStreams:
84  numStreams = stream
85  f.close()
86  return (processingSteps,numStreams,maxNameSize)
87 
88 
#----------------------------------------------
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run

Variable Documentation

edmStreamStallGrapher.doGraphic = False

Definition at line 264 of file edmStreamStallGrapher.py.

list edmStreamStallGrapher.fileName = sys.argv[-1]

Definition at line 271 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.stalledModules = findStalledModules(processingSteps, numStreams)

Definition at line 274 of file edmStreamStallGrapher.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5