test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
edmStreamStallGrapher Namespace Reference

Functions

def createAsciiImage
 
def createPDFImage
 
def findStalledModules
 
def getTime
 
def printHelp
 
def printStalledModulesInOrder
 
def readLogFile
 

Variables

 doGraphic = False
 
list fileName = sys.argv[-1]
 
string kSourceFindEvent = "sourceFindEvent"
 
tuple stalledModules = findStalledModules(processingSteps, numStreams)
 

Function Documentation

def edmStreamStallGrapher.createAsciiImage (   processingSteps,
  numStreams,
  maxNameSize 
)

Definition at line 146 of file edmStreamStallGrapher.py.

146 
147 def createAsciiImage(processingSteps, numStreams, maxNameSize):
148  #print processingSteps
149  #exit(1)
150  streamState = [1]*(numStreams+1)
151  streamTime = [0]*(numStreams+1)
152  #lastTime = 0
153  seenInit = False
154  previousStartWasADelayed = [False]*(numStreams+1)
155  for n,trans,s,time,delayed in processingSteps:
156  if n == "FINISH INIT":
157  seenInit = True
158  continue
159  oldState = streamState[s]
160  streamState[s]=trans
161  waitTime = None
162  if trans == kStarted:
163  if delayed or (n == kSourceFindEvent):
164  previousStartWasADelayed[s] = True
165  streamTime[s] = time
166  continue
167  else:
168  previousStartWasADelayed[s] = False
169  waitTime = time - streamTime[s]
170  streamState[s]=2
171  else:
172  if delayed or (n == kSourceFindEvent):
173  if previousStartWasADelayed[s]:
174  if n != kSourceFindEvent:
175  n="source"
176  waitTime = time - streamTime[s]
177  streamState[s]=2
178  previousStartWasADelayed[s] = False
179  else:
180  streamTime[s] = time
181  continue
182  if oldState != trans:
183  streamState[s]=3
184  streamTime[s] = time
185  states = "%-*s: " % (maxNameSize,n)
186  for state in streamState:
187  if state == 0:
188  states +=" "
189  elif state == 1:
190  states +="|"
191  elif state == 2:
192  states +="-"
193  elif state == 3:
194  states +="+"
195  if waitTime is not None:
196  states += " %.2f"% waitTime
197  if waitTime > 0.1 and seenInit:
198  states += " STALLED "+str(time)+" "+str(s)
199 
200  print states
201  streamState[s]=trans
202  return stalledModules
203 
#----------------------------------------------
def edmStreamStallGrapher.createPDFImage (   processingSteps,
  numStreams,
  stalledModuleInfo 
)

Definition at line 231 of file edmStreamStallGrapher.py.

References funct.abs(), bitset_utilities.append(), cmsRelvalreport.exit, and printHelp().

231 
232 def createPDFImage(processingSteps, numStreams, stalledModuleInfo):
233  import matplotlib.pyplot as plt
234 
235  previousStartWasADelayed=[False]*(numStreams+1)
236  streamTime = [0]*(numStreams+1)
237  streamStartTimes = [ [] for x in xrange(numStreams+1)]
238  streamColors = [[] for x in xrange(numStreams+1)]
239 
240  stalledModuleNames = [ x for x in stalledModuleInfo.iterkeys()]
241 
242  streamStartTimes = [ [] for x in xrange(numStreams+1)]
243  streamColors = [[] for x in xrange(numStreams+1)]
244 
245  for n,trans,s,time,delayed in processingSteps:
246  if n == "FINISH INIT":
247  continue
248  if trans == kStarted:
249  previousStartWasADelayed[s] = delayed
250  streamTime[s] = time
251  else:
252  c="green"
253  if delayed:
254  if previousStartWasADelayed[s]:
255  #this was time for a source
256  c="orange"
257  previousStartWasADelayed[s] = False
258  else:
259  continue
260  if n == kSourceFindEvent:
261  c = "orange"
262  streamStartTimes[s].append((streamTime[s],time-streamTime[s]))
263  if c == "green" and n in stalledModuleNames:
264  c="red"
265  #elif len(streamColors[s]) %2:
266  # c="blue"
267  streamColors[s].append(c)
268 
269  #consolodate contiguous blocks with the same color
270  # this drastically reduces the size of the pdf file
271  oldStreamTimes = streamStartTimes
272  oldStreamColors = streamColors
273 
274  streamStartTimes = [ [] for x in xrange(numStreams+1)]
275  streamColors = [[] for x in xrange(numStreams+1)]
276 
277  for s in xrange(numStreams+1):
278  streamStartTimes[s].append(oldStreamTimes[s][0])
279  streamColors[s].append(oldStreamColors[s][0])
280  lastStartTime,lastTimeLength = oldStreamTimes[s][0]
281  lastColor = oldStreamColors[s][0]
282  for i in xrange(1, len(oldStreamTimes[s])):
283  start,length = oldStreamTimes[s][i]
284  color = oldStreamColors[s][i]
285  #use a millisecond tolerance to avoid rounding
286  if color == lastColor and abs(lastStartTime+lastTimeLength-start)<0.001:
287  lastTimeLength += length
288  else:
289  streamStartTimes[s].append((lastStartTime,lastTimeLength))
290  streamColors[s].append(lastColor)
291  lastStartTime = start
292  lastTimeLength = length
293  lastColor = color
294  streamStartTimes[s].append((lastStartTime,lastTimeLength))
295  streamColors[s].append(lastColor)
296 
297  fig, ax = plt.subplots()
298  ax.set_xlabel("Time (sec)")
299  ax.set_ylabel("Stream ID")
300 
301  i=1
302  for s in xrange(numStreams+1):
303  t = streamStartTimes[s]
304  ax.broken_barh(t,(i-0.4,0.8),facecolors=streamColors[s],edgecolors=streamColors[s],linewidth=0)
305  i=i+1
306 
307  #add key .1, .3, .7
308  fig.text(0.1, 0.95, "modules running", color = "green", horizontalalignment = 'left')
309  fig.text(0.5, 0.95, "stalled module running", color = "red", horizontalalignment = 'center')
310  fig.text(0.9, 0.95, "read from input", color = "orange", horizontalalignment = 'right')
311  plt.savefig("stall.pdf")
312 
313 
314 
#=======================================
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
funct::abs
Abs< T >::type abs(const T &t)
Definition: Abs.h:22
def edmStreamStallGrapher.findStalledModules (   processingSteps,
  numStreams 
)

Definition at line 117 of file edmStreamStallGrapher.py.

117 
118 def findStalledModules(processingSteps, numStreams):
119  streamTime = [0]*(numStreams+1)
120  stalledModules = {}
121  previousStartWasADelayed = [False]*(numStreams+1)
122  for n,trans,s,time,delayed in processingSteps:
123  waitTime = None
124  if trans == kStarted:
125  if delayed or (n == kSourceFindEvent):
126  previousStartWasADelayed[s] = True
127  streamTime[s] = time
128  else:
129  previousStartWasADelayed[s] = False
130  waitTime = time - streamTime[s]
131  else:
132  if (delayed or (n == kSourceFindEvent)) and previousStartWasADelayed[s]:
133  #for a source we only know the combined time for waiting and running
134  if n != kSourceFindEvent:
135  n = "source"
136  waitTime = time - streamTime[s]
137  previousStartWasADelayed[s] = False
138  streamTime[s] = time
139  if waitTime is not None:
140  if waitTime > 0.1:
141  t = stalledModules.setdefault(n,[])
142  t.append(waitTime)
143  return stalledModules
144 
145 
#----------------------------------------------
def edmStreamStallGrapher.getTime (   line)

Definition at line 40 of file edmStreamStallGrapher.py.

Referenced by EcalPerEvtLaserAnalyzer.endJob(), EcalLaserAnalyzer2.endJob(), EcalLaserAnalyzer.endJob(), readLogFile(), and pos::PixelFEDTestDAC.writeXMLHeader().

40 
41 def getTime(line):
42  time = line.split(" ")[1]
43  time = time.split(":")
44  time = int(time[0])*60*60+int(time[1])*60+float(time[2])
45  return time
46 
47 #Stream states
48 kStarted=0
49 kFinished=1
50 
#Special names
def edmStreamStallGrapher.printHelp ( )

Definition at line 4 of file edmStreamStallGrapher.py.

Referenced by createPDFImage().

4 
5 def printHelp():
6  s = """Purpose: Convert a cmsRun log with Tracer info into a stream stall graph.
7 
8 edmStreamStallGrapher [-g] <log file name>
9 
10 Option: -g instead of ascii art, create a pdf file showing the work being done on each stream
11 
12 To Use: Add the Tracer Service to the cmsRun job you want to check for stream stalls.
13  Make sure to use the 'printTimstamps' option
14  cms.Service("Tracer", printTimestamps = cms.untracked.bool(True))
15  After running the job, execute this script and pass the name of the log file to the
16  script as the only command line argument.
17 
18 To Read: The script will then print an 'ASCII art' stall graph which consists of the name of
19  the module which either started or stopped running on a stream, and the state of each
20  stream at that moment in time and if the module just started, you will also see the
21  amount of time on that stream between the previous module finishing and this module starting.
22  The state of a stream is represented by a symbol:
23  blank (" ") the stream is currently running a module
24  line ("|") the stream is waiting to run a module
25  minus ("-") the stream has just finished waiting and is starting a module
26  plus ("+") the stream just finished running a module
27  If a module had to wait more than 0.1 seconds, the end of the line will have "STALLED".
28  Once the first 4 events have finished processing, the program prints "FINISH INIT".
29  This is useful if one wants to ignore stalled caused by startup actions, e.g. reading
30  conditions.
31 
32  Once the graph is completed, the program outputs the list of modules which had
33  the greatest total stall times. The list is sorted by total stall time and
34  written in descending order. In addition, the list of all stall times for the
35  module is given.
36 """
37  print s
38 
39 
#----------------------------------------------
def edmStreamStallGrapher.printStalledModulesInOrder (   stalledModules)

Definition at line 204 of file edmStreamStallGrapher.py.

References join(), and list().

204 
205 def printStalledModulesInOrder(stalledModules):
206  priorities = list()
207  maxNameSize = 0
208  for n,t in stalledModules.iteritems():
209  nameLength = len(n)
210  if nameLength > maxNameSize:
211  maxNameSize = nameLength
212  t.sort(reverse=True)
213  priorities.append((n,sum(t),t))
214 
215  def sumSort(i,j):
216  return cmp(i[1],j[1])
217  priorities.sort(cmp=sumSort, reverse=True)
218 
219  nameColumn = "Stalled Module"
220  if len(nameColumn) > maxNameSize:
221  maxNameSize = len(nameColumn)
222 
223  stallColumn = "Tot Stall Time"
224  stallColumnLength = len(stallColumn)
225 
226  print "%-*s" % (maxNameSize, nameColumn), "%-*s"%(stallColumnLength,stallColumn), " Stall Times"
227  for n,s,t in priorities:
228  paddedName = "%-*s:" % (maxNameSize,n)
229  print paddedName, "%-*.2f"%(stallColumnLength,s), ", ".join([ "%.2f"%x for x in t])
230 
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def edmStreamStallGrapher.readLogFile (   fileName)

Definition at line 54 of file edmStreamStallGrapher.py.

References getTime(), and list().

54 
55 def readLogFile(fileName):
56  f = open(fileName,"r")
57 
58  processingSteps = list()
59  numStreams = 0
60  maxNameSize = 0
61  startTime = 0
62  foundEventToStartFrom = False
63  for l in f:
64  if not foundEventToStartFrom:
65  if l.find("event = 5") != -1:
66  foundEventToStartFrom = True
67  stream = int( l[l.find("stream = ")+9])
68  processingSteps.append(("FINISH INIT",kFinished,stream,getTime(l)-startTime,False))
69  if l.find("processing event :") != -1:
70  time = getTime(l)
71  if startTime == 0:
72  startTime = time
73  time = time - startTime
74  delayed = False
75  streamIndex = l.find("stream = ")
76  stream = int( l[streamIndex+9:l.find(" ",streamIndex+10)])
77  name = kSourceFindEvent
78  trans = kStarted
79  #the start of an event is the end of the framework part
80  if l.find("starting:") != -1:
81  trans = kFinished
82  processingSteps.append((name,trans,stream,time,delayed))
83  if stream > numStreams:
84  numStreams = stream
85  if l.find("processing event for module") != -1:
86  time = getTime(l)
87  if startTime == 0:
88  startTime = time
89  time = time - startTime
90  trans = kStarted
91  stream = 0
92  delayed = False
93  if l.find("finished:") != -1:
94  trans = kFinished
95  streamIndex = l.find("stream = ")
96  stream = int( l[streamIndex+9:l.find(" ",streamIndex+10)])
97  name = l.split("'")[1]
98  if l.find("delayed") != -1:
99  delayed = True
100  if len(name) > maxNameSize:
101  maxNameSize = len(name)
102  processingSteps.append((name,trans,stream,time,delayed))
103  if stream > numStreams:
104  numStreams = stream
105  f.close()
106  return (processingSteps,numStreams,maxNameSize)
107 
108 
109 #----------------------------------------------
110 # Patterns:
111 #
112 # source: The source is identified by having a 'starting: delayed' followed immediately by a 'finished: delayed'
113 # the total time is actually the wait time plus the time the source was doing work
114 # scheduled module: This is identified by no delays before call to 'starting' and then immediate call to 'finished'
115 # unscheduled module: Has a 'starting: delayed' followed by a 'starting' followed by a 'finished' the a 'finished: delayed'
116 #
#
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run

Variable Documentation

edmStreamStallGrapher.doGraphic = False

Definition at line 322 of file edmStreamStallGrapher.py.

list edmStreamStallGrapher.fileName = sys.argv[-1]

Definition at line 329 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.kSourceFindEvent = "sourceFindEvent"

Definition at line 51 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.stalledModules = findStalledModules(processingSteps, numStreams)

Definition at line 332 of file edmStreamStallGrapher.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5