CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
edmStreamStallGrapher Namespace Reference

Classes

class  Point
 
class  RefCountSet
 
class  Stack
 
class  StallMonitorParser
 
class  StreamInfoElement
 
class  TracerParser
 

Functions

def adjacentDiff
 
def chooseParser
 
def consolidateContiguousBlocks
 
def createAsciiImage
 
def createModuleTiming
 
def createPDFImage
 
def findStalledModules
 
def getTime
 
def mergeContiguousBlocks
 
def parseTracerOutput
 
def plotPerStreamAboveFirstAndPrepareStack
 
def printHelp
 
def printStalledModulesInOrder
 
def processingStepsFromStallMonitorOutput
 
def readLogFile
 
def reduceSortedPoints
 

Variables

string action = 'store_true'
 
tuple args = parser.parse_args()
 
 checkOrder = args.order
 
string const = 'stall.pdf'
 
float default = 0.0
 
string dest = 'graph'
 
 displayExternalWork = args.external
 
 doGraphic = False
 
 doModuleTimings = False
 
tuple extension = pdfFile.split('.')
 
string help = 'file to process'
 
 inputFile = args.filename
 
string kSourceDelayedRead = "sourceDelayedRead"
 
string kSourceFindEvent = "sourceFindEvent"
 
string metavar = "'stall.pdf'"
 
string nargs = '?'
 
tuple parser
 
 pdfFile = args.graph
 
tuple reader = readLogFile(inputFile)
 
 setXAxis = False
 
 shownStacks = args.stack
 
string stackType = 'stack'
 
tuple stalledModules = findStalledModules(reader.processingSteps(), reader.numStreams)
 
tuple supported_filetypes = plt.figure()
 
tuple type = argparse.FileType('r')
 
 xLower = args.lowerxaxis
 
float xUpper = 0.0
 

Function Documentation

def edmStreamStallGrapher.adjacentDiff (   pairLists)

Definition at line 487 of file edmStreamStallGrapher.py.

Referenced by createPDFImage().

487 
488 def adjacentDiff(*pairLists):
489  points = []
490  for pairList in pairLists:
491  points += [Point(x[0], 1) for x in pairList if x[1] != 0]
492  points += [Point(sum(x),-1) for x in pairList if x[1] != 0]
493  points.sort(key=attrgetter('x'))
494  return points
def edmStreamStallGrapher.chooseParser (   inputFile)

Definition at line 299 of file edmStreamStallGrapher.py.

References beamvalidation.exit(), print(), and sistrip::SpyUtilities.range().

Referenced by readLogFile().

299 
300 def chooseParser(inputFile):
301 
302  firstLine = inputFile.readline().rstrip()
303  for i in range(3):
304  inputFile.readline()
305  # Often the Tracer log file starts with 4 lines not from the Tracer
306  fifthLine = inputFile.readline().rstrip()
307  inputFile.seek(0) # Rewind back to beginning
308  if (firstLine.find("# Transition") != -1) or (firstLine.find("# Step") != -1):
309  print("> ... Parsing StallMonitor output.")
310  return StallMonitorParser
311 
312  if firstLine.find("++") != -1 or fifthLine.find("++") != -1:
313  global kTracerInput
314  kTracerInput = True
315  print("> ... Parsing Tracer output.")
316  return TracerParser
317  else:
318  inputFile.close()
319  print("Unknown input format.")
320  exit(1)
321 
#----------------------------------------------
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def edmStreamStallGrapher.consolidateContiguousBlocks (   numStreams,
  streamInfo 
)

Definition at line 525 of file edmStreamStallGrapher.py.

References bitset_utilities.append(), sistrip::SpyUtilities.range(), and BeamSpotPI.unpack().

Referenced by createPDFImage().

525 
526 def consolidateContiguousBlocks(numStreams, streamInfo):
527  oldStreamInfo = streamInfo
528  streamInfo = [[] for x in range(numStreams)]
529 
530  for s in range(numStreams):
531  if oldStreamInfo[s]:
532  lastStartTime,lastTimeLength,lastColor = oldStreamInfo[s][0].unpack()
533  for info in oldStreamInfo[s][1:]:
534  start,length,color = info.unpack()
535  if color == lastColor and lastStartTime+lastTimeLength == start:
536  lastTimeLength += length
537  else:
538  streamInfo[s].append(StreamInfoElement(lastStartTime,lastTimeLength,lastColor))
539  lastStartTime = start
540  lastTimeLength = length
541  lastColor = color
542  streamInfo[s].append(StreamInfoElement(lastStartTime,lastTimeLength,lastColor))
543 
544  return streamInfo
545 
546 #----------------------------------------------
547 # Consolidating contiguous blocks with the same color drastically
548 # reduces the size of the pdf file. Same functionality as the
# previous function, but with slightly different implementation.
BeamSpotPI::unpack
std::pair< unsigned int, unsigned int > unpack(cond::Time_t since)
Definition: BeamSpotPayloadInspectorHelper.h:26
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
def edmStreamStallGrapher.createAsciiImage (   processingSteps,
  numStreams,
  maxNameSize 
)

Definition at line 393 of file edmStreamStallGrapher.py.

References print(), sistrip::SpyUtilities.range(), and str.

393 
394 def createAsciiImage(processingSteps, numStreams, maxNameSize):
395  streamTime = [0]*numStreams
396  streamState = [0]*numStreams
397  modulesActiveOnStreams = [{} for x in range(numStreams)]
398  for n,trans,s,time,isEvent in processingSteps:
399  waitTime = None
400  modulesActiveOnStream = modulesActiveOnStreams[s]
401  if trans == kPrefetchEnd:
402  modulesActiveOnStream[n] = time
403  continue
404  elif trans == kStartedAcquire or trans == kStarted:
405  if n in modulesActiveOnStream:
406  waitTime = time - modulesActiveOnStream[n]
407  modulesActiveOnStream.pop(n, None)
408  streamState[s] +=1
409  elif trans == kFinishedAcquire or trans == kFinished:
410  streamState[s] -=1
411  streamTime[s] = time
412  elif trans == kStartedSourceDelayedRead:
413  if streamState[s] == 0:
414  waitTime = time - streamTime[s]
415  elif trans == kStartedSource:
416  modulesActiveOnStream.clear()
417  elif trans == kFinishedSource or trans == kFinishedSourceDelayedRead:
418  streamTime[s] = time
419  states = "%-*s: " % (maxNameSize,n)
420  if trans == kStartedAcquire or trans == kStarted or trans == kStartedSourceDelayedRead or trans == kStartedSource:
421  states +="+ "
422  else:
423  states +="- "
424  for index, state in enumerate(streamState):
425  if n==kSourceFindEvent and index == s:
426  states +="* "
427  else:
428  states +=str(state)+" "
429  states += " -- " + str(time/1000.) + " " + str(s) + " "
430  if waitTime is not None:
431  states += " %.2f"% (waitTime/1000.)
432  if waitTime > kStallThreshold:
433  states += " STALLED"
434 
435  print(states)
436 
#----------------------------------------------
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
str
#define str(s)
Definition: TestProcessor.cc:54
def edmStreamStallGrapher.createModuleTiming (   processingSteps,
  numStreams 
)

Definition at line 370 of file edmStreamStallGrapher.py.

References bitset_utilities.append(), and sistrip::SpyUtilities.range().

370 
371 def createModuleTiming(processingSteps, numStreams):
372  import json
373  streamTime = [0]*numStreams
374  streamState = [0]*numStreams
375  moduleTimings = defaultdict(list)
376  modulesActiveOnStream = [defaultdict(int) for x in range(numStreams)]
377  for n,trans,s,time,isEvent in processingSteps:
378  waitTime = None
379  modulesOnStream = modulesActiveOnStream[s]
380  if isEvent:
381  if trans == kStarted:
382  streamState[s] = 1
383  modulesOnStream[n]=time
384  elif trans == kFinished:
385  waitTime = time - modulesOnStream[n]
386  modulesOnStream.pop(n, None)
387  streamState[s] = 0
388  moduleTimings[n].append(float(waitTime/1000.))
389 
390  with open('module-timings.json', 'w') as outfile:
391  outfile.write(json.dumps(moduleTimings, indent=4))
392 
#----------------------------------------------
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
def edmStreamStallGrapher.createPDFImage (   pdfFile,
  shownStacks,
  processingSteps,
  numStreams,
  stalledModuleInfo,
  displayExternalWork,
  checkOrder,
  setXAxis,
  xLower,
  xUpper 
)

Definition at line 618 of file edmStreamStallGrapher.py.

References PVValHelper.add(), adjacentDiff(), bitset_utilities.append(), consolidateContiguousBlocks(), SiStripPI.max, mergeContiguousBlocks(), plotPerStreamAboveFirstAndPrepareStack(), print(), sistrip::SpyUtilities.range(), reduceSortedPoints(), MatrixUtil.remove(), and ComparisonHelper.zip().

618 
619 def createPDFImage(pdfFile, shownStacks, processingSteps, numStreams, stalledModuleInfo, displayExternalWork, checkOrder, setXAxis, xLower, xUpper):
620 
621  stalledModuleNames = set([x for x in iter(stalledModuleInfo)])
622  streamLowestRow = [[] for x in range(numStreams)]
623  modulesActiveOnStreams = [RefCountSet() for x in range(numStreams)]
624  acquireActiveOnStreams = [set() for x in range(numStreams)]
625  externalWorkOnStreams = [set() for x in range(numStreams)]
626  previousFinishTime = [None for x in range(numStreams)]
627  streamRunningTimes = [[] for x in range(numStreams)]
628  streamExternalWorkRunningTimes = [[] for x in range(numStreams)]
629  maxNumberOfConcurrentModulesOnAStream = 1
630  externalWorkModulesInJob = False
631  previousTime = [0 for x in range(numStreams)]
632 
633  # The next five variables are only used to check for out of order transitions
634  finishBeforeStart = [set() for x in range(numStreams)]
635  finishAcquireBeforeStart = [set() for x in range(numStreams)]
636  countSource = [0 for x in range(numStreams)]
637  countDelayedSource = [0 for x in range(numStreams)]
638  countExternalWork = [defaultdict(int) for x in range(numStreams)]
639 
640  timeOffset = None
641  for n,trans,s,time,isEvent in processingSteps:
642  if timeOffset is None:
643  timeOffset = time
644  startTime = None
645  time -=timeOffset
646  # force the time to monotonically increase on each stream
647  if time < previousTime[s]:
648  time = previousTime[s]
649  previousTime[s] = time
650 
651  activeModules = modulesActiveOnStreams[s]
652  acquireModules = acquireActiveOnStreams[s]
653  externalWorkModules = externalWorkOnStreams[s]
654 
655  if trans == kStarted or trans == kStartedSourceDelayedRead or trans == kStartedAcquire or trans == kStartedSource :
656  if checkOrder:
657  # Note that the code which checks the order of transitions assumes that
658  # all the transitions exist in the input. It is checking only for order
659  # problems, usually a start before a finish. Problems are fixed and
660  # silently ignored. Nothing gets plotted for transitions that are
661  # in the wrong order.
662  if trans == kStarted:
663  countExternalWork[s][n] -= 1
664  if n in finishBeforeStart[s]:
665  finishBeforeStart[s].remove(n)
666  continue
667  elif trans == kStartedAcquire:
668  if n in finishAcquireBeforeStart[s]:
669  finishAcquireBeforeStart[s].remove(n)
670  continue
671 
672  if trans == kStartedSourceDelayedRead:
673  countDelayedSource[s] += 1
674  if countDelayedSource[s] < 1:
675  continue
676  elif trans == kStartedSource:
677  countSource[s] += 1
678  if countSource[s] < 1:
679  continue
680 
681  moduleNames = activeModules.copy()
682  moduleNames.update(acquireModules)
683  if trans == kStartedAcquire:
684  acquireModules.add(n)
685  else:
686  activeModules.add(n)
687  streamRunningTimes[s].append(Point(time,1))
688  if moduleNames or externalWorkModules:
689  startTime = previousFinishTime[s]
690  previousFinishTime[s] = time
691 
692  if trans == kStarted and n in externalWorkModules:
693  externalWorkModules.remove(n)
694  streamExternalWorkRunningTimes[s].append(Point(time, -1))
695  else:
696  nTotalModules = len(activeModules) + len(acquireModules) + len(externalWorkModules)
697  maxNumberOfConcurrentModulesOnAStream = max(maxNumberOfConcurrentModulesOnAStream, nTotalModules)
698  elif trans == kFinished or trans == kFinishedSourceDelayedRead or trans == kFinishedAcquire or trans == kFinishedSource :
699  if checkOrder:
700  if trans == kFinished:
701  if n not in activeModules:
702  finishBeforeStart[s].add(n)
703  continue
704 
705  if trans == kFinishedSourceDelayedRead:
706  countDelayedSource[s] -= 1
707  if countDelayedSource[s] < 0:
708  continue
709  elif trans == kFinishedSource:
710  countSource[s] -= 1
711  if countSource[s] < 0:
712  continue
713 
714  if trans == kFinishedAcquire:
715  if checkOrder:
716  countExternalWork[s][n] += 1
717  if displayExternalWork:
718  externalWorkModulesInJob = True
719  if (not checkOrder) or countExternalWork[s][n] > 0:
720  externalWorkModules.add(n)
721  streamExternalWorkRunningTimes[s].append(Point(time,+1))
722  if checkOrder and n not in acquireModules:
723  finishAcquireBeforeStart[s].add(n)
724  continue
725  streamRunningTimes[s].append(Point(time,-1))
726  startTime = previousFinishTime[s]
727  previousFinishTime[s] = time
728  moduleNames = activeModules.copy()
729  moduleNames.update(acquireModules)
730 
731  if trans == kFinishedAcquire:
732  acquireModules.remove(n)
733  elif trans == kFinishedSourceDelayedRead:
734  if countDelayedSource[s] == 0:
735  activeModules.remove(n)
736  elif trans == kFinishedSource:
737  if countSource[s] == 0:
738  activeModules.remove(n)
739  else:
740  activeModules.remove(n)
741 
742  if startTime is not None:
743  c="green"
744  if not isEvent:
745  c="limegreen"
746  if not moduleNames:
747  c = "darkviolet"
748  elif (kSourceDelayedRead in moduleNames) or (kSourceFindEvent in moduleNames):
749  c = "orange"
750  else:
751  for n in moduleNames:
752  if n in stalledModuleNames:
753  c="red"
754  break
755  streamLowestRow[s].append(StreamInfoElement(startTime, time-startTime, c))
756  streamLowestRow = consolidateContiguousBlocks(numStreams, streamLowestRow)
757 
758  nr = 1
759  if shownStacks:
760  nr += 1
761  fig, ax = plt.subplots(nrows=nr, squeeze=True)
762  axStack = None
763  if shownStacks:
764  [xH,yH] = fig.get_size_inches()
765  fig.set_size_inches(xH,yH*4/3)
766  ax = plt.subplot2grid((4,1),(0,0), rowspan=3)
767  axStack = plt.subplot2grid((4,1),(3,0))
768 
769  ax.set_xlabel("Time (sec)")
770  ax.set_ylabel("Stream ID")
771  ax.set_ylim(-0.5,numStreams-0.5)
772  ax.yaxis.set_ticks(range(numStreams))
773  if (setXAxis):
774  ax.set_xlim((xLower, xUpper))
775 
776  height = 0.8/maxNumberOfConcurrentModulesOnAStream
777  allStackTimes={'green': [],'limegreen':[], 'red': [], 'blue': [], 'orange': [], 'darkviolet': []}
778  for iStream,lowestRow in enumerate(streamLowestRow):
779  times=[(x.begin/1000000., x.delta/1000000.) for x in lowestRow] # Scale from microsec to sec.
780  colors=[x.color for x in lowestRow]
781  # for each stream, plot the lowest row
782  ax.broken_barh(times,(iStream-0.4,height),facecolors=colors,edgecolors=colors,linewidth=0)
783  # record them also for inclusion in the stack plot
784  # the darkviolet ones get counted later so do not count them here
785  for info in lowestRow:
786  if not info.color == 'darkviolet':
787  allStackTimes[info.color].append((info.begin, info.delta))
788 
789  # Now superimpose the number of concurrently running modules on to the graph.
790  if maxNumberOfConcurrentModulesOnAStream > 1 or externalWorkModulesInJob:
791 
792  for i,perStreamRunningTimes in enumerate(streamRunningTimes):
793 
794  perStreamTimesWithExtendedWork = list(perStreamRunningTimes)
795  perStreamTimesWithExtendedWork.extend(streamExternalWorkRunningTimes[i])
796 
797  plotPerStreamAboveFirstAndPrepareStack(perStreamTimesWithExtendedWork,
798  allStackTimes, ax, i, height,
799  streamHeightCut=2,
800  doPlot=True,
801  addToStackTimes=False,
802  color='darkviolet',
803  threadOffset=1)
804 
805  plotPerStreamAboveFirstAndPrepareStack(perStreamRunningTimes,
806  allStackTimes, ax, i, height,
807  streamHeightCut=2,
808  doPlot=True,
809  addToStackTimes=True,
810  color='blue',
811  threadOffset=1)
812 
813  plotPerStreamAboveFirstAndPrepareStack(streamExternalWorkRunningTimes[i],
814  allStackTimes, ax, i, height,
815  streamHeightCut=1,
816  doPlot=False,
817  addToStackTimes=True,
818  color='darkviolet',
819  threadOffset=0)
820 
821  if shownStacks:
822  print("> ... Generating stack")
823  stack = Stack()
824  for color in ['green','limegreen','blue','red','orange','darkviolet']:
825  tmp = allStackTimes[color]
826  tmp = reduceSortedPoints(adjacentDiff(tmp))
827  stack.update(color, tmp)
828 
829  for stk in reversed(stack.data):
830  color = stk[0]
831 
832  # Now arrange list in a manner that it can be grouped by the height of the block
833  height = 0
834  xs = []
835  for p1,p2 in zip(stk[1], stk[1][1:]):
836  height += p1.y
837  xs.append((p1.x, p2.x-p1.x, height))
838  xs.sort(key = itemgetter(2))
839  xs = mergeContiguousBlocks(xs)
840 
841  for height, xpairs in groupby(xs, itemgetter(2)):
842  finalxs = [(e[0]/1000000.,e[1]/1000000.) for e in xpairs]
843  # plot the stacked plot, one color and one height on each call to broken_barh
844  axStack.broken_barh(finalxs, (0, height), facecolors=color, edgecolors=color, linewidth=0)
845 
846  axStack.set_xlabel("Time (sec)");
847  axStack.set_ylabel("# modules");
848  axStack.set_xlim(ax.get_xlim())
849  axStack.tick_params(top='off')
850 
851  fig.text(0.1, 0.95, "modules running event", color = "green", horizontalalignment = 'left')
852  fig.text(0.1, 0.92, "modules running other", color = "limegreen", horizontalalignment = 'left')
853  fig.text(0.5, 0.95, "stalled module running", color = "red", horizontalalignment = 'center')
854  fig.text(0.9, 0.95, "read from input", color = "orange", horizontalalignment = 'right')
855  fig.text(0.5, 0.92, "multiple modules running", color = "blue", horizontalalignment = 'center')
856  if displayExternalWork:
857  fig.text(0.9, 0.92, "external work", color = "darkviolet", horizontalalignment = 'right')
858  print("> ... Saving to file: '{}'".format(pdfFile))
859  plt.savefig(pdfFile)
860 
#=======================================
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:39
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
PVValHelper::add
void add(std::map< std::string, TH1 * > &h, TH1 *hist)
Definition: PVValidationHelpers.cc:12
def edmStreamStallGrapher.findStalledModules (   processingSteps,
  numStreams 
)

Definition at line 337 of file edmStreamStallGrapher.py.

References sistrip::SpyUtilities.range().

337 
338 def findStalledModules(processingSteps, numStreams):
339  streamTime = [0]*numStreams
340  streamState = [0]*numStreams
341  stalledModules = {}
342  modulesActiveOnStream = [{} for x in range(numStreams)]
343  for n,trans,s,time,isEvent in processingSteps:
344 
345  waitTime = None
346  modulesOnStream = modulesActiveOnStream[s]
347  if trans == kPrefetchEnd:
348  modulesOnStream[n] = time
349  elif trans == kStarted or trans == kStartedAcquire:
350  if n in modulesOnStream:
351  waitTime = time - modulesOnStream[n]
352  modulesOnStream.pop(n, None)
353  streamState[s] +=1
354  elif trans == kFinished or trans == kFinishedAcquire:
355  streamState[s] -=1
356  streamTime[s] = time
357  elif trans == kStartedSourceDelayedRead:
358  if streamState[s] == 0:
359  waitTime = time - streamTime[s]
360  elif trans == kStartedSource:
361  modulesOnStream.clear()
362  elif trans == kFinishedSource or trans == kFinishedSourceDelayedRead:
363  streamTime[s] = time
364  if waitTime is not None:
365  if waitTime > kStallThreshold:
366  t = stalledModules.setdefault(n,[])
367  t.append(waitTime)
368  return stalledModules
369 
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
def edmStreamStallGrapher.getTime (   line)

Definition at line 200 of file edmStreamStallGrapher.py.

Referenced by EcalLaserAnalyzer.endJob(), EcalLaserAnalyzer2.endJob(), parseTracerOutput(), and pos::PixelFEDTestDAC.writeXMLHeader().

200 
201 def getTime(line):
202  time = line.split(" ")[1]
203  time = time.split(":")
204  time = int(time[0])*60*60+int(time[1])*60+float(time[2])
205  time = int(1000000*time) # convert to microseconds
206  return time
207 
208 #----------------------------------------------
209 # The next function parses the Tracer output.
210 # Here are some differences to consider if you use Tracer output
211 # instead of the StallMonitor output.
212 # - The time in the text of the Tracer output is not as precise
213 # as the StallMonitor (.01 s vs .001 s)
214 # - The MessageLogger bases the time on when the message printed
215 # and not when it was initially queued up to print which smears
216 # the accuracy of the times.
217 # - Both of the previous things can produce some strange effects
218 # in the output plots.
219 # - The file size of the Tracer text file is much larger.
220 # - The CPU work needed to parse the Tracer files is larger.
221 # - The Tracer log file is expected to have "++" in the first
222 # or fifth line. If there are extraneous lines at the beginning
223 # you have to remove them.
224 # - The ascii printout out will have one extraneous line
225 # near the end for the SourceFindEvent start.
226 # - The only advantage I can see is that you have only
227 # one output file to handle instead of two, the regular
228 # log file and the StallMonitor output.
229 # We might should just delete the Tracer option because it is
# clearly inferior ...
def edmStreamStallGrapher.mergeContiguousBlocks (   blocks)

Definition at line 549 of file edmStreamStallGrapher.py.

Referenced by createPDFImage(), and plotPerStreamAboveFirstAndPrepareStack().

549 
550 def mergeContiguousBlocks(blocks):
551  oldBlocks = blocks
552 
553  blocks = []
554  if not oldBlocks:
555  return blocks
556 
557  lastStartTime,lastTimeLength,lastHeight = oldBlocks[0]
558  for start,length,height in oldBlocks[1:]:
559  if height == lastHeight and lastStartTime+lastTimeLength == start:
560  lastTimeLength += length
561  else:
562  blocks.append((lastStartTime,lastTimeLength,lastHeight))
563  lastStartTime = start
564  lastTimeLength = length
565  lastHeight = height
566  blocks.append((lastStartTime,lastTimeLength,lastHeight))
567 
568  return blocks
569 
#----------------------------------------------
def edmStreamStallGrapher.parseTracerOutput (   f)

Definition at line 230 of file edmStreamStallGrapher.py.

References getTime(), and SiStripPI.max.

230 
231 def parseTracerOutput(f):
232  processingSteps = []
233  numStreams = 0
234  maxNameSize = 0
235  startTime = 0
236  streamsThatSawFirstEvent = set()
237  for l in f:
238  trans = None
239  # We estimate the start and stop of the source
240  # by the end of the previous event and start of
241  # the event. This is historical, probably because
242  # the Tracer output for the begin and end of the
243  # source event does not include the stream number.
244  if l.find("processing event :") != -1:
245  name = kSourceFindEvent
246  trans = kStartedSource
247  # the end of the source is estimated using the start of the event
248  if l.find("starting:") != -1:
249  trans = kFinishedSource
250  elif l.find("processing event for module") != -1:
251  trans = kStarted
252  if l.find("finished:") != -1:
253  if l.find("prefetching") != -1:
254  trans = kPrefetchEnd
255  else:
256  trans = kFinished
257  else:
258  if l.find("prefetching") != -1:
259  #skip this since we don't care about prefetch starts
260  continue
261  name = l.split("'")[1]
262  elif l.find("processing event acquire for module:") != -1:
263  trans = kStartedAcquire
264  if l.find("finished:") != -1:
265  trans = kFinishedAcquire
266  name = l.split("'")[1]
267  elif l.find("event delayed read from source") != -1:
268  trans = kStartedSourceDelayedRead
269  if l.find("finished:") != -1:
270  trans = kFinishedSourceDelayedRead
271  name = kSourceDelayedRead
272  if trans is not None:
273  time = getTime(l)
274  if startTime == 0:
275  startTime = time
276  time = time - startTime
277  streamIndex = l.find("stream = ")
278  stream = int(l[streamIndex+9:l.find(" ",streamIndex+10)])
279  maxNameSize = max(maxNameSize, len(name))
280 
281  if trans == kFinishedSource and not stream in streamsThatSawFirstEvent:
282  # This is wrong but there is no way to estimate the time better
283  # because there is no previous event for the first event.
284  processingSteps.append((name,kStartedSource,stream,time,True))
285  streamsThatSawFirstEvent.add(stream)
286 
287  processingSteps.append((name,trans,stream,time, True))
288  numStreams = max(numStreams, stream+1)
289 
290  f.close()
291  return (processingSteps,numStreams,maxNameSize)
def edmStreamStallGrapher.plotPerStreamAboveFirstAndPrepareStack (   points,
  allStackTimes,
  ax,
  stream,
  height,
  streamHeightCut,
  doPlot,
  addToStackTimes,
  color,
  threadOffset 
)

Definition at line 570 of file edmStreamStallGrapher.py.

References mergeContiguousBlocks(), reduceSortedPoints(), and ComparisonHelper.zip().

Referenced by createPDFImage().

570 
571 def plotPerStreamAboveFirstAndPrepareStack(points, allStackTimes, ax, stream, height, streamHeightCut, doPlot, addToStackTimes, color, threadOffset):
572  points = sorted(points, key=attrgetter('x'))
573  points = reduceSortedPoints(points)
574  streamHeight = 0
575  preparedTimes = []
576  for t1,t2 in zip(points, points[1:]):
577  streamHeight += t1.y
578  # We make a cut here when plotting because the first row for
579  # each stream was already plotted previously and we do not
580  # need to plot it again. And also we want to count things
581  # properly in allStackTimes. We want to avoid double counting
582  # or missing running modules and this is complicated because
583  # we counted the modules in the first row already.
584  if streamHeight < streamHeightCut:
585  continue
586  preparedTimes.append((t1.x,t2.x-t1.x, streamHeight))
587  preparedTimes.sort(key=itemgetter(2))
588  preparedTimes = mergeContiguousBlocks(preparedTimes)
589 
590  for nthreads, ts in groupby(preparedTimes, itemgetter(2)):
591  theTS = [(t[0],t[1]) for t in ts]
592  if doPlot:
593  theTimes = [(t[0]/1000000.,t[1]/1000000.) for t in theTS]
594  yspan = (stream-0.4+height,height*(nthreads-1))
595  ax.broken_barh(theTimes, yspan, facecolors=color, edgecolors=color, linewidth=0)
596  if addToStackTimes:
597  allStackTimes[color].extend(theTS*(nthreads-threadOffset))
598 
599 #----------------------------------------------
600 # The same ES module can have multiple Proxies running concurrently
# so we need to reference count the names of the active modules
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:39
def edmStreamStallGrapher.printHelp ( )

Definition at line 9 of file edmStreamStallGrapher.py.

9 
10 def printHelp():
11  s = '''
12 To Use: Add the StallMonitor Service to the cmsRun job you want to check for
13  stream stalls. Use something like this in the configuration:
14 
15  process.add_(cms.Service("StallMonitor", fileName = cms.untracked.string("stallMonitor.log")))
16 
17  After running the job, execute this script and pass the name of the
18  StallMonitor log file to the script.
19 
20  By default, the script will then print an 'ASCII art' stall graph
21  which consists of a line of text for each time a module or the
22  source stops or starts. Each line contains the name of the module
23  which either started or stopped running, and the number of modules
24  running on each stream at that moment in time. After that will be
25  the time and stream number. Then if a module just started, you
26  will also see the amount of time the module spent between finishing
27  its prefetching and starting. The state of a module is represented
28  by a symbol:
29 
30  plus ("+") the stream has just finished waiting and is starting a module
31  minus ("-") the stream just finished running a module
32 
33  If a module had to wait more than 0.1 seconds, the end of the line
34  will have "STALLED". Startup actions, e.g. reading conditions,
35  may affect results for the first few events.
36 
37  Using the command line arguments described above you can make the
38  program create a PDF file with actual graphs instead of the 'ASCII art'
39  output.
40 
41  Once the graph is completed, the program outputs the list of modules
42  which had the greatest total stall times. The list is sorted by
43  total stall time and written in descending order. In addition, the
44  list of all stall times for the module is given.
45 
46  There is an inferior alternative (an old obsolete way).
47  Instead of using the StallMonitor Service, you can use the
48  Tracer Service. Make sure to use the 'printTimestamps' option
49  cms.Service("Tracer", printTimestamps = cms.untracked.bool(True))
50  There are problems associated with this and it is not recommended.'''
51  return s
52 
53 kStallThreshold=100000 #in microseconds
54 kTracerInput=False
55 
56 #Stream states
57 kStarted=0
58 kFinished=1
59 kPrefetchEnd=2
60 kStartedAcquire=3
61 kFinishedAcquire=4
62 kStartedSource=5
63 kFinishedSource=6
64 kStartedSourceDelayedRead=7
65 kFinishedSourceDelayedRead=8
66 
#Special names
def edmStreamStallGrapher.printStalledModulesInOrder (   stalledModules)

Definition at line 437 of file edmStreamStallGrapher.py.

References join(), SiStripPI.max, and print().

437 
438 def printStalledModulesInOrder(stalledModules):
439  priorities = []
440  maxNameSize = 0
441  for name,t in stalledModules.items():
442  maxNameSize = max(maxNameSize, len(name))
443  t.sort(reverse=True)
444  priorities.append((name,sum(t),t))
445 
446  priorities.sort(key=lambda a: a[1], reverse=True)
447 
448  nameColumn = "Stalled Module"
449  maxNameSize = max(maxNameSize, len(nameColumn))
450 
451  stallColumn = "Tot Stall Time"
452  stallColumnLength = len(stallColumn)
453 
454  print("%-*s" % (maxNameSize, nameColumn), "%-*s"%(stallColumnLength,stallColumn), " Stall Times")
455  for n,s,t in priorities:
456  paddedName = "%-*s:" % (maxNameSize,n)
457  print(paddedName, "%-*.2f"%(stallColumnLength,s/1000.), ", ".join([ "%.2f"%(x/1000.) for x in t]))
458 
#--------------------------------------------------------
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def edmStreamStallGrapher.processingStepsFromStallMonitorOutput (   f,
  moduleNames,
  esModuleNames 
)

Definition at line 71 of file edmStreamStallGrapher.py.

Referenced by edmStreamStallGrapher.StallMonitorParser.processingSteps().

71 
72 def processingStepsFromStallMonitorOutput(f,moduleNames, esModuleNames):
73  for rawl in f:
74  l = rawl.strip()
75  if not l or l[0] == '#':
76  continue
77  (step,payload) = tuple(l.split(None,1))
78  payload=payload.split()
79 
80  # Ignore these
81  if step == 'E' or step == 'e':
82  continue
83 
84  # Payload format is:
85  # <stream id> <..other fields..> <time since begin job>
86  stream = int(payload[0])
87  time = int(payload[-1])
88  trans = None
89  isEvent = True
90 
91  name = None
92  # 'S' = begin of event creation in source
93  # 's' = end of event creation in source
94  if step == 'S' or step == 's':
95  name = kSourceFindEvent
96  trans = kStartedSource
97  # The start of an event is the end of the framework part
98  if step == 's':
99  trans = kFinishedSource
100  else:
101  # moduleID is the second payload argument for all steps below
102  moduleID = payload[1]
103 
104  # 'p' = end of module prefetching
105  # 'M' = begin of module processing
106  # 'm' = end of module processing
107  if step == 'p' or step == 'M' or step == 'm':
108  trans = kStarted
109  if step == 'p':
110  trans = kPrefetchEnd
111  elif step == 'm':
112  trans = kFinished
113  if step == 'm' or step == 'M':
114  isEvent = (int(payload[2]) == 0)
115  name = moduleNames[moduleID]
116 
117  # 'q' = end of esmodule prefetching
118  # 'N' = begin of esmodule processing
119  # 'n' = end of esmodule processing
120  if step == 'q' or step == 'N' or step == 'n':
121  trans = kStarted
122  if step == 'q':
123  trans = kPrefetchEnd
124  elif step == 'n':
125  trans = kFinished
126  if step == 'n' or step == 'N':
127  isEvent = (int(payload[2]) == 0)
128  name = esModuleNames[moduleID]
129 
130  # 'A' = begin of module acquire function
131  # 'a' = end of module acquire function
132  elif step == 'A' or step == 'a':
133  trans = kStartedAcquire
134  if step == 'a':
135  trans = kFinishedAcquire
136  name = moduleNames[moduleID]
137 
138  # Delayed read from source
139  # 'R' = begin of delayed read from source
140  # 'r' = end of delayed read from source
141  elif step == 'R' or step == 'r':
142  trans = kStartedSourceDelayedRead
143  if step == 'r':
144  trans = kFinishedSourceDelayedRead
145  name = kSourceDelayedRead
146 
147  if trans is not None:
148  yield (name,trans,stream,time, isEvent)
149 
150  return
def edmStreamStallGrapher.readLogFile (   inputFile)

Definition at line 322 of file edmStreamStallGrapher.py.

References chooseParser().

322 
323 def readLogFile(inputFile):
324  parseInput = chooseParser(inputFile)
325  return parseInput(inputFile)
326 
327 #----------------------------------------------
328 #
329 # modules: The time between prefetch finished and 'start processing' is
330 # the time it took to acquire any resources which is by definition the
331 # stall time.
332 #
333 # source: The source just records how long it spent doing work,
334 # not how long it was stalled. We can get a lower bound on the stall
335 # time for delayed reads by measuring the time the stream was doing
336 # no work up till the start of the source delayed read.
#
def edmStreamStallGrapher.reduceSortedPoints (   ps)

Definition at line 471 of file edmStreamStallGrapher.py.

Referenced by createPDFImage(), plotPerStreamAboveFirstAndPrepareStack(), and edmStreamStallGrapher.Stack.update().

471 
472 def reduceSortedPoints(ps):
473  if len(ps) < 2:
474  return ps
475  reducedPoints = []
476  tmp = Point(ps[0].x, ps[0].y)
477  for p in ps[1:]:
478  if tmp.x == p.x:
479  tmp.y += p.y
480  else:
481  reducedPoints.append(tmp)
482  tmp = Point(p.x, p.y)
483  reducedPoints.append(tmp)
484  reducedPoints = [p for p in reducedPoints if p.y != 0]
485  return reducedPoints
486 
# -------------------------------------------

Variable Documentation

string edmStreamStallGrapher.action = 'store_true'

Definition at line 882 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.args = parser.parse_args()

Definition at line 901 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.checkOrder = args.order

Definition at line 908 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.const = 'stall.pdf'

Definition at line 876 of file edmStreamStallGrapher.py.

float edmStreamStallGrapher.default = 0.0

Definition at line 896 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.dest = 'graph'

Definition at line 877 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.displayExternalWork = args.external

Definition at line 907 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.doGraphic = False

Definition at line 920 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.doModuleTimings = False

Definition at line 909 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.extension = pdfFile.split('.')

Definition at line 933 of file edmStreamStallGrapher.py.

Referenced by nanoaod::FlatTable.addExtension(), and ExternalLHEAsciiDumper.endRun().

string edmStreamStallGrapher.help = 'file to process'

Definition at line 872 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.inputFile = args.filename

Definition at line 904 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.kSourceDelayedRead = "sourceDelayedRead"

Definition at line 68 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.kSourceFindEvent = "sourceFindEvent"

Definition at line 67 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.metavar = "'stall.pdf'"

Definition at line 875 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.nargs = '?'

Definition at line 874 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.parser
Initial value:
1 = argparse.ArgumentParser(description='Convert a text file created by cmsRun into a stream stall graph.',
2  formatter_class=argparse.RawDescriptionHelpFormatter,
3  epilog=printHelp())

Definition at line 867 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.pdfFile = args.graph

Definition at line 905 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.reader = readLogFile(inputFile)

Definition at line 947 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.setXAxis = False

Definition at line 913 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.shownStacks = args.stack

Definition at line 906 of file edmStreamStallGrapher.py.

string edmStreamStallGrapher.stackType = 'stack'

Definition at line 495 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.stalledModules = findStalledModules(reader.processingSteps(), reader.numStreams)

Definition at line 951 of file edmStreamStallGrapher.py.

tuple edmStreamStallGrapher.supported_filetypes = plt.figure()

Definition at line 934 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.type = argparse.FileType('r')

Definition at line 871 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.xLower = args.lowerxaxis

Definition at line 918 of file edmStreamStallGrapher.py.

edmStreamStallGrapher.xUpper = 0.0

Definition at line 914 of file edmStreamStallGrapher.py.

Referenced by RPixChargeShare.RPixChargeShare().

Generated for CMSSW Reference Manual by doxygen 1.8.5