CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
generateEDF Namespace Reference

Classes

class  LumiInfo
 

LumiInfo Class

More...
 
class  LumiInfoCont
 

LumiInfoCont Class

More...
 

Functions

def loadEvents
 

General Functions

More...
 
def makeEDFplot
 

Variables

string action = 'store_true'
 
list allowedEDF = ['time', 'instLum', 'instIntLum']
 

## Main

More...
 
tuple cont = LumiInfoCont(args[0], **options.__dict__)
 load Luminosity info ## More...
 
string default = 'Empirical Distribution Function'
 
 done = False
 
string help = 'title of plot (default %default)'
 
tuple inputGroup = optparse.OptionGroup(parser, "Input Options")
 
tuple modeGroup = optparse.OptionGroup(parser, "Mode Options")
 
tuple nonSpaceRE = re.compile(r'\S')
 
tuple parser = optparse.OptionParser("Usage: %prog [options] lumi.csv events.txt output.png", description='Script for generating EDF curves. See https://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideGenerateEDF for more details.')
 
tuple pieces = sepRE.split(line)
 
tuple plotGroup = optparse.OptionGroup(parser, "Plot Options")
 
int prevRecLumi = 0
 
tuple rangeGroup = optparse.OptionGroup(parser, "Range Options")
 
int recLumIndex = 0
 
list recLumis = []
 look for which runs correspond to what total ## recorded integrated luminosity ## More...
 
tuple recLumValue = float(piece)
 
tuple sepRE = re.compile(r'[\s,;:]+')
 
string type = 'string'
 

Function Documentation

def generateEDF.loadEvents (   filename,
  cont,
  options 
)

General Functions

Definition at line 240 of file generateEDF.py.

References bitset_utilities.append(), and print().

241 def loadEvents (filename, cont, options):
242  eventsDict = {}
243  print("loading events from '%s'" % filename)
244  events = open (filename, 'r')
245  runIndex, lumiIndex, eventIndex, weightIndex = 0, 1, 2, 3
246  if options.relOrder:
247  lumiIndex, eventIndex = 2,1
248  minPieces = 3
249  totalWeight = 0.
250  if options.weights:
251  minPieces = 4
252  for line in events:
253  pieces = sepRE.split (line.strip())
254  if len (pieces) < minPieces:
255  if nonSpaceRE.search (line):
256  print("skipping", line)
257  continue
258  try:
259  run, lumi, event = int( pieces[runIndex] ), \
260  int( pieces[lumiIndex] ), \
261  int( pieces[eventIndex] )
262  except:
263  continue
264  key = (run, lumi)
265  if key not in cont:
266  if options.ignore:
267  print("Warning, %s is not found in the lumi information" \
268  % key.__str__())
269  continue
270  else:
271  raise RuntimeError("%s is not found in lumi information. Use '--ignoreNoLumiEvents' option to ignore these events and continue." \
272  % key.__str__())
273  if options.edfMode != 'time' and not cont[key].xingInfo:
274  if options.ignore:
275  print("Warning, %s does not have Xing information" \
276  % key.__str__())
277  continue
278  else:
279  raise RuntimeError("%s does not have Xing information. Use '--ignoreNoLumiEvents' option to ignore these events and continue." \
280  % key.__str__())
281  if options.weights:
282  weight = float (pieces[weightIndex])
283  else:
284  weight = 1
285  eventsDict.setdefault( key, []).append( (event, weight) )
286  totalWeight += weight
287  events.close()
288  return eventsDict, totalWeight
289 
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def loadEvents
General Functions
Definition: generateEDF.py:240
def generateEDF.makeEDFplot (   lumiCont,
  eventsDict,
  totalWeight,
  outputFile,
  options 
)

Definition at line 290 of file generateEDF.py.

References SiStripPI.max, SiStripPI.min, and print().

291 def makeEDFplot (lumiCont, eventsDict, totalWeight, outputFile, options):
292  # make TGraph
293  xVals = [0]
294  yVals = [0]
295  expectedVals = [0]
296  predVals = [0]
297  weight = 0
298  expectedChunks = []
299  ########################
300  ## Time Ordering Mode ##
301  ########################
302  if 'time' == options.edfMode:
303  # if we have a minimum run number, clear the lists
304  if lumiCont.minRun or lumiCont.minIntLum:
305  xVals = []
306  yVals = []
307  expectedVals = []
308  predVals = []
309  # loop over events
310  for key, eventList in sorted( eventsDict.items() ):
311  usePoints = True
312  # should we add this point?
313  if lumiCont.minRun and lumiCont.minRun > key[0] or \
314  lumiCont.maxRun and lumiCont.maxRun < key[0]:
315  usePoints = False
316  for event in eventList:
317  weight += event[1]
318  if not usePoints:
319  continue
320  factor = old_div(weight, totalWeight)
321  try:
322  intLum = lumiCont[key].totalRecorded
323  except:
324  raise RuntimeError("key %s not found in lumi information" \
325  % key.__str__())
326  if lumiCont.minIntLum and lumiCont.minIntLum > intLum or \
327  lumiCont.maxIntLum and lumiCont.maxIntLum < intLum:
328  continue
329  lumFrac = old_div(intLum, lumiCont.totalRecLum)
330  xVals.append( lumiCont[key].totalRecorded)
331  yVals.append (factor)
332  expectedVals.append (lumFrac)
333  predVals.append (lumFrac * options.pred)
334  # put on the last point if we aren't giving a maximum run
335  if not lumiCont.maxRun and not lumiCont.maxIntLum:
336  xVals.append (lumiCont.totalRecLum)
337  yVals.append (1)
338  expectedVals.append (1)
339  predVals.append (options.pred)
340  ####################
341  ## Reset Expected ##
342  ####################
343  if options.resetExpected:
344  slope = old_div((yVals[-1] - yVals[0]), (xVals[-1] - xVals[0]))
345  print("slope", slope)
346  for index, old in enumerate (expectedVals):
347  expectedVals[index] = yVals[0] + \
348  slope * (xVals[index] - xVals[0])
349  #############################################
350  ## Break Expected by Integrated Luminosity ##
351  #############################################
352  if options.breakExpectedIntLum:
353  breakExpectedIntLum = []
354  for chunk in options.breakExpectedIntLum:
355  pieces = sepRE.split (chunk)
356  try:
357  for piece in pieces:
358  breakExpectedIntLum.append( float(piece) )
359  except:
360  raise RuntimeError("'%s' from '%s' is not a valid float" \
361  % (piece, chunk))
362  breakExpectedIntLum.sort()
363  boundaries = []
364  breakIndex = 0
365  done = False
366  for index, xPos in enumerate (xVals):
367  if xPos > breakExpectedIntLum[breakIndex]:
368  boundaries.append (index)
369  while breakIndex < len (breakExpectedIntLum):
370  breakIndex += 1
371  if breakIndex >= len (breakExpectedIntLum):
372  done = True
373  break
374  # If this next position is different, than
375  # we're golden. Otherwise, let it go through
376  # the loop again.
377  if xPos <= breakExpectedIntLum[breakIndex]:
378  break
379  if done:
380  break
381  # do we have any boundaries?
382  if not boundaries:
383  raise RuntimeError("No values of 'breakExpectedIntLum' are in current range.")
384  # is the first boundary at 0? If not, add 0
385  if boundaries[0]:
386  boundaries.insert (0, 0)
387  # is the last boundary at the end? If not, make the end a
388  # boundary
389  if boundaries[-1] != len (xVals) - 1:
390  boundaries.append( len (xVals) - 1 )
391  rangeList = list(zip (boundaries, boundaries[1:]))
392  for thisRange in rangeList:
393  upper = thisRange[1]
394  lower = thisRange[0]
395  slope = old_div((yVals[upper] - yVals[lower]), \
396  (xVals[upper] - xVals[lower]))
397  print("slope", slope)
398  # now go over the range inclusively
399  pairList = []
400  for index in range (lower, upper + 1):
401  newExpected = yVals[lower] + \
402  slope * (xVals[index] - xVals[lower])
403  pairList.append( (xVals[index], newExpected) )
404  expectedVals[index] = newExpected
405  expectedChunks.append (pairList)
406  ###########################################
407  ## Instantanous Luminosity Ordering Mode ##
408  ###########################################
409  elif 'instLum' == options.edfMode or 'instIntLum' == options.edfMode:
410  eventTupList = []
411  if not lumiCont.xingInfo:
412  raise RuntimeError("Luminosity Xing information missing.")
413  for key, eventList in sorted( eventsDict.items() ):
414  try:
415  lumi = lumiCont[key]
416  instLum = lumi.aveInstLum
417  fracAXIL = lumi.fracAXILrecorded
418  totalAXIL = lumi.totalAXILrecorded
419  except:
420  raise RuntimeError("key %s not found in lumi information" \
421  % key.__str__())
422  for event in eventList:
423  eventTupList.append( (instLum, fracAXIL, totalAXIL, key,
424  event[0], event[1], ) )
425  eventTupList.sort()
426  for eventTup in eventTupList:
427  weight += eventTup[5]
428  factor = old_div(weight, totalWeight)
429  if 'instLum' == options.edfMode:
430  xVals.append (eventTup[0])
431  else:
432  xVals.append (eventTup[2])
433  yVals.append (factor)
434  expectedVals.append (eventTup[1])
435  predVals.append (eventTup[1] * options.pred)
436  else:
437  raise RuntimeError("It looks like Charles screwed up if you are seeing this.")
438 
439  size = len (xVals)
440  step = int (old_div(math.sqrt(size), 2) + 1)
441  if options.printValues:
442  for index in range (size):
443  print("%8f %8f %8f" % (xVals[index], yVals[index], expectedVals[index]), end=' ')
444  if index > step:
445  denom = xVals[index] - xVals[index - step]
446  numer = yVals[index] - yVals[index - step]
447  if denom:
448  print(" %8f" % (old_div(numer, denom)), end=' ')
449  if 0 == index % step:
450  print(" **", end=' ') ## indicates statistically independent
451  ## slope measurement
452  print()
453  print()
454 
455  xArray = array.array ('d', xVals)
456  yArray = array.array ('d', yVals)
457  expected = array.array ('d', expectedVals)
458  graph = ROOT.TGraph( size, xArray, yArray)
459  graph.SetTitle (options.title)
460  graph.SetMarkerStyle (20)
461  expectedGraph = ROOT.TGraph( size, xArray, expected)
462  expectedGraph.SetLineColor (ROOT.kRed)
463  expectedGraph.SetLineWidth (3)
464  if options.noDataPoints:
465  expectedGraph.SetLineStyle (2)
466 
467  # run statistical tests
468  if options.weights:
469  print("average weight per event:", old_div(weight, ( size - 1)))
470  maxDistance = ROOT.TMath.KolmogorovTest (size, yArray,
471  size, expected,
472  "M")
473  prob = ROOT.TMath.KolmogorovProb( maxDistance * math.sqrt( size ) )
474 
475  # display everything
476  ROOT.gROOT.SetStyle('Plain')
477  ROOT.gROOT.SetBatch()
478  c1 = ROOT.TCanvas()
479  graph.GetXaxis().SetRangeUser (min (xVals), max (xVals))
480  minValue = min (min(yVals), min(expected))
481  if options.pred:
482  minValue = min (minValue, min (predVals))
483  graph.GetYaxis().SetRangeUser (minValue,
484  max (max(yVals), max(expected), max(predVals)))
485  graph.SetLineWidth (3)
486  if options.noDataPoints:
487  graph.Draw ("AL")
488  else:
489  graph.Draw ("ALP")
490  if 'instLum' == options.edfMode:
491  graph.GetXaxis().SetTitle ("Average Xing Inst. Luminosity (1/ub/s)")
492  graph.GetXaxis().SetRangeUser (0., lumiCont.max('aveInstLum'))
493  else:
494  if 'instIntLum' == options.edfMode:
495  graph.GetXaxis().SetTitle ("Integrated Luminosity - Inst. Lum. Ordered (1/%s)" \
496  % lumiCont.invunits)
497  else:
498  graph.GetXaxis().SetTitle ("Integrated Luminosity (1/%s)" \
499  % lumiCont.invunits)
500  graph.GetYaxis().SetTitle ("Fraction of Events Seen")
501  expectedGraphs = []
502  if expectedChunks:
503  for index, chunk in enumerate (expectedChunks):
504  expectedXarray = array.array ('d', [item[0] for item in chunk])
505  expectedYarray = array.array ('d', [item[1] for item in chunk])
506  expectedGraph = ROOT.TGraph( len(chunk),
507  expectedXarray,
508  expectedYarray )
509  expectedGraph.SetLineWidth (3)
510  if options.noDataPoints:
511  expectedGraph.SetLineStyle (2)
512  if index % 2:
513  expectedGraph.SetLineColor (ROOT.kBlue)
514  else:
515  expectedGraph.SetLineColor (ROOT.kRed)
516  expectedGraph.Draw("L")
517  expectedGraphs.append (expectedGraph)
518  exptectedGraph = expectedGraphs[0]
519  else:
520  expectedGraph.Draw ("L")
521  green = 0
522  if options.pred:
523  predArray = array.array ('d', predVals)
524  green = ROOT.TGraph (size, xArray, predArray)
525  green.SetLineWidth (3)
526  green.SetLineColor (8)
527  green.Draw ('l')
528  legend = ROOT.TLegend(0.15, 0.65, 0.50, 0.85)
529  legend.SetFillStyle (0)
530  legend.SetLineColor(ROOT.kWhite)
531  observed = 'Observed'
532  if options.weights:
533  observed += ' (weighted)'
534  legend.AddEntry(graph, observed,"PL")
535  if options.resetExpected:
536  legend.AddEntry(expectedGraph, "Expected from partial yield","L")
537  else:
538  legend.AddEntry(expectedGraph, "Expected from total yield","L")
539  if options.pred:
540  legend.AddEntry(green, options.predLabel,"L")
541  legend.AddEntry("","D_{stat}=%.3f, N=%d" % (maxDistance, size),"")
542  legend.AddEntry("","P_{KS}=%.3f" % prob,"")
543  legend.Draw()
544 
545  # save file
546  c1.Print (outputFile)
547 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

Variable Documentation

string generateEDF.action = 'store_true'

Definition at line 576 of file generateEDF.py.

list generateEDF.allowedEDF = ['time', 'instLum', 'instIntLum']

## Main

command line options ##

Definition at line 560 of file generateEDF.py.

tuple generateEDF.cont = LumiInfoCont(args[0], **options.__dict__)

load Luminosity info ##

Definition at line 628 of file generateEDF.py.

Referenced by UHTRpacker.addChannel(), CaloTowersClient.CaloTowersEndjob(), CaloTowersDQMClient.CaloTowersEndjob(), DTHVStatusHandler.checkForPeriod(), Primary4DVertexHarvester.dqmEndJob(), HcalObjRepresent::HcalDataContainer< Items, Item >.fillValConts(), SymmetryFit.findUsableMinMax(), MuScleFitUtils.fitMass(), MuScleFitUtils.fitReso(), Phase2ITRecHitHarvester.gausFitslices(), DirectMuonNavigation.inOutBackward(), DirectMuonNavigation.inOutBarrel(), DirectMuonNavigation.inOutForward(), DQMGenericClient.limitedFit(), DirectMuonNavigation.outInBackward(), DirectMuonNavigation.outInBarrel(), DirectMuonNavigation.outInForward(), PATTauIDEmbedder.PATTauIDEmbedder(), cond::XMLAuthenticationService::XMLAuthenticationService.processFile(), TrackProducerWithSCAssociation.produce(), BoostedDoubleSVProducer.produce(), DAFTrackProducerAlgorithm.runWithCandidate(), and HcalSimHitsClient.SimHitsEndjob().

string generateEDF.default = 'Empirical Distribution Function'

Definition at line 567 of file generateEDF.py.

generateEDF.done = False

Definition at line 657 of file generateEDF.py.

string generateEDF.help = 'title of plot (default %default)'

Definition at line 568 of file generateEDF.py.

tuple generateEDF.inputGroup = optparse.OptionGroup(parser, "Input Options")

Definition at line 564 of file generateEDF.py.

tuple generateEDF.modeGroup = optparse.OptionGroup(parser, "Mode Options")

Definition at line 565 of file generateEDF.py.

tuple generateEDF.nonSpaceRE = re.compile(r'\S')

Definition at line 18 of file generateEDF.py.

tuple generateEDF.parser = optparse.OptionParser("Usage: %prog [options] lumi.csv events.txt output.png", description='Script for generating EDF curves. See https://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideGenerateEDF for more details.')

Definition at line 561 of file generateEDF.py.

tuple generateEDF.pieces = sepRE.split(line)

Definition at line 641 of file generateEDF.py.

tuple generateEDF.plotGroup = optparse.OptionGroup(parser, "Plot Options")

Definition at line 562 of file generateEDF.py.

generateEDF.prevRecLumi = 0

Definition at line 656 of file generateEDF.py.

tuple generateEDF.rangeGroup = optparse.OptionGroup(parser, "Range Options")

Definition at line 563 of file generateEDF.py.

int generateEDF.recLumIndex = 0

Definition at line 654 of file generateEDF.py.

list generateEDF.recLumis = []

look for which runs correspond to what total ## recorded integrated luminosity ##

Definition at line 639 of file generateEDF.py.

list generateEDF.recLumValue = float(piece)

Definition at line 644 of file generateEDF.py.

tuple generateEDF.sepRE = re.compile(r'[\s,;:]+')

Definition at line 17 of file generateEDF.py.

string generateEDF.type = 'string'

Definition at line 590 of file generateEDF.py.