CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
generateEDF Namespace Reference

Classes

class  LumiInfo
 

LumiInfo Class

More...
 
class  LumiInfoCont
 

LumiInfoCont Class

More...
 

Functions

def loadEvents
 

General Functions

More...
 
def makeEDFplot
 

Variables

string action = 'store_true'
 
list allowedEDF = ['time', 'instLum', 'instIntLum']
 

## Main

More...
 
tuple cont = LumiInfoCont(args[0], **options.__dict__)
 load Luminosity info ## More...
 
string default = 'Empirical Distribution Function'
 
 done = False
 
string help = 'title of plot (default %default)'
 
tuple inputGroup = optparse.OptionGroup(parser, "Input Options")
 
tuple modeGroup = optparse.OptionGroup(parser, "Mode Options")
 
tuple nonSpaceRE = re.compile(r'\S')
 
tuple parser = optparse.OptionParser("Usage: %prog [options] lumi.csv events.txt output.png", description='Script for generating EDF curves. See https://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideGenerateEDF for more details.')
 
tuple pieces = sepRE.split(line)
 
tuple plotGroup = optparse.OptionGroup(parser, "Plot Options")
 
int prevRecLumi = 0
 
tuple rangeGroup = optparse.OptionGroup(parser, "Range Options")
 
int recLumIndex = 0
 
list recLumis = []
 look for which runs correspond to what total ## recorded integrated luminosity ## More...
 
tuple recLumValue = float(piece)
 
tuple sepRE = re.compile(r'[\s,;:]+')
 
string type = 'string'
 

Function Documentation

def generateEDF.loadEvents (   filename,
  cont,
  options 
)

General Functions

Definition at line 234 of file generateEDF.py.

References bitset_utilities.append().

235 def loadEvents (filename, cont, options):
236  eventsDict = {}
237  print "loading events from '%s'" % filename
238  events = open (filename, 'r')
239  runIndex, lumiIndex, eventIndex, weightIndex = 0, 1, 2, 3
240  if options.relOrder:
241  lumiIndex, eventIndex = 2,1
242  minPieces = 3
243  totalWeight = 0.
244  if options.weights:
245  minPieces = 4
246  for line in events:
247  pieces = sepRE.split (line.strip())
248  if len (pieces) < minPieces:
249  if nonSpaceRE.search (line):
250  print "skipping", line
251  continue
252  try:
253  run, lumi, event = int( pieces[runIndex] ), \
254  int( pieces[lumiIndex] ), \
255  int( pieces[eventIndex] )
256  except:
257  continue
258  key = (run, lumi)
259  if key not in cont:
260  if options.ignore:
261  print "Warning, %s is not found in the lumi information" \
262  % key.__str__()
263  continue
264  else:
265  raise RuntimeError("%s is not found in lumi information. Use '--ignoreNoLumiEvents' option to ignore these events and continue." \
266  % key.__str__())
267  if options.edfMode != 'time' and not cont[key].xingInfo:
268  if options.ignore:
269  print "Warning, %s does not have Xing information" \
270  % key.__str__()
271  continue
272  else:
273  raise RuntimeError("%s does not have Xing information. Use '--ignoreNoLumiEvents' option to ignore these events and continue." \
274  % key.__str__())
275  if options.weights:
276  weight = float (pieces[weightIndex])
277  else:
278  weight = 1
279  eventsDict.setdefault( key, []).append( (event, weight) )
280  totalWeight += weight
281  events.close()
282  return eventsDict, totalWeight
283 
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
def loadEvents
General Functions
Definition: generateEDF.py:234
def generateEDF.makeEDFplot (   lumiCont,
  eventsDict,
  totalWeight,
  outputFile,
  options 
)

Definition at line 284 of file generateEDF.py.

References list(), bookConverter.max, and min().

285 def makeEDFplot (lumiCont, eventsDict, totalWeight, outputFile, options):
286  # make TGraph
287  xVals = [0]
288  yVals = [0]
289  expectedVals = [0]
290  predVals = [0]
291  weight = 0
292  expectedChunks = []
293  ########################
294  ## Time Ordering Mode ##
295  ########################
296  if 'time' == options.edfMode:
297  # if we have a minimum run number, clear the lists
298  if lumiCont.minRun or lumiCont.minIntLum:
299  xVals = []
300  yVals = []
301  expectedVals = []
302  predVals = []
303  # loop over events
304  for key, eventList in sorted( eventsDict.iteritems() ):
305  usePoints = True
306  # should we add this point?
307  if lumiCont.minRun and lumiCont.minRun > key[0] or \
308  lumiCont.maxRun and lumiCont.maxRun < key[0]:
309  usePoints = False
310  for event in eventList:
311  weight += event[1]
312  if not usePoints:
313  continue
314  factor = weight / totalWeight
315  try:
316  intLum = lumiCont[key].totalRecorded
317  except:
318  raise RuntimeError("key %s not found in lumi information" \
319  % key.__str__())
320  if lumiCont.minIntLum and lumiCont.minIntLum > intLum or \
321  lumiCont.maxIntLum and lumiCont.maxIntLum < intLum:
322  continue
323  lumFrac = intLum / lumiCont.totalRecLum
324  xVals.append( lumiCont[key].totalRecorded)
325  yVals.append (factor)
326  expectedVals.append (lumFrac)
327  predVals.append (lumFrac * options.pred)
328  # put on the last point if we aren't giving a maximum run
329  if not lumiCont.maxRun and not lumiCont.maxIntLum:
330  xVals.append (lumiCont.totalRecLum)
331  yVals.append (1)
332  expectedVals.append (1)
333  predVals.append (options.pred)
334  ####################
335  ## Reset Expected ##
336  ####################
337  if options.resetExpected:
338  slope = (yVals[-1] - yVals[0]) / (xVals[-1] - xVals[0])
339  print "slope", slope
340  for index, old in enumerate (expectedVals):
341  expectedVals[index] = yVals[0] + \
342  slope * (xVals[index] - xVals[0])
343  #############################################
344  ## Break Expected by Integrated Luminosity ##
345  #############################################
346  if options.breakExpectedIntLum:
347  breakExpectedIntLum = []
348  for chunk in options.breakExpectedIntLum:
349  pieces = sepRE.split (chunk)
350  try:
351  for piece in pieces:
352  breakExpectedIntLum.append( float(piece) )
353  except:
354  raise RuntimeError("'%s' from '%s' is not a valid float" \
355  % (piece, chunk))
356  breakExpectedIntLum.sort()
357  boundaries = []
358  breakIndex = 0
359  done = False
360  for index, xPos in enumerate (xVals):
361  if xPos > breakExpectedIntLum[breakIndex]:
362  boundaries.append (index)
363  while breakIndex < len (breakExpectedIntLum):
364  breakIndex += 1
365  if breakIndex >= len (breakExpectedIntLum):
366  done = True
367  break
368  # If this next position is different, than
369  # we're golden. Otherwise, let it go through
370  # the loop again.
371  if xPos <= breakExpectedIntLum[breakIndex]:
372  break
373  if done:
374  break
375  # do we have any boundaries?
376  if not boundaries:
377  raise RuntimeError("No values of 'breakExpectedIntLum' are in current range.")
378  # is the first boundary at 0? If not, add 0
379  if boundaries[0]:
380  boundaries.insert (0, 0)
381  # is the last boundary at the end? If not, make the end a
382  # boundary
383  if boundaries[-1] != len (xVals) - 1:
384  boundaries.append( len (xVals) - 1 )
385  rangeList = list(zip (boundaries, boundaries[1:]))
386  for thisRange in rangeList:
387  upper = thisRange[1]
388  lower = thisRange[0]
389  slope = (yVals[upper] - yVals[lower]) / \
390  (xVals[upper] - xVals[lower])
391  print "slope", slope
392  # now go over the range inclusively
393  pairList = []
394  for index in range (lower, upper + 1):
395  newExpected = yVals[lower] + \
396  slope * (xVals[index] - xVals[lower])
397  pairList.append( (xVals[index], newExpected) )
398  expectedVals[index] = newExpected
399  expectedChunks.append (pairList)
400  ###########################################
401  ## Instantanous Luminosity Ordering Mode ##
402  ###########################################
403  elif 'instLum' == options.edfMode or 'instIntLum' == options.edfMode:
404  eventTupList = []
405  if not lumiCont.xingInfo:
406  raise RuntimeError("Luminosity Xing information missing.")
407  for key, eventList in sorted( eventsDict.iteritems() ):
408  try:
409  lumi = lumiCont[key]
410  instLum = lumi.aveInstLum
411  fracAXIL = lumi.fracAXILrecorded
412  totalAXIL = lumi.totalAXILrecorded
413  except:
414  raise RuntimeError("key %s not found in lumi information" \
415  % key.__str__())
416  for event in eventList:
417  eventTupList.append( (instLum, fracAXIL, totalAXIL, key,
418  event[0], event[1], ) )
419  eventTupList.sort()
420  for eventTup in eventTupList:
421  weight += eventTup[5]
422  factor = weight / totalWeight
423  if 'instLum' == options.edfMode:
424  xVals.append (eventTup[0])
425  else:
426  xVals.append (eventTup[2])
427  yVals.append (factor)
428  expectedVals.append (eventTup[1])
429  predVals.append (eventTup[1] * options.pred)
430  else:
431  raise RuntimeError("It looks like Charles screwed up if you are seeing this.")
432 
433  size = len (xVals)
434  step = int (math.sqrt(size) / 2 + 1)
435  if options.printValues:
436  for index in range (size):
437  print "%8f %8f %8f" % (xVals[index], yVals[index], expectedVals[index]),
438  if index > step:
439  denom = xVals[index] - xVals[index - step]
440  numer = yVals[index] - yVals[index - step]
441  if denom:
442  print " %8f" % (numer / denom),
443  if 0 == index % step:
444  print " **", ## indicates statistically independent
445  ## slope measurement
446  print
447  print
448 
449  xArray = array.array ('d', xVals)
450  yArray = array.array ('d', yVals)
451  expected = array.array ('d', expectedVals)
452  graph = ROOT.TGraph( size, xArray, yArray)
453  graph.SetTitle (options.title)
454  graph.SetMarkerStyle (20)
455  expectedGraph = ROOT.TGraph( size, xArray, expected)
456  expectedGraph.SetLineColor (ROOT.kRed)
457  expectedGraph.SetLineWidth (3)
458  if options.noDataPoints:
459  expectedGraph.SetLineStyle (2)
460 
461  # run statistical tests
462  if options.weights:
463  print "average weight per event:", weight / ( size - 1)
464  maxDistance = ROOT.TMath.KolmogorovTest (size, yArray,
465  size, expected,
466  "M")
467  prob = ROOT.TMath.KolmogorovProb( maxDistance * math.sqrt( size ) )
468 
469  # display everything
470  ROOT.gROOT.SetStyle('Plain')
471  ROOT.gROOT.SetBatch()
472  c1 = ROOT.TCanvas()
473  graph.GetXaxis().SetRangeUser (min (xVals), max (xVals))
474  minValue = min (min(yVals), min(expected))
475  if options.pred:
476  minValue = min (minValue, min (predVals))
477  graph.GetYaxis().SetRangeUser (minValue,
478  max (max(yVals), max(expected), max(predVals)))
479  graph.SetLineWidth (3)
480  if options.noDataPoints:
481  graph.Draw ("AL")
482  else:
483  graph.Draw ("ALP")
484  if 'instLum' == options.edfMode:
485  graph.GetXaxis().SetTitle ("Average Xing Inst. Luminosity (1/ub/s)")
486  graph.GetXaxis().SetRangeUser (0., lumiCont.max('aveInstLum'))
487  else:
488  if 'instIntLum' == options.edfMode:
489  graph.GetXaxis().SetTitle ("Integrated Luminosity - Inst. Lum. Ordered (1/%s)" \
490  % lumiCont.invunits)
491  else:
492  graph.GetXaxis().SetTitle ("Integrated Luminosity (1/%s)" \
493  % lumiCont.invunits)
494  graph.GetYaxis().SetTitle ("Fraction of Events Seen")
495  expectedGraphs = []
496  if expectedChunks:
497  for index, chunk in enumerate (expectedChunks):
498  expectedXarray = array.array ('d', [item[0] for item in chunk])
499  expectedYarray = array.array ('d', [item[1] for item in chunk])
500  expectedGraph = ROOT.TGraph( len(chunk),
501  expectedXarray,
502  expectedYarray )
503  expectedGraph.SetLineWidth (3)
504  if options.noDataPoints:
505  expectedGraph.SetLineStyle (2)
506  if index % 2:
507  expectedGraph.SetLineColor (ROOT.kBlue)
508  else:
509  expectedGraph.SetLineColor (ROOT.kRed)
510  expectedGraph.Draw("L")
511  expectedGraphs.append (expectedGraph)
512  exptectedGraph = expectedGraphs[0]
513  else:
514  expectedGraph.Draw ("L")
515  green = 0
516  if options.pred:
517  predArray = array.array ('d', predVals)
518  green = ROOT.TGraph (size, xArray, predArray)
519  green.SetLineWidth (3)
520  green.SetLineColor (8)
521  green.Draw ('l')
522  legend = ROOT.TLegend(0.15, 0.65, 0.50, 0.85)
523  legend.SetFillStyle (0)
524  legend.SetLineColor(ROOT.kWhite)
525  observed = 'Observed'
526  if options.weights:
527  observed += ' (weighted)'
528  legend.AddEntry(graph, observed,"PL")
529  if options.resetExpected:
530  legend.AddEntry(expectedGraph, "Expected from partial yield","L")
531  else:
532  legend.AddEntry(expectedGraph, "Expected from total yield","L")
533  if options.pred:
534  legend.AddEntry(green, options.predLabel,"L")
535  legend.AddEntry("","D_{stat}=%.3f, N=%d" % (maxDistance, size),"")
536  legend.AddEntry("","P_{KS}=%.3f" % prob,"")
537  legend.Draw()
538 
539  # save file
540  c1.Print (outputFile)
541 
T min(T a, T b)
Definition: MathUtil.h:58
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run

Variable Documentation

string generateEDF.action = 'store_true'

Definition at line 570 of file generateEDF.py.

list generateEDF.allowedEDF = ['time', 'instLum', 'instIntLum']

## Main

command line options ##

Definition at line 554 of file generateEDF.py.

tuple generateEDF.cont = LumiInfoCont(args[0], **options.__dict__)

load Luminosity info ##

Definition at line 622 of file generateEDF.py.

Referenced by UHTRpacker.addChannel(), CaloTowersClient.CaloTowersEndjob(), CaloTowersDQMClient.CaloTowersEndjob(), DTHVStatusHandler.checkForPeriod(), SymmetryFit.findUsableMinMax(), MuScleFitUtils.fitMass(), MuScleFitUtils.fitReso(), DirectMuonNavigation.inOutBackward(), DirectMuonNavigation.inOutBarrel(), DirectMuonNavigation.inOutForward(), DQMGenericClient.limitedFit(), DirectMuonNavigation.outInBackward(), DirectMuonNavigation.outInBarrel(), DirectMuonNavigation.outInForward(), cond::XMLAuthenticationService::XMLAuthenticationService.processFile(), TrackProducerWithSCAssociation.produce(), BoostedDoubleSVProducer.produce(), DAFTrackProducerAlgorithm.runWithCandidate(), DetGeomDesc.setComponents(), and HcalSimHitsClient.SimHitsEndjob().

string generateEDF.default = 'Empirical Distribution Function'

Definition at line 561 of file generateEDF.py.

generateEDF.done = False

Definition at line 651 of file generateEDF.py.

string generateEDF.help = 'title of plot (default %default)'

Definition at line 562 of file generateEDF.py.

tuple generateEDF.inputGroup = optparse.OptionGroup(parser, "Input Options")

Definition at line 558 of file generateEDF.py.

tuple generateEDF.modeGroup = optparse.OptionGroup(parser, "Mode Options")

Definition at line 559 of file generateEDF.py.

tuple generateEDF.nonSpaceRE = re.compile(r'\S')

Definition at line 12 of file generateEDF.py.

tuple generateEDF.parser = optparse.OptionParser("Usage: %prog [options] lumi.csv events.txt output.png", description='Script for generating EDF curves. See https://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideGenerateEDF for more details.')

Definition at line 555 of file generateEDF.py.

tuple generateEDF.pieces = sepRE.split(line)

Definition at line 635 of file generateEDF.py.

tuple generateEDF.plotGroup = optparse.OptionGroup(parser, "Plot Options")

Definition at line 556 of file generateEDF.py.

generateEDF.prevRecLumi = 0

Definition at line 650 of file generateEDF.py.

tuple generateEDF.rangeGroup = optparse.OptionGroup(parser, "Range Options")

Definition at line 557 of file generateEDF.py.

int generateEDF.recLumIndex = 0

Definition at line 648 of file generateEDF.py.

list generateEDF.recLumis = []

look for which runs correspond to what total ## recorded integrated luminosity ##

Definition at line 633 of file generateEDF.py.

list generateEDF.recLumValue = float(piece)

Definition at line 638 of file generateEDF.py.

tuple generateEDF.sepRE = re.compile(r'[\s,;:]+')

Definition at line 11 of file generateEDF.py.

string generateEDF.type = 'string'

Definition at line 584 of file generateEDF.py.