CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
DatacardParser Namespace Reference

Classes

class  Datacard
 

Functions

def addDatacardParserOptions
 
def isVetoed
 
def parseCard
 

Variables

tuple globalNuisances = re.compile('(lumi|pdf_(qqbar|gg|qg)|QCDscale_(ggH|qqH|VH|ggH1in|ggH2in|VV)|UEPS|FakeRate|CMS_(eff|fake|trigger|scale|res)_([gemtjb]|met))')
 

Function Documentation

def DatacardParser.addDatacardParserOptions (   parser)

Definition at line 6 of file DatacardParser.py.

6 
7 def addDatacardParserOptions(parser):
8  parser.add_option("-s", "--stat", dest="stat", default=False, action="store_true", help="keep only statistical uncertainties, no systematics")
9  parser.add_option("-f", "--fix-pars", dest="fixpars",default=False, action="store_true", help="fix all floating parameters of the pdfs except for the POI")
10  parser.add_option("-c", "--compiled", dest="cexpr", default=False, action="store_true", help="use compiled expressions (not suggested)")
11  parser.add_option("-a", "--ascii", dest="bin", default=True, action="store_false", help="produce a Workspace in a rootfile in an HLF file (legacy, unsupported)")
12  parser.add_option("-b", "--binary", dest="bin", default=True, action="store_true", help="produce a Workspace in a rootfile (default)")
13  parser.add_option("-o", "--out", dest="out", default=None, type="string", help="output file (if none, it will print to stdout). Required for binary mode.")
14  parser.add_option("-v", "--verbose", dest="verbose", default=0, type="int", help="Verbosity level (0 = quiet, 1 = verbose, 2+ = more)")
15  parser.add_option("-m", "--mass", dest="mass", default=0, type="float", help="Higgs mass to use. Will also be written in the Workspace as RooRealVar 'MH'.")
16  parser.add_option("-D", "--dataset", dest="dataname", default="data_obs", type="string", help="Name of the observed dataset")
17  parser.add_option("-L", "--LoadLibrary", dest="libs", type="string" , action="append", help="Load these libraries")
18  parser.add_option("--poisson", dest="poisson", default=0, type="int", help="If set to a positive number, binned datasets wih more than this number of entries will be generated using poissonians")
19  parser.add_option("--default-morphing", dest="defMorph", type="string", default="shape2N", help="Default template morphing algorithm (to be used when the datacard has just 'shape')")
20  parser.add_option("--X-exclude-nuisance", dest="nuisancesToExclude", type="string", action="append", default=[], help="Exclude nuisances that match these regular expressions.")
21  parser.add_option("--X-force-simpdf", dest="forceSimPdf", default=False, action="store_true", help="FOR DEBUG ONLY: Always produce a RooSimultaneous, even for single channels.")
22  parser.add_option("--X-no-check-norm", dest="noCheckNorm", default=False, action="store_true", help="FOR DEBUG ONLY: Turn off the consistency check between datacard norms and shape norms. Will give you nonsensical results if you have shape uncertainties.")
23  parser.add_option("--X-no-jmax", dest="noJMax", default=False, action="store_true", help="FOR DEBUG ONLY: Turn off the consistency check between jmax and number of processes.")
24 
DatacardParser.addDatacardParserOptions
def addDatacardParserOptions
Definition: DatacardParser.py:6
def DatacardParser.isVetoed (   name,
  vetoList 
)

Definition at line 38 of file DatacardParser.py.

Referenced by parseCard().

38 
39 def isVetoed(name,vetoList):
40  for pattern in vetoList:
41  if not pattern: continue
42  if re.match(pattern,name): return True
43  return False
def DatacardParser.parseCard (   file,
  options 
)

Definition at line 44 of file DatacardParser.py.

References python.multivaluedict.dict, and isVetoed().

44 
45 def parseCard(file, options):
46  if type(file) == type("str"):
47  raise RuntimeError, "You should pass as argument to parseCards a file object, stream or a list of lines, not a string"
48  ret = Datacard()
49  #
50  nbins = -1;
51  nprocesses = -1;
52  nuisances = -1;
53  binline = []; processline = []; sigline = []
54  for l in file:
55  f = l.split();
56  if len(f) < 1: continue
57  if f[0] == "imax":
58  nbins = int(f[1]) if f[1] != "*" else -1
59  if f[0] == "jmax":
60  nprocesses = int(f[1])+1 if f[1] != "*" else -1
61  if f[0] == "kmax":
62  nuisances = int(f[1]) if f[1] != "*" else -1
63  if f[0] == "shapes":
64  if not options.bin: raise RuntimeError, "Can use shapes only with binary output mode"
65  if len(f) < 4: raise RuntimeError, "Malformed shapes line"
66  if not ret.shapeMap.has_key(f[2]): ret.shapeMap[f[2]] = {}
67  if ret.shapeMap[f[2]].has_key(f[1]): raise RuntimeError, "Duplicate definition for process '%s', channel '%s'" % (f[1], f[2])
68  ret.shapeMap[f[2]][f[1]] = f[3:]
69  if f[0] == "Observation" or f[0] == "observation":
70  ret.obs = [ float(x) for x in f[1:] ]
71  if nbins == -1: nbins = len(ret.obs)
72  if len(ret.obs) != nbins: raise RuntimeError, "Found %d observations but %d bins have been declared" % (len(ret.obs), nbins)
73  if binline != []:
74  if len(binline) != len(ret.obs): raise RuntimeError, "Found %d bins (%s) but %d bins have been declared" % (len(ret.bins), ret.bins, nbins)
75  ret.bins = binline
76  ret.obs = dict([(b,ret.obs[i]) for i,b in enumerate(ret.bins)])
77  binline = []
78  if f[0] == "bin":
79  binline = []
80  for b in f[1:]:
81  if re.match("[0-9]+", b): b = "bin"+b
82  binline.append(b)
83  if f[0] == "process":
84  if processline == []: # first line contains names
85  processline = f[1:]
86  if len(binline) != len(processline): raise RuntimeError, "'bin' line has a different length than 'process' line."
87  continue
88  sigline = f[1:] # second line contains ids
89  if re.match("-?[0-9]+", processline[0]) and not re.match("-?[0-9]+", sigline[0]):
90  (processline,sigline) = (sigline,processline)
91  if len(sigline) != len(processline): raise RuntimeError, "'bin' line has a different length than 'process' line."
92  hadBins = (len(ret.bins) > 0)
93  for i,b in enumerate(binline):
94  p = processline[i];
95  s = (int(sigline[i]) <= 0) # <=0 for signals, >0 for backgrounds
96  ret.keyline.append((b, processline[i], s))
97  if hadBins:
98  if b not in ret.bins: raise RuntimeError, "Bin %s not among the declared bins %s" % (b, ret.bins)
99  else:
100  if b not in ret.bins: ret.bins.append(b)
101  if p not in ret.processes: ret.processes.append(p)
102  if nprocesses == -1: nprocesses = len(ret.processes)
103  if nbins == -1: nbins = len(ret.bins)
104  if not options.noJMax:
105  if nprocesses != len(ret.processes): raise RuntimeError, "Found %d processes (%s), declared jmax = %d" % (len(ret.processes),ret.processes,nprocesses)
106  if nbins != len(ret.bins): raise RuntimeError, "Found %d bins (%s), declared imax = %d" % (len(ret.bins),ret.bins,nbins)
107  ret.exp = dict([(b,{}) for b in ret.bins])
108  ret.isSignal = dict([(p,None) for p in ret.processes])
109  if ret.obs != [] and type(ret.obs) == list: # still as list, must change into map with bin names
110  ret.obs = dict([(b,ret.obs[i]) for i,b in enumerate(ret.bins)])
111  for (b,p,s) in ret.keyline:
112  if ret.isSignal[p] == None:
113  ret.isSignal[p] = s
114  elif ret.isSignal[p] != s:
115  raise RuntimeError, "Process %s is declared as signal in some bin and as background in some other bin" % p
116  ret.signals = [p for p,s in ret.isSignal.items() if s == True]
117  if len(ret.signals) == 0: raise RuntimeError, "You must have at least one signal process (id <= 0)"
118  if f[0] == "rate":
119  if processline == []: raise RuntimeError, "Missing line with process names before rate line"
120  if sigline == []: raise RuntimeError, "Missing line with process id before rate line"
121  if len(f[1:]) != len(ret.keyline): raise RuntimeError, "Malformed rate line: length %d, while bins and process lines have length %d" % (len(f[1:]), len(ret.keyline))
122  for (b,p,s),r in zip(ret.keyline,f[1:]):
123  ret.exp[b][p] = float(r)
124  break # rate is the last line before nuisances
125  # parse nuisances
126  for l in file:
127  if l.startswith("--"): continue
128  l = re.sub("\\s*#.*","",l)
129  l = re.sub("(?<=\\s)-+(\\s|$)"," 0\\1",l);
130  f = l.split();
131  if len(f) <= 1: continue
132  nofloat = False
133  lsyst = f[0]; pdf = f[1]; args = []; numbers = f[2:];
134  if lsyst.endswith("[nofloat]"):
135  lsyst = lsyst.replace("[nofloat]","")
136  nofloat = True
137  if options.nuisancesToExclude and isVetoed(lsyst, options.nuisancesToExclude):
138  if options.verbose > 0: stderr.write("Excluding nuisance %s selected by a veto pattern among %s\n" % (lsyst, options.nuisancesToExclude))
139  if nuisances != -1: nuisances -= 1
140  continue
141  if re.match("[0-9]+",lsyst): lsyst = "theta"+lsyst
142  if pdf == "lnN" or pdf == "lnU" or pdf == "gmM" or pdf == "trG" or pdf.startswith("shape"):
143  pass # nothing special to do
144  elif pdf == "gmN":
145  args = [int(f[2])]; numbers = f[3:];
146  elif pdf == "unif":
147  args = [float(f[2]), float(f[3])]; numbers = f[4:];
148  elif pdf == "param":
149  # for parametric uncertainties, there's no line to account per bin/process effects
150  # just assume everything else is an argument and move on
151  args = f[2:]
152  if len(args) <= 1: raise RuntimeError, "Uncertainties of type 'param' must have at least two arguments (mean and sigma)"
153  ret.systs.append([lsyst,nofloat,pdf,args,[]])
154  continue
155  elif pdf == "flatParam":
156  ret.flatParamNuisances[lsyst] = True
157  #for flat parametric uncertainties, code already does the right thing as long as they are non-constant RooRealVars linked to the model
158  continue
159  else:
160  raise RuntimeError, "Unsupported pdf %s" % pdf
161  if len(numbers) < len(ret.keyline): raise RuntimeError, "Malformed systematics line %s of length %d: while bins and process lines have length %d" % (lsyst, len(numbers), len(ret.keyline))
162  errline = dict([(b,{}) for b in ret.bins])
163  nonNullEntries = 0
164  for (b,p,s),r in zip(ret.keyline,numbers):
165  if "/" in r: # "number/number"
166  if (pdf not in ["lnN","lnU"]) and ("?" not in pdf): raise RuntimeError, "Asymmetric errors are allowed only for Log-normals"
167  errline[b][p] = [ float(x) for x in r.split("/") ]
168  else:
169  errline[b][p] = float(r)
170  # set the rate to epsilon for backgrounds with zero observed sideband events.
171  if pdf == "gmN" and ret.exp[b][p] == 0 and float(r) != 0: ret.exp[b][p] = 1e-6
172  ret.systs.append([lsyst,nofloat,pdf,args,errline])
173  # check if there are bins with no rate
174  for b in ret.bins:
175  np_bin = sum([(ret.exp[b][p] != 0) for (b1,p,s) in ret.keyline if b1 == b])
176  ns_bin = sum([(ret.exp[b][p] != 0) for (b1,p,s) in ret.keyline if b1 == b and s == True])
177  nb_bin = sum([(ret.exp[b][p] != 0) for (b1,p,s) in ret.keyline if b1 == b and s != True])
178  if np_bin == 0: raise RuntimeError, "Bin %s has no processes contributing to it" % b
179  if ns_bin == 0: raise RuntimeError, "Bin %s has no signal processes contributing to it" % b
180  if nb_bin == 0: raise RuntimeError, "Bin %s has no background processes contributing to it" % b
181  # cleanup systematics that have no effect to avoid zero derivatives
182  syst2 = []
183  for lsyst,nofloat,pdf,args,errline in ret.systs:
184  nonNullEntries = 0
185  if pdf == "param": # this doesn't have an errline
186  syst2.append((lsyst,nofloat,pdf,args,errline))
187  continue
188  for (b,p,s) in ret.keyline:
189  r = errline[b][p]
190  nullEffect = (r == 0.0 or (pdf == "lnN" and r == 1.0))
191  if not nullEffect and ret.exp[b][p] != 0: nonNullEntries += 1 # is this a zero background?
192  if nonNullEntries != 0: syst2.append((lsyst,nofloat,pdf,args,errline))
193  elif nuisances != -1: nuisances -= 1 # remove from count of nuisances, since qe skipped it
194  ret.systs = syst2
195  # remove them if options.stat asks so
196  if options.stat:
197  nuisances = 0
198  ret.systs = []
199  # check number of nuisances
200  if nuisances == -1:
201  nuisances = len(ret.systs)
202  elif len(ret.systs) != nuisances:
203  raise RuntimeError, "Found %d systematics, expected %d" % (len(ret.systs), nuisances)
204  # set boolean to know about shape
205  ret.hasShapes = (len(ret.shapeMap) > 0)
206  # return result
207  return ret

Variable Documentation

tuple DatacardParser.globalNuisances = re.compile('(lumi|pdf_(qqbar|gg|qg)|QCDscale_(ggH|qqH|VH|ggH1in|ggH2in|VV)|UEPS|FakeRate|CMS_(eff|fake|trigger|scale|res)_([gemtjb]|met))')

Definition at line 4 of file DatacardParser.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5