CMS 3D CMS Logo
Functions
DMR Namespace Reference

Functions

def DMR (config, validationDir)
 

Function Documentation

◆ DMR()

def DMR.DMR (   config,
  validationDir 
)

Definition at line 4 of file DMR.py.

References mps_setup.append, createfilelist.int, relativeConstraints.keys, print(), FastTimerService_cff.range, python.rootplot.root2matplotlib.replace(), submitPVValidationJobs.split(), and str.

Referenced by trends().

4 def DMR(config, validationDir):
5 
6  jobs = []
7  dmrType = "single"
8 
9 
10  IOVs = {}
11 
12 
13  isDataMerged = {}
14 
15 
16  if not dmrType in config["validations"]["DMR"]:
17  raise Exception("No 'single' key word in config for DMR")
18 
19  for singleName in config["validations"]["DMR"][dmrType]:
20  aux_IOV = config["validations"]["DMR"][dmrType][singleName]["IOV"]
21  if not isinstance(aux_IOV, list) and aux_IOV.endswith(".txt"):
22  config["validations"]["DMR"][dmrType][singleName]["IOV"] = []
23  with open(aux_IOV, 'r') as IOVfile: for line in IOVfile.readlines():
24  if len(line) != 0: config["validations"]["DMR"][dmrType][singleName]["IOV"].append(int(line))
25  for IOV in config["validations"]["DMR"][dmrType][singleName]["IOV"]:
26 
27  if singleName not in IOVs.keys():
28  IOVs[singleName] = []
29  if IOV not in IOVs[singleName]:
30  IOVs[singleName].append(IOV)
31 
32  for alignment in config["validations"]["DMR"][dmrType][singleName]["alignments"]:
33 
34  workDir = "{}/DMR/{}/{}/{}/{}".format(validationDir, dmrType, singleName, alignment, IOV)
35 
36 
37  local = {}
38  local["output"] = "{}/{}/DMR/{}/{}/{}/{}".format(config["LFS"], config["name"], dmrType, alignment, singleName, IOV)
39  local["alignment"] = copy.deepcopy(config["alignments"][alignment])
40  local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][singleName])
41  local["validation"].pop("alignments")
42  local["validation"]["IOV"] = IOV
43  if "dataset" in local["validation"]:
44  local["validation"]["dataset"] = local["validation"]["dataset"].format(IOV)
45  if "goodlumi" in local["validation"]:
46  local["validation"]["goodlumi"] = local["validation"]["goodlumi"].format(IOV)
47 
48 
49  job = {
50  "name": "DMR_{}_{}_{}_{}".format(dmrType, alignment, singleName, IOV),
51  "dir": workDir,
52  "exe": "cmsRun",
53  "cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/DMR_cfg.py".format(os.environ["CMSSW_BASE"]),
54  "run-mode": "Condor",
55  "dependencies": [],
56  "config": local,
57  }
58 
59  jobs.append(job)
60 
61 
62  if "merge" in config["validations"]["DMR"]:
63 
64  mergeJobs = []
65  dmrType = "merge"
66 
67 
68  for mergeName in config["validations"]["DMR"][dmrType]:
69 
70  singlesMC = []
71  for singleName in config["validations"]["DMR"][dmrType][mergeName]['singles']:
72  if len(IOVs[singleName]) == 1 and int(IOVs[singleName][0]) == 1: singlesMC.append(singleName)
73  isMConly = (len(singlesMC) == len(config["validations"]["DMR"][dmrType][mergeName]['singles']))
74  if isMConly:
75  isDataMerged[mergeName] = 0
76  elif len(singlesMC) == 0:
77  isDataMerged[mergeName] = 1
78  else:
79  isDataMerged[mergeName] = -1
80 
81 
82  for iname,singleName in enumerate(config["validations"]["DMR"][dmrType][mergeName]['singles']):
83  isMC = (singleName in singlesMC)
84  if isMConly and iname > 0: continue #special case for MC only comparison
85  elif isMConly: singlesMC.pop(singlesMC.index(singleName))
86 
87  for IOV in IOVs[singleName]:
88  if isMC and not isMConly: continue #ignore IOV=1 as it is automatically added to each DATA IOV unless MC only comparison
89 
90 
91  workDir = "{}/DMR/{}/{}/{}".format(validationDir, dmrType, mergeName, IOV) #Different (DATA) single jobs must contain different set of IOVs
92 
93 
94  local = {}
95 
96  job = {
97  "name": "DMR_{}_{}_{}".format(dmrType, mergeName, IOV),
98  "dir": workDir,
99  "exe": "DMRmerge",
100  "run-mode": "Condor",
101  "dependencies": [],
102  "config": local,
103  }
104 
105 
106  for alignment in config["alignments"]:
107  idxIncrement = 0
108  local.setdefault("alignments", {})
109  if alignment in config["validations"]["DMR"]["single"][singleName]["alignments"]: #Cover all DATA validations
110  local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
111  local["alignments"][alignment]['index'] = config["validations"]["DMR"]["single"][singleName]["alignments"].index(alignment)
112  for singleMCname in singlesMC:
113  if alignment in config["validations"]["DMR"]["single"][singleMCname]["alignments"]: #Add MC objects
114  local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
115  local["alignments"][alignment]['index'] = len(config["validations"]["DMR"]["single"][singleName]["alignments"])
116  local["alignments"][alignment]['index'] += idxIncrement + config["validations"]["DMR"]["single"][singleMCname]["alignments"].index(alignment)
117  idxIncrement += len(config["validations"]["DMR"]["single"][singleMCname]["alignments"])
118  local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][mergeName])
119  local["validation"]["IOV"] = IOV #is it really needed here?
120  if "customrighttitle" in local["validation"].keys():
121  if "IOV" in local["validation"]["customrighttitle"]:
122  local["validation"]["customrighttitle"] = local["validation"]["customrighttitle"].replace("IOV",str(IOV))
123  local["output"] = "{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], dmrType, mergeName, IOV)
124 
125 
126  if "style" in config.keys():
127  if "DMR" in config['style'].keys():
128  if dmrType in config['style']['DMR'].keys():
129  local["style"] = copy.deepcopy(config["style"]["DMR"][dmrType])
130  if "Rlabel" in local["style"] and "customrighttitle" in local["validation"].keys():
131  print("WARNING: custom right label is overwritten by global settings")
132 
133 
134  for singleJob in jobs:
135 
136  _alignment, _singleName, _singleIOV = singleJob["name"].split("_")[2:]
137  if _singleName in config["validations"]["DMR"][dmrType][mergeName]["singles"]:
138  if int(_singleIOV) == IOV or (int(_singleIOV) == 1 and _singleName in singlesMC): #matching DATA job or any MC single job
139  local["alignments"][_alignment]["file"] = singleJob["config"]["output"]
140  job["dependencies"].append(singleJob["name"])
141 
142 
143  mergeJobs.append(job)
144 
145 
146  jobs.extend(mergeJobs)
147 
148  if "trends" in config["validations"]["DMR"]:
149 
150 
151  trendJobs = []
152  dmrType = "trends"
153 
154  for trendName in config["validations"]["DMR"][dmrType]:
155  #print("trendName = {}".format(trendName))
156 
157  workDir = "{}/DMR/{}/{}".format(validationDir, dmrType, trendName)
158 
159 
160  local = {}
161  job = {
162  "name": "DMR_{}_{}".format(dmrType, trendName),
163  "dir": workDir,
164  "exe": "DMRtrends",
165  "run-mode": "Condor",
166  "dependencies": [],
167  "config": local,
168  }
169 
170 
171  mergesDATA = []
172  for mergeName in config["validations"]["DMR"][dmrType][trendName]["merges"]:
173 
174  if isDataMerged[mergeName] < 0:
175  raise Exception("Trend jobs cannot process merge jobs containing both DATA and MC objects.")
176  elif isDataMerged[mergeName] == 1:
177  mergesDATA.append(mergeName)
178  else:
179  if "doUnitTest" in config["validations"]["DMR"][dmrType][trendName].keys() and config["validations"]["DMR"][dmrType][trendName]["doUnitTest"]:
180  local.setdefault("alignments", {})
181  continue
182  else:
183  raise Exception("Trend jobs are not implemented for treating MC.")
184 
185 
186  trendIOVs = []
187  _mergeFiles = []
188  for mergeName in mergesDATA:
189  for iname,singleName in enumerate(config["validations"]["DMR"]['merge'][mergeName]['singles']):
190  trendIOVs += [IOV for IOV in IOVs[singleName]]
191 
192  for alignment in config["alignments"]:
193  local.setdefault("alignments", {})
194  if alignment in config["validations"]["DMR"]["single"][singleName]["alignments"]: #Cover all DATA validations
195  local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
196  local["alignments"][alignment]['index'] = config["validations"]["DMR"]["single"][singleName]["alignments"].index(alignment)
197  _mergeFiles.append("{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], "merge", mergeName, "{}"))
198  trendIOVs.sort()
199  local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][trendName])
200  if len(_mergeFiles) == 1:
201  local["validation"]["mergeFile"] = _mergeFiles[0]
202  else:
203  local["validation"]["mergeFile"] = _mergeFiles #FIXME for multiple merge files in backend
204  local["validation"]["IOV"] = trendIOVs
205  local["output"] = "{}/{}/DMR/{}/{}/".format(config["LFS"], config["name"], dmrType, trendName)
206  if "style" in config.keys() and "trends" in config["style"].keys():
207  local["style"] = copy.deepcopy(config["style"])
208  if "DMR" in local["style"].keys(): local["style"].pop("DMR")
209  if "CMSlabel" in config["style"]["trends"].keys(): local["style"]["CMSlabel"] = config["style"]["trends"]["CMSlabel"]
210  if "Rlabel" in config["style"]["trends"].keys():
211  local["style"]["trends"].pop("Rlabel")
212  local["style"]["trends"]["TitleCanvas"] = config["style"]["trends"]["Rlabel"]
213  else:
214  raise Exception("You want to create 'trends' jobs, but there are no 'lines' section in the config for pixel updates!")
215 
216  #Loop over all merge jobs
217  for mergeName in mergesDATA:
218  for mergeJob in mergeJobs:
219  alignment, mergeJobName, mergeIOV = mergeJob["name"].split("_")[1:]
220  if mergeJobName == mergeName and int(mergeIOV) in trendIOVs:
221  job["dependencies"].append(mergeJob["name"])
222 
223  trendJobs.append(job)
224 
225  jobs.extend(trendJobs)
226 
227  if "averaged" in config["validations"]["DMR"]:
228 
229 
230  avpJobs = []
231  dmrType = "averaged"
232  for avpName in config["validations"]["DMR"][dmrType]:
233 
234  workDir = "{}/DMR/{}/{}".format(validationDir, dmrType, avpName)
235  output = "{}/{}/DMR/{}/{}".format(config["LFS"], config["name"], dmrType, avpName)
236 
237 
238  mergesDATA = []
239  mergesMC = []
240  for mergeName in config["validations"]["DMR"][dmrType][avpName]["merges"]:
241 
242  if isDataMerged[mergeName] < 0:
243  raise Exception("Average jobs cannot process merge jobs containing both DATA and MC objects.")
244  elif isDataMerged[mergeName] == 1:
245  mergesDATA.append(mergeName)
246  else:
247  mergesMC.append(mergeName)
248 
249  lumiPerRun = []
250  lumiPerIoV = []
251  lumiMC = []
252  if len(mergesDATA) > 0:
253  if "lumiPerRun" in config["validations"]["DMR"][dmrType][avpName].keys():
254  for lumifile in config["validations"]["DMR"][dmrType][avpName]['lumiPerRun']:
255  if lumifile.split(".")[-1] in ["txt","csv"]:
256  lumiPerRun.append(lumifile)
257  if "lumiPerIoV" in config["validations"]["DMR"][dmrType][avpName].keys():
258  for lumifile in config["validations"]["DMR"][dmrType][avpName]['lumiPerIoV']:
259  if lumifile.split(".")[-1] in ["txt","csv"]:
260  lumiPerIoV.append(lumifile)
261  if len(lumiPerRun) == 0 and len(lumiPerIoV) == 0:
262  raise Exception("No lumi per run/IoV file found or not specified in .csv/.txt format.")
263  if len(mergesMC) > 0:
264  if 'lumiMC' in config["validations"]["DMR"][dmrType][avpName].keys():
265  lumiMC = config["validations"]["DMR"][dmrType][avpName]['lumiMC']
266 
267 
268  plotJob = {}
269  plotJob['workdir'] = "{}/{}".format(workDir,"plots")
270  plotJob['output'] = "{}/{}".format(output,"plots")
271  plotJob['inputData'] = []
272  plotJob['inputMC'] = []
273  plotJob['dependencies'] = []
274 
275 
276  for mergeName in mergesDATA:
277 
278  workDirMerge = "{}/{}".format(workDir, mergeName)
279  outputMerge = "{}/{}".format(output, mergeName)
280 
281 
282  local = {}
283  local["type"] = "DMR"
284  local["mode"] = "merge"
285  local["isData"] = True
286  local["isMC"] = False
287 
288 
289  for alignment in config["alignments"]:
290  local.setdefault("alignments", {})
291  for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
292  if alignment in config["validations"]["DMR"]["single"][singleName]['alignments']:
293  local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
294  local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][avpName])
295  local["validation"]["mergeFile"] = "{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], "merge", mergeName, "{}")
296  local["validation"]["lumiPerRun"] = lumiPerRun
297  local["validation"]["lumiPerIoV"] = lumiPerIoV
298  local["validation"]["lumiMC"] = lumiMC
299  local["validation"]["firstFromNext"] = []
300 
301 
302  IOVsPerMergeStep = []
303  for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
304  for IOV in IOVs[singleName]:
305  if IOV not in IOVsPerMergeStep:
306  IOVsPerMergeStep.append(IOV)
307  IOVsPerMergeStep.sort()
308 
309 
310  extra_part = 0
311  maxfiles = int(config["validations"]["DMR"][dmrType][avpName]['maxfiles'])
312  if len(IOVsPerMergeStep)%maxfiles >= 2:
313  extra_part = 1
314  parts = extra_part+len(IOVsPerMergeStep)//maxfiles
315 
316  subJob = {'name' : [], 'output' : [], 'lumiPerFile' : []}
317  for ipart in range(0,parts):
318  #Adapt workdir per each subjob
319  workDirSub = workDirMerge+"_"+str(ipart)
320  outputSub = outputMerge+"_"+str(ipart)
321 
322  #Define IOV group
323  IOVGroup = []
324  lastIndex = 0
325  for iIOV,IOV in enumerate(IOVsPerMergeStep):
326  if (iIOV//maxfiles == ipart) or (ipart == parts-1 and iIOV//maxfiles > ipart):
327  IOVGroup.append(IOV)
328  lastIndex = iIOV
329  firstFromNext = []
330  if lastIndex != len(IOVsPerMergeStep)-1:
331  firstFromNext.append(IOVsPerMergeStep[lastIndex+1])
332 
333  #Write job info
334  _local = copy.deepcopy(local)
335  _local["output"] = outputSub
336  _local["validation"]["IOV"] = IOVGroup
337  _local["validation"]["firstFromNext"] = firstFromNext
338  job = {
339  "name": "DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_"+str(ipart)),
340  "dir": workDirSub,
341  "exe": "mkLumiAveragedPlots.py",
342  "run-mode": "Condor",
343  "dependencies": [],
344  "config": _local,
345  }
346  subJob['output'].append(outputSub)
347  subJob['name'].append("DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_"+str(ipart)))
348  subJob['lumiPerFile'].append(os.path.join(outputSub,"lumiPerFile.csv"))
349  if parts == 1:
350  plotJob['inputData'].append(outputSub)
351  plotJob['dependencies'].append(job['name'])
352 
353  #Set average job dependencies from the list of all merge jobs
354  for mergeJob in mergeJobs:
355  alignment, mergeJobName, mergeIOV = mergeJob["name"].split("_")[1:]
356  if mergeJobName == mergeName and int(mergeIOV) in IOVGroup:
357  job["dependencies"].append(mergeJob["name"])
358  #if mergeJobName in config["validations"]["DMR"][dmrType][avpName]["merges"]:
359  # job["dependencies"].append(mergeJob["name"])
360 
361  #Add to queue
362  avpJobs.append(job)
363 
364 
365  if parts > 1:
366  localFinalize = copy.deepcopy(local)
367  localFinalize['mode'] = "finalize"
368  localFinalize['output'] = outputMerge
369  localFinalize["validation"]["IOV"] = []
370  localFinalize["validation"]["mergeFile"] = subJob['output']
371  localFinalize["validation"]["lumiPerRun"] = []
372  localFinalize["validation"]["lumiPerIoV"] = subJob['lumiPerFile']
373  job = {
374  "name": "DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_finalize"),
375  "dir": workDirMerge,
376  "exe": "mkLumiAveragedPlots.py",
377  "run-mode": "Condor",
378  "dependencies": subJob['name'],
379  "config": localFinalize,
380  }
381  avpJobs.append(job)
382  plotJob['inputData'].append(outputMerge)
383  plotJob['dependencies'].append(job['name'])
384 
385  #Second create one averager job per all MC merge jobs
386  if len(mergesMC) != 0:
387 
388  workDirMerge = "{}/{}".format(workDir, "MC")
389  outputMerge = "{}/{}".format(output, "MC")
390 
391 
392  local = {}
393  local["type"] = "DMR"
394  local["mode"] = "merge"
395  local["isData"] = False
396  local["isMC"] = True
397  local["output"] = outputMerge
398 
399 
400  local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][avpName])
401  local["validation"]["mergeFile"] = []
402  for mergeName in mergesMC:
403  for alignment in config["alignments"]:
404  local.setdefault("alignments", {})
405  for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
406  if alignment in config["validations"]["DMR"]["single"][singleName]['alignments']:
407  local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
408  local["validation"]["mergeFile"].append("{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], "merge", mergeName, "{}"))
409  local["validation"]["lumiPerRun"] = lumiPerRun
410  local["validation"]["lumiPerIoV"] = lumiPerIoV
411  local["validation"]["lumiMC"] = lumiMC
412  local["validation"]["IOV"] = [1]
413 
414 
415  job = {
416  "name": "DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_MC"),
417  "dir": workDirMerge,
418  "exe": "mkLumiAveragedPlots.py",
419  "run-mode": "Condor",
420  "dependencies": [],
421  "config": local,
422  }
423  plotJob['inputMC'].append(outputMerge)
424  plotJob['dependencies'].append(job['name'])
425 
426 
427  for mergeJob in mergeJobs:
428  alignment, mergeJobName, mergeIOV = mergeJob["name"].split("_")[1:]
429  if mergeJobName in mergesMC:
430  job["dependencies"].append(mergeJob["name"])
431 
432 
433  avpJobs.append(job)
434 
435 
436  if len(plotJob['inputData'])+len(plotJob['inputMC']) > 0:
437  local = {}
438  local["type"] = "DMR"
439  local["mode"] = "plot"
440  local["isData"] = True if len(plotJob['inputData']) > 0 else False
441  local["isMC"] = True if len(plotJob['inputMC']) > 0 else False
442  local["output"] = plotJob['output']
443  local["plot"] = { "inputData" : plotJob['inputData'],
444  "inputMC" : plotJob['inputMC'],
445  "alignments" : [],
446  "objects" : [],
447  "labels" : [],
448  "colors" : [],
449  "styles" : [],
450  "useFit" : True,
451  "useFitError" : False,
452  "showMean" : True,
453  "showMeanError" : False,
454  "showRMS" : False,
455  "showRMSError" : False}
456 
457  for mergeName in mergesDATA+mergesMC:
458  for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
459  for alignment in config["validations"]["DMR"]["single"][singleName]['alignments']:
460  if alignment in config['alignments'] and alignment not in local["plot"]["alignments"]:
461  local["plot"]["alignments"].append(alignment)
462  objectName = config["alignments"][alignment]["title"].replace(" ","_")
463  if objectName not in local["plot"]["objects"]: #can happen for MC
464  local["plot"]["objects"].append(objectName)
465  local["plot"]["labels"].append(config["alignments"][alignment]["title"])
466  local["plot"]["colors"].append(config["alignments"][alignment]["color"])
467  local["plot"]["styles"].append(config["alignments"][alignment]["style"])
468 
469  for extraKey in ["objects","labels","colors","styles","useFit","useFitError","showMean","showMeamError","showRMS","showRMSError"]:
470  if extraKey in config["validations"]["DMR"][dmrType][avpName].keys():
471  local["plot"][extraKey] = config["validations"]["DMR"][dmrType][avpName][extraKey]
472 
473  if "style" in config.keys():
474  if "DMR" in config['style'].keys():
475  if dmrType in config['style']['DMR'].keys():
476  local["plotGlobal"] = copy.deepcopy(config["style"]['DMR'][dmrType])
477 
478 
479  job = {
480  "name": "DMR_{}_{}_{}".format(dmrType, avpName, "plot"),
481  "dir": plotJob['workdir'],
482  "exe": "mkLumiAveragedPlots.py",
483  "run-mode": "Condor",
484  "dependencies": plotJob['dependencies'],
485  "config": local,
486  }
487  avpJobs.append(job)
488 
489  #Finally extend main job collection
490  jobs.extend(avpJobs)
491 
492  return jobs
493 
python.rootplot.root2matplotlib.replace
def replace(string, replacements)
Definition: root2matplotlib.py:444
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
DMR.DMR
def DMR(config, validationDir)
Definition: DMR.py:4
mps_setup.append
append
Definition: mps_setup.py:85
FastTimerService_cff.range
range
Definition: FastTimerService_cff.py:34
str
#define str(s)
Definition: TestProcessor.cc:56
Generated for CMSSW Reference Manual by doxygen 1.8.14