CMS 3D CMS Logo
hgcalPerformanceValidation.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 #------------------------------------------------------------------------------------------
4 # Description: This script is used to produce the results of the regurarly announced RelVal campaings.
5 # Essentially, this is a wrapper around the basic HGCal Validation script:
6 # Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py
7 # Documentation: Full documentation on this script and details on how to run it are in the section
8 # "Campaign Validation" in the HGCal DPG website:
9 # http://hgcal.web.cern.ch/hgcal/Validation/RelVals/
10 # Information on the CMSSW HGCalValidation package and relevant objects can be
11 # found in the "Validation" section.
12 #------------------------------------------------------------------------------------------
13 
14 import sys
15 import os
16 import commands
17 import optparse
18 import pandas as pd
19 
20 from Validation.RecoTrack.plotting.validation import Sample, Validation
21 from Validation.HGCalValidation.html import _sampleName,_pageNameMap,_summary,_summobj
22 
23 #------------------------------------------------------------------------------------------
24 #Parsing input options
25 def parseOptions():
26 
27  usage = ('usage: %prog [options]\n'
28  + '%prog -h for help')
29  parser = optparse.OptionParser(usage)
30 
31  parser.add_option('', '--Obj', dest='OBJ', type='string', default=None, help='Object to run. Options are: Geometry, SimHits, Digis, RecHits, Calibrations, CaloParticles, hgcalLayerClusters')
32  parser.add_option('', '--html-validation-name', dest='HTMLVALNAME', type='string', default='', help='Could be either be hgcalLayerClusters or hgcalMultiClusters')
33  parser.add_option('-d', '--download', action='store_true', dest='DOWNLOAD', default=False, help='Download DQM files from RelVals')
34  parser.add_option('-g', '--gather', dest='GATHER', type='string', default=None, help='Objects to gather: hitValidation, hitCalibration, hgcalLayerClusters, hgcalMultiClusters, ticlMultiClustersFromTrackstersEM, ticlMultiClustersFromTrackstersHAD')
35  parser.add_option('-w', '--wwwarea', dest='WWWAREA', type='string', default='/eos/project/h/hgcaldpg/www', help='Objects to gather: hitValidation, hitCalibration, hgcalLayerClusters, hgcalMultiClusters, ticlMultiClustersFromTrackstersEM, ticlMultiClustersFromTrackstersHAD')
36  parser.add_option('-y', '--dry-run', action='store_true', dest='DRYRUN', default=False, help='perform a dry run (nothing is lauched).')
37  parser.add_option('-i', '--inputeosarea', dest='INPUT', type='string', default='/eos/cms/store/group/dpg_hgcal/comm_hgcal/apsallid/RelVals', help='Eos area where we will place all DQM files of the new and reference release campaign')
38 
39  # store options and arguments as global variables
40  global opt, args
41  (opt, args) = parser.parse_args()
42 
43 parseOptions()
44 
45 #------------------------------------------------------------------------------------------
46 #Some helpful functions
47 #Processing the external os commands
48 def processCmd(cmd, quite = 0):
49  print cmd
50  status, output = commands.getstatusoutput(cmd)
51  if (status !=0 and not quite):
52  print 'Error in processing command:\n ['+cmd+']'
53  print 'Output:\n ['+output+'] \n'
54  return output
55 
56 #PUtype
57 def putype(t):
58  if "_pmx" in NewRelease:
59  if "_pmx" in RefRelease:
60  return {"default": "pmx"+t}
61  return {"default": t, NewRelease: "pmx"+t}
62  return t
63 
64 #------------------------------------------------------------------------------------------
65 #Input section: Each time a new RelVal campaign is announced the following variables should
66 # be updated:
67 # NewRelease: The new release being validated.
68 # RefRelease: The reference release we want to test the new one against.
69 # thereleases: All releases validated for which we have validation results
70 # including the NewRelease.
71 # NotNormalRelease , NotNormalRefRelease: If one of the releases to be validated has
72 # used the DIGI on 11_0_0 put "raw". Otherwise put "normal".
73 # phase2samples_noPU: These are the noPU phase 2 RelVal samples that are regurarly
74 # produced.
75 # phase2samples_PU: These are the PU phase 2 RelVal samples that are regurarly
76 # produced.
77 # RefRepository, NewRepository: The path where the DQM files of the campaign
78 # will be placed.
79 #------------------------------------------------------------------------------------------
80 #thereleases = { "CMSSW 11_1_X" : ["CMSSW_11_1_0_pre4_GEANT4","CMSSW_11_1_0_pre3","CMSSW_11_1_0_pre2"] }
81 thereleases = { "CMSSW 11_2_X" : [
82  "CMSSW_11_2_0_pre7_vs_CMSSW_11_2_0_pre6",
83  "CMSSW_11_2_0_pre6_ROOT622_vs_CMSSW_11_2_0_pre6",
84  "CMSSW_11_2_0_pre6_vs_CMSSW_11_2_0_pre5",
85  "CMSSW_11_2_0_pre5_GEANT106_vs_CMSSW_11_2_0_pre5",
86  "CMSSW_11_2_0_pre5_vs_CMSSW_11_2_0_pre3",
87  "CMSSW_11_2_0_pre3_vs_CMSSW_11_2_0_pre1",
88  "CMSSW_11_2_0_pre1_vs_CMSSW_11_1_0_pre8"
89  ],
90  "CMSSW 11_1_X" : [
91  "CMSSW_11_1_0_pre8_raw1100_vs_CMSSW_11_1_0_pre8",
92  "CMSSW_11_1_0_pre8_raw1100_vs_CMSSW_11_1_0_pre7_raw1100",
93  "CMSSW_11_1_0_pre8_vs_CMSSW_11_1_0_pre7",
94  "CMSSW_11_1_0_pre7_raw1100_vs_CMSSW_11_1_0_pre7",
95  "CMSSW_11_1_0_pre7_raw1100_vs_CMSSW_11_1_0_pre6_raw1100",
96  "CMSSW_11_1_0_pre7_vs_CMSSW_11_1_0_pre6",
97  "CMSSW_11_1_0_pre6_raw1100_vs_CMSSW_11_1_0_pre6",
98  "CMSSW_11_1_0_pre6_raw1100_vs_CMSSW_11_1_0_pre5_raw1100",
99  "CMSSW_11_1_0_pre6_vs_CMSSW_11_1_0_pre5",
100  "CMSSW_11_1_0_pre5_vs_CMSSW_11_1_0_pre4",
101  "CMSSW_11_1_0_pre5_raw1100_vs_CMSSW_11_1_0_pre5",
102  "CMSSW_11_1_0_pre5_raw1100_vs_CMSSW_11_1_0_pre4_raw1100",
103  "CMSSW_11_1_0_pre4_raw1100_vs_CMSSW_11_1_0_pre4",
104  "CMSSW_11_1_0_pre4_raw1100_vs_CMSSW_11_1_0_pre3_raw1100",
105  "CMSSW_11_1_0_pre4_GEANT4","CMSSW_11_1_0_pre4"
106  ]
107 }
108 
109 RefRelease='CMSSW_11_2_0_pre7'
110 
111 NewRelease='CMSSW_11_2_0_pre7'
112 
113 NotNormalRelease = "raw"
114 NotNormalRefRelease = "normal"
115 
116 if ( os.path.isdir('%s/%s' %(opt.WWWAREA, NewRelease))) :
117  print("The campaign you are trying to validate has already an existing validation folder in the official www area.")
118  print("Make sure you are not overwriting anything and try again.")
119  exit()
120 
121 if "raw" in NotNormalRelease:
122 # appendglobaltag = "_2026D49noPU_raw1100_rsb"
123 # appendglobaltag = "_2026D49noPU_raw1100"
124  appendglobaltag = "_2026D49noPU_gcc900"
125 else:
126  appendglobaltag = "_2026D49noPU"
127 
128 #Until the final list of RelVals settles down the following sample list is under constant review
129 '''
130 phase2samples_noPU_oldnaming = [
131 # Sample("RelValCloseByParticleGun_CE_H_Fine_300um", dqmVersion="0002", scenario="2026D49", appendGlobalTag=appendglobaltag ),
132  Sample("RelValCloseByParticleGun_CE_H_Fine_300um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
133  Sample("RelValCloseByParticleGun_CE_H_Fine_200um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
134  Sample("RelValCloseByParticleGun_CE_H_Fine_120um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
135  Sample("RelValCloseByParticleGun_CE_H_Coarse_Scint", scenario="2026D49", appendGlobalTag=appendglobaltag ),
136  Sample("RelValCloseByParticleGun_CE_H_Coarse_300um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
137  Sample("RelValCloseByParticleGun_CE_E_Front_300um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
138  Sample("RelValCloseByParticleGun_CE_E_Front_200um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
139  Sample("RelValCloseByParticleGun_CE_E_Front_120um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
140  #Sample("RelValCloseByParticleGun_CE_E_Front_120um", scenario="2026D49", appendGlobalTag=appendglobaltag , version="v2"),
141  Sample("RelValTTbar", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
142  Sample("RelValZMM", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
143  Sample("RelValSingleGammaFlatPt8To150", scenario="2026D49", appendGlobalTag=appendglobaltag ),
144  Sample("RelValSingleMuPt10", scenario="2026D49", appendGlobalTag=appendglobaltag ),
145  Sample("RelValSingleMuPt100", scenario="2026D49", appendGlobalTag=appendglobaltag ),
146  Sample("RelValSingleMuPt1000", scenario="2026D49", appendGlobalTag=appendglobaltag ),
147  Sample("RelValSingleMuFlatPt2To100", scenario="2026D49", appendGlobalTag=appendglobaltag ),
148  Sample("RelValSingleMuFlatPt0p7To10", scenario="2026D49", appendGlobalTag=appendglobaltag ),
149  Sample("RelValSingleEFlatPt2To100", scenario="2026D49", appendGlobalTag=appendglobaltag ),
150  Sample("RelValSingleTauFlatPt2To150", scenario="2026D49", appendGlobalTag=appendglobaltag ),
151  Sample("RelValSinglePiFlatPt0p7To10", scenario="2026D49", appendGlobalTag=appendglobaltag ),
152  Sample("RelValQCD_Pt20toInfMuEnrichPt15", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
153  Sample("RelValQCD_Pt15To7000_Flat", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
154  Sample("RelValZTT", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
155  Sample("RelValZMM", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
156  Sample("RelValZEE", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
157  Sample("RelValB0ToKstarMuMu", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
158  Sample("RelValBsToEleEle", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
159  Sample("RelValBsToMuMu", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
160  Sample("RelValBsToJpsiGamma", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
161  Sample("RelValBsToJpsiPhi_mumuKK", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
162  Sample("RelValBsToPhiPhi_KKKK", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
163  Sample("RelValDisplacedMuPt30To100", scenario="2026D49", appendGlobalTag=appendglobaltag ),
164  Sample("RelValDisplacedMuPt2To10", scenario="2026D49", appendGlobalTag=appendglobaltag ),
165  Sample("RelValDisplacedMuPt10To30", scenario="2026D49", appendGlobalTag=appendglobaltag ),
166  Sample("RelValTauToMuMuMu", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
167  #Sample("RelValMinBias", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
168  Sample("RelValH125GGgluonfusion", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
169  Sample("RelValNuGun", scenario="2026D49", appendGlobalTag=appendglobaltag )
170  ]
171 '''
172 
173 #Main workflow RelVals
174 phase2samples_noPU = [
175 
176  #Sample("RelValZpTT_1500", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
177 
190 
191 
192  Sample("RelValZpTT_1500", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
193  #Sample("RelValZpTT_1500", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
194  Sample("RelValZTT", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
195  Sample("RelValZMM", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
196  Sample("RelValZEE", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
197  #Sample("RelValTenTau_15_500_Eta3p1", scenario="2026D49", appendGlobalTag=appendglobaltag ),
198  Sample("RelValTenTau_15_500", scenario="2026D49", appendGlobalTag=appendglobaltag ),
199  Sample("RelValTTbar", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
200  Sample("RelValQCD_Pt15To7000_Flat", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
201  #Sample("RelValQCD_Pt15To7000_Flat", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
202  Sample("RelValNuGun", scenario="2026D49", appendGlobalTag=appendglobaltag ),
203  #Sample("RelValMinBias", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
204  Sample("RelValMinBias", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
205  Sample("RelValH125GGgluonfusion", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag )
206 
207 
208 ]
209 
210 
211 #More workflows
212 phase2samples_noPU_extend = [
213 
214  #Sample("RelValSingleMuPt10", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2"),
215  #Sample("RelValSingleMuPt100", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
216  #Sample("RelValSingleMuPt1000", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" )
217 
218 
219 
220  Sample("RelValSingleMuPt10", scenario="2026D49", appendGlobalTag=appendglobaltag ),
221  Sample("RelValSingleMuPt100", scenario="2026D49", appendGlobalTag=appendglobaltag ),
222  Sample("RelValSingleMuPt1000", scenario="2026D49", appendGlobalTag=appendglobaltag )
223 
224 ]
225 
226 #These workflows were added in CMSSW_11_1_0_pre6 but there were missing from CMSSW_11_1_0_pre5.
227 #So, I am only download them to be reary for pre7. Then, I comment them out
228 #For the moment I cannot find these in pre7.
229 phase2samples_noPU_extend_more = [
230 
231  #Sample("RelValCloseByPGun_CE_H_Fine_300um", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
232  #Sample("RelValCloseByPGun_CE_H_Fine_200um", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
233  #Sample("RelValCloseByPGun_CE_H_Fine_120um", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
234  #Sample("RelValCloseByPGun_CE_H_Coarse_Scint", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
235  #Sample("RelValCloseByPGun_CE_H_Coarse_300um", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
236  #Sample("RelValCloseByPGun_CE_E_Front_300um", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
237  #Sample("RelValCloseByPGun_CE_E_Front_200um", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
238  #Sample("RelValCloseByPGun_CE_E_Front_120um", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
239  #Sample("RelValSingleGammaFlatPt8To150", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
240  #Sample("RelValSingleEFlatPt2To100", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" ),
241  #Sample("RelValSinglePiFlatPt0p7To10", scenario="2026D49", appendGlobalTag=appendglobaltag, version="v2" )
242 
243 
244 
245 
246  Sample("RelValCloseByPGun_CE_H_Fine_300um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
247  Sample("RelValCloseByPGun_CE_H_Fine_200um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
248  Sample("RelValCloseByPGun_CE_H_Fine_120um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
249  Sample("RelValCloseByPGun_CE_H_Coarse_Scint", scenario="2026D49", appendGlobalTag=appendglobaltag ),
250  Sample("RelValCloseByPGun_CE_H_Coarse_300um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
251  Sample("RelValCloseByPGun_CE_E_Front_300um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
252  Sample("RelValCloseByPGun_CE_E_Front_200um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
253  Sample("RelValCloseByPGun_CE_E_Front_120um", scenario="2026D49", appendGlobalTag=appendglobaltag ),
254  Sample("RelValSingleGammaFlatPt8To150", scenario="2026D49", appendGlobalTag=appendglobaltag ),
255  Sample("RelValSingleEFlatPt2To100", scenario="2026D49", appendGlobalTag=appendglobaltag ),
256  Sample("RelValSinglePiFlatPt0p7To10", scenario="2026D49", appendGlobalTag=appendglobaltag )
257 
258 
259  #Sample("RelValCloseByPGun_CE_H_Fine_300um", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
260  #Sample("RelValCloseByPGun_CE_H_Fine_200um", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
261  #Sample("RelValCloseByPGun_CE_H_Fine_120um", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
262  #Sample("RelValCloseByPGun_CE_H_Coarse_Scint", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
263  #Sample("RelValCloseByPGun_CE_H_Coarse_300um", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
264  #Sample("RelValCloseByPGun_CE_E_Front_300um", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
265  #Sample("RelValCloseByPGun_CE_E_Front_200um", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
266  #Sample("RelValCloseByPGun_CE_E_Front_120um", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
267  #Sample("RelValSingleGammaFlatPt8To150", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
268  #Sample("RelValSingleEFlatPt2To100", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" ),
269  #Sample("RelValSinglePiFlatPt0p7To10", scenario="2026D49", appendGlobalTag=appendglobaltag + "_HGCal" )
270 
271 
272  #Sample("RelValQCD_Pt20toInfMuEnrichPt15", midfix="14", scenario="2026D49", appendGlobalTag=appendglobaltag ),
273  #Sample("RelValDisplacedMuPt30To100", scenario="2026D49", appendGlobalTag=appendglobaltag ),
274  #Sample("RelValDisplacedMuPt2To10", scenario="2026D49", appendGlobalTag=appendglobaltag ),
275  #Sample("RelValDisplacedMuPt10To30", scenario="2026D49", appendGlobalTag=appendglobaltag ),
276  #Sample("RelValB0ToKstarMuMu", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
277  #Sample("RelValBsToEleEle", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
278  #Sample("RelValBsToMuMu", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
279  #Sample("RelValBsToJpsiGamma", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
280  #Sample("RelValBsToJpsiPhi_mumuKK", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
281  #Sample("RelValBsToPhiPhi_KKKK", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
282  #Sample("RelValTauToMuMuMu", midfix="14TeV", scenario="2026D49", appendGlobalTag=appendglobaltag ),
283  #Sample("RelValSingleTauFlatPt2To150", scenario="2026D49", appendGlobalTag=appendglobaltag ),
284  #Sample("RelValSingleMuFlatPt0p7To10", scenario="2026D49", appendGlobalTag=appendglobaltag )
285 
286 ]
287 
288 phase2samples_noPU.extend(phase2samples_noPU_extend)
289 phase2samples_noPU.extend(phase2samples_noPU_extend_more)
290 #phase2samples_noPU.extend(phase2samples_noPU_oldnaming)
291 
292 #For the PU samples
293 phase2samples_PU = [
294  Sample("RelValTTbar", midfix="14TeV", scenario="2026D49", putype=putype("25ns"), punum=200, appendGlobalTag="_2026D49PU200", version="v2"),
295 ]
296 
297 # Reference and new repository
298 RefRepository = '%s' %(opt.INPUT)
299 NewRepository = '%s' %(opt.INPUT)
300 
301 #------------------------------------------------------------------------------------------
302 #Download section: The basic HGCal validation object is created and the DQM files of the
303 # DQM files of the RelVals are downloaded.
304 #------------------------------------------------------------------------------------------
305 
306 #Create basic object for the HGCal validation plots
307 val = Validation(
308  fullsimSamples = phase2samples_noPU,
309  fastsimSamples = [],
310  refRelease=RefRelease, refRepository=RefRepository,
311  newRelease=NewRelease, newRepository=NewRepository
312 )
313 
314 #------------------------------------------------------------------------------------------
315 #Download the DQM files of the RelVals.
316 if(opt.DOWNLOAD):
317  val.download()
318 
319  #Keep them in eos, save afs space.
320  if (not os.path.isdir(RefRepository+'/'+NewRelease)) :
321  processCmd('mkdir -p '+RefRepository+'/'+NewRelease)
322 
323  for infi in phase2samples_noPU:
324  if "_HGCal" in infi.filename(NewRelease):
325  processCmd('mv ' + infi.filename(NewRelease) + ' ' + infi.filename(NewRelease).replace("_HGCal",""))
326  processCmd('mv ' + infi.filename(NewRelease).replace("_HGCal","") + ' ' + RefRepository+'/'+NewRelease)
327  else:
328  #processCmd('mv ' + infi.filename(NewRelease) + ' ' + infi.filename(NewRelease).replace("2026D49noPU-v2","2026D49noPU-v1"))
329  #processCmd('mv ' + infi.filename(NewRelease).replace("2026D49noPU-v2","2026D49noPU-v1") + ' ' + RefRepository+'/'+NewRelease)
330  processCmd('mv ' + infi.filename(NewRelease) + ' ' + RefRepository+'/'+NewRelease)
331 
332 #------------------------------------------------------------------------------------------
333 #Objects processing section: The objects defined in --Obj are analyzed here.
334 #------------------------------------------------------------------------------------------
335 
336 #This is the hgcalLayerClusters, ticlMultiClustersFromTrackstersEM, ticlMultiClustersFromTrackstersHAD, and hitCalibration part
337 if (opt.OBJ == 'hgcalLayerClusters' or opt.OBJ == 'hitCalibration' or opt.OBJ == 'ticlMultiClustersFromTrackstersEM' or opt.OBJ == 'ticlMultiClustersFromTrackstersHAD'):
338  fragments = []
339  #Now that we have them in eos lets produce plots
340  #Let's loop through RelVals
341  for infi in phase2samples_noPU:
342  samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").replace("__DQMIO.root","")
343  #samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").split("__CMSSW_10_6_0_pre4",1)[0]
344  #samplename = samplename + infi.pileup()
345  if infi.pileup() == "PU":
346  samplename = samplename + str(infi.pileupNumber())
347 
348  #print( infi.name() )
349  print(_sampleName[infi.name()])
350  print("="*40)
351  print(samplename)
352  print("="*40)
353 
354  inputpathRef = ""
355  if RefRelease != None: inputpathRef = RefRepository +'/' + RefRelease +'/'
356  inputpathNew = NewRepository +'/' + NewRelease+ '/'
357 
358  if RefRelease == None:
359  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename)+ ' --collection %s' %(opt.HTMLVALNAME)
360  elif "raw" in NotNormalRelease and "normal" in NotNormalRefRelease:
361  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("mcRun4_realistic_v2_2026D49noPU_gcc900-v1","mcRun4_realistic_v2_2026D49noPU-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
362  #cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("mcRun4_realistic_v3_2026D49noPU_raw1100_rsb-v1","mcRun4_realistic_v3_2026D49noPU-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
363  elif "raw" in NotNormalRelease and "raw" in NotNormalRefRelease:
364  #cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease) + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
365  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("_raw1100","_raw1100_rsb") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
366  elif "normal" in NotNormalRelease and "normal" in NotNormalRefRelease:
367  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease) + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
368  else:
369  #print inputpathRef, infi.filename(RefRelease).replace("D49","D41")
370  #YOU SHOULD INSPECT EACH TIME THIS COMMAND AND THE REPLACE
371  #cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("D49","D41").replace("200-v2","200-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME) .replace("v2__", "v1__")
372  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("mcRun4_realistic_v2-v1", "mcRun4_realistic_v2_2026D49noPU-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
373  #print cmd
374 
375  if(opt.DRYRUN):
376  print 'Dry-run: ['+cmd+']'
377  else:
378  output = processCmd(cmd)
379  if opt.OBJ == 'hgcalLayerClusters':
380  processCmd('awk \'NR>=6&&NR<=44\' HGCValid_%s_Plots/index.html > HGCValid_%s_Plots/index_%s.html '% (opt.HTMLVALNAME,opt.HTMLVALNAME, samplename))
381  if opt.OBJ == 'hitCalibration':
382  processCmd('awk \'NR>=6&&NR<=15\' HGCValid_%s_Plots/index.html > HGCValid_%s_Plots/index_%s.html '% (opt.HTMLVALNAME,opt.HTMLVALNAME, samplename))
383  if opt.OBJ == 'ticlMultiClustersFromTrackstersEM' or opt.OBJ == 'ticlMultiClustersFromTrackstersHAD':
384  processCmd('awk \'NR>=6&&NR<=25\' HGCValid_%s_Plots/index.html > HGCValid_%s_Plots/index_%s.html '% (opt.HTMLVALNAME,opt.HTMLVALNAME, samplename))
385 
386 
387  fragments.append( 'HGCValid_%s_Plots/index_%s.html'% (opt.HTMLVALNAME, samplename) )
388 
389 
390  #Let's also create the final index xml file.
391  processCmd('mv HGCValid_%s_Plots/index.html HGCValid_%s_Plots/test.html' %(opt.HTMLVALNAME,opt.HTMLVALNAME) )
392  index_file = open('HGCValid_%s_Plots/index.html'%(opt.HTMLVALNAME),'w')
393  #Write preamble
394  index_file.write('<html>\n')
395  index_file.write(' <head>\n')
396  index_file.write(' <title>HGCal validation %s </title>\n' %(opt.HTMLVALNAME) )
397  index_file.write(' </head>\n')
398  index_file.write(' <body>\n')
399 
400  for frag in fragments:
401  with open(frag,'r') as f:
402  lines = f.read().splitlines()
403  for line in lines:
404  print line
405  index_file.write(line + '\n')
406  #processCmd( 'cat ' + frag + ' >> HGCalValidationPlots/index.html ' )
407  #index_file.write(frag)
408 
409 
410  #Writing postamble"
411  index_file.write(' </body>\n')
412  index_file.write('</html>\n')
413  index_file.close()
414 
415 #------------------------------------------------------------------------------------------
416 #This is the SimHits part
417 if (opt.OBJ == 'SimHits'):
418  #This is where we will save the final output pngs:
419  if (not os.path.isdir("hgcalSimHitStudy")) :
420  processCmd('mkdir -p hgcalSimHitStudy')
421  #Prepare for www
422  processCmd('cp %s/../public/index.php hgcalSimHitStudy/.'%(opt.WWWAREA) )
423 
424  #The input to this is for the moment 100 GeV muon from runnin cmsRun runHGCalSimHitStudy_cfg.py
425  #Input: hgcSimHits.root
426  cmd = 'root.exe -b -q Validation/HGCalValidation/macros/validationplots.C\(\\"hgcSimHit.root' + '\\",\\"'+ opt.OBJ + '\\"\)'
427  if(opt.DRYRUN):
428  print 'Dry-run: ['+cmd+']'
429  else:
430  output = processCmd(cmd)
431 
432 #------------------------------------------------------------------------------------------
433 if (opt.OBJ == 'hitValidation'):
434  fragments = []
435  #Now that we have them in eos lets produce plots
436  #Let's loop through RelVals
437  for infi in phase2samples_noPU:
438  samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").replace("__DQMIO.root","")
439  #samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").split("__CMSSW_10_6_0_pre4",1)[0]
440  #samplename = samplename + infi.pileup()
441  if infi.pileup() == "PU":
442  samplename = samplename + str(infi.pileupNumber())
443 
444  print("="*40)
445  print(samplename)
446  print("="*40)
447 
448  inputpathRef = ""
449  if RefRelease != None: inputpathRef = RefRepository +'/' + RefRelease +'/'
450  inputpathNew = NewRepository +'/' + NewRelease+ '/'
451 
452  if RefRelease == None:
453  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename)+ ' --collection %s' %(opt.HTMLVALNAME)
454  elif "raw" in NotNormalRelease and "normal" in NotNormalRefRelease:
455  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("mcRun4_realistic_v2_2026D49noPU_gcc900-v1","mcRun4_realistic_v2_2026D49noPU-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
456  #cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("mcRun4_realistic_v3_2026D49noPU_raw1100_rsb-v1","mcRun4_realistic_v3_2026D49noPU-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
457  elif "raw" in NotNormalRelease and "raw" in NotNormalRefRelease:
458  #cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease) + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
459  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("_raw1100","_raw1100_rsb") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
460  elif "normal" in NotNormalRelease and "normal" in NotNormalRefRelease:
461  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease) + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
462  else:
463  #cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("D49","D41").replace("200-v2","200-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
464  cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease).replace("mcRun4_realistic_v2-v1", "mcRun4_realistic_v2_2026D49noPU-v1") + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
465  #cmd = 'python Validation/HGCalValidation/scripts/makeHGCalValidationPlots.py ' + inputpathRef + infi.filename(RefRelease) + ' ' + inputpathNew + infi.filename(NewRelease) + ' --outputDir HGCValid_%s_Plots --no-ratio --png --separate --html-sample "%s" ' %(opt.HTMLVALNAME, _sampleName[infi.name()] ) + ' --html-validation-name %s --subdirprefix ' %(opt.HTMLVALNAME) + ' plots_%s' % (samplename) + ' --collection %s' %(opt.HTMLVALNAME)
466 
467 
468  if(opt.DRYRUN):
469  print 'Dry-run: ['+cmd+']'
470  else:
471  output = processCmd(cmd)
472  processCmd('awk \'NR>=6&&NR<=28\' HGCValid_%s_Plots/index.html > HGCValid_%s_Plots/index_%s.html '% (opt.HTMLVALNAME,opt.HTMLVALNAME, samplename))
473 
474  fragments.append( 'HGCValid_%s_Plots/index_%s.html'% (opt.HTMLVALNAME, samplename) )
475 
476 
477  #Let's also create the final index xml file.
478  processCmd('mv HGCValid_%s_Plots/index.html HGCValid_%s_Plots/test.html' %(opt.HTMLVALNAME,opt.HTMLVALNAME) )
479  index_file = open('HGCValid_%s_Plots/index.html'%(opt.HTMLVALNAME),'w')
480  #Write preamble
481  index_file.write('<html>\n')
482  index_file.write(' <head>\n')
483  index_file.write(' <title>HGCal validation %s </title>\n' %(opt.HTMLVALNAME) )
484  index_file.write(' </head>\n')
485  index_file.write(' <body>\n')
486 
487  for frag in fragments:
488  with open(frag,'r') as f:
489  lines = f.read().splitlines()
490  for line in lines:
491  print line
492  index_file.write(line + '\n')
493  #processCmd( 'cat ' + frag + ' >> HGCalValidationPlots/index.html ' )
494  #index_file.write(frag)
495 
496 
497  #Writing postamble"
498  index_file.write(' </body>\n')
499  index_file.write('</html>\n')
500  index_file.close()
501 
502 #-------------------------------------------------------------------------------------------
503 #This is the Digis part
504 if (opt.OBJ == 'Digis'):
505  #This is where we will save the final output pngs:
506  if (not os.path.isdir("hgcalDigiStudy")) :
507  processCmd('mkdir -p hgcalDigiStudy')
508  processCmd('mkdir -p hgcalDigiStudyEE')
509  processCmd('mkdir -p hgcalDigiStudyHEF')
510  processCmd('mkdir -p hgcalDigiStudyHEB')
511  #Prepare for www
512  processCmd('cp %s/../public/index.php hgcalDigiStudy/.'%(opt.WWWAREA) )
513  processCmd('cp %s/../public/index.php hgcalDigiStudyEE/.'%(opt.WWWAREA) )
514  processCmd('cp %s/../public/index.php hgcalDigiStudyHEF/.'%(opt.WWWAREA) )
515  processCmd('cp %s/../public/index.php hgcalDigiStudyHEB/.'%(opt.WWWAREA) )
516  #The input here is from running cmsRun runHGCalDigiStudy_cfg.py, to which
517  #we usually give ttbar noPU as input
518  #Input: hgcDigi.root
519  cmd = 'root.exe -b -q Validation/HGCalValidation/macros/validationplots.C\(\\"hgcDigi.root' + '\\",\\"'+ opt.OBJ + '\\"\)'
520  if(opt.DRYRUN):
521  print 'Dry-run: ['+cmd+']'
522  else:
523  output = processCmd(cmd)
524  #mv the output under the main directory
525  processCmd('mv hgcalDigiStudyEE hgcalDigiStudy/.')
526  processCmd('mv hgcalDigiStudyHEF hgcalDigiStudy/.')
527  processCmd('mv hgcalDigiStudyHEB hgcalDigiStudy/.')
528 
529 #-------------------------------------------------------------------------------------------
530 #This is the RecHits part
531 if (opt.OBJ == 'RecHits'):
532  #This is where we will save the final output pngs:
533  if (not os.path.isdir("hgcalRecHitStudy")) :
534  processCmd('mkdir -p hgcalRecHitStudy')
535  processCmd('mkdir -p hgcalRecHitStudyEE')
536  processCmd('mkdir -p hgcalRecHitStudyHEF')
537  processCmd('mkdir -p hgcalRecHitStudyHEB')
538  #Prepare for www
539  processCmd('cp %s/../public/index.php hgcalRecHitStudy/.'%(opt.WWWAREA) )
540  processCmd('cp %s/../public/index.php hgcalRecHitStudyEE/.'%(opt.WWWAREA) )
541  processCmd('cp %s/../public/index.php hgcalRecHitStudyHEF/.'%(opt.WWWAREA) )
542  processCmd('cp %s/../public/index.php hgcalRecHitStudyHEB/.'%(opt.WWWAREA) )
543  #The input here is from running cmsRun runHGCalRecHitStudy_cfg.py, to which
544  #we usually give ttbar noPU as input
545  #Input: hgcRecHit.root
546  cmd = 'root.exe -b -q Validation/HGCalValidation/macros/validationplots.C\(\\"hgcRecHit.root' + '\\",\\"'+ opt.OBJ + '\\"\)'
547  if(opt.DRYRUN):
548  print 'Dry-run: ['+cmd+']'
549  else:
550  output = processCmd(cmd)
551  #mv the output under the main directory
552  processCmd('mv hgcalRecHitStudyEE hgcalRecHitStudy/.')
553  processCmd('mv hgcalRecHitStudyHEF hgcalRecHitStudy/.')
554  processCmd('mv hgcalRecHitStudyHEB hgcalRecHitStudy/.')
555 
556 #-------------------------------------------------------------------------------------------
557 
558 if (opt.OBJ == 'CaloParticles'):
559  particletypes = ["-11","-13","-211","-321","11","111","13","211","22","321"]
560  #This is where we will save the final output pngs:
561  if (not os.path.isdir("CaloParticles")) :
562  processCmd('mkdir -p CaloParticles')
563  #Prepare for www
564  processCmd('cp %s/../public/index.php CaloParticles/.'%(opt.WWWAREA) )
565 
566  #Let's loop through RelVals
567  for infi in phase2samples_noPU:
568  #samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").replace("__DQMIO.root","")
569  samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").split("__"+NewRelease,1)[0]
570  samplename = samplename + infi.pileup()
571  if infi.pileup() == "PU":
572  samplename = samplename + str(infi.pileupNumber())
573 
574  print("="*40)
575  print(samplename)
576  print("="*40)
577  if (not os.path.isdir(samplename)) :
578  processCmd('mkdir -p ' + samplename )
579  processCmd('cp %s/RelVals/index.php '%(opt.WWWAREA) + samplename + '/.')
580  for part in particletypes:
581  processCmd('mkdir -p ' + samplename + '/' +part )
582  #Prepare for www
583  processCmd('cp %s/RelVals/index.php '%(opt.WWWAREA) + samplename + '/' +part + '/.')
584 
585  inputpathRef = ""
586  if RefRelease != None: inputpathRef = RefRepository +'/' + RefRelease +'/'
587  inputpathNew = NewRepository +'/' + NewRelease+ '/'
588  cmd = 'root.exe -b -q Validation/HGCalValidation/macros/validationplots.C\(\\"'+ inputpathNew + infi.filename(NewRelease) + '\\",\\"'+ opt.OBJ + '\\",\\"'+ samplename + '\\"\\)'
589  if(opt.DRYRUN):
590  print 'Dry-run: ['+cmd+']'
591  else:
592  output = processCmd(cmd)
593  processCmd('mv ' +samplename+ ' CaloParticles/.' )
594 
595 #------------------------------------------------------------------------------------------
596 #Summary section: After processing all the objects the results are gathered, webpages are
597 # created and a summary page is added.
598 #-------------------------------------------------------------------------------------------
599 #Here we will gather all results.
600 if (opt.GATHER != None) :
601 
602  #First we need the top folder to contain all validation releases.
603  index_file = open('index.html','w')
604  #Write preamble
605  index_file.write('<html>\n')
606  index_file.write(' <head>\n')
607  index_file.write(' <title>HGCAL validation results </title>\n' )
608  index_file.write(' </head>\n')
609  index_file.write(' <body>\n')
610  index_file.write(' <h1>\n')
611  index_file.write(' HGCAL Validation Results \n' )
612  index_file.write(' </h1>\n')
613  index_file.write(' <ul>\n' )
614 
615  for trel in thereleases.keys():
616  index_file.write(' <li>\n' )
617  index_file.write(' %s\n' %(trel) )
618  for rel in thereleases[trel]:
619  index_file.write(' <ul>\n' )
620  index_file.write(' <li><a href="%s/index.html">%s</a></li>\n' %(rel, rel ) )
621  index_file.write(' </ul>\n' )
622  index_file.write(' </li>\n' )
623  index_file.write(' <br>\n' )
624  index_file.write(' <br>\n' )
625  index_file.write(' <br>\n' )
626 
627  #Writing postamble"
628  index_file.write(' </ul>\n' )
629  index_file.write(' </body>\n')
630  index_file.write('</html>\n')
631  index_file.close()
632 
633  processCmd('cp index.html %s/.' %(opt.WWWAREA) )
634 
635  #Let's make also the summary folder
636  if (not os.path.isdir("HGCValid_summary_Plots")):
637  processCmd('mkdir -p HGCValid_summary_Plots')
638 
639  #To avoid the nans transpose later
640  df = pd.DataFrame.from_dict(_summary, orient = 'index').transpose()
641  #Make a specific order in columns
642  df = df[_summobj]
643 
644  index_file = open('HGCValid_summary_Plots/index.html','w')
645  #Write preamble
646  index_file.write('<html>\n')
647  index_file.write(' <body>\n')
648 
649  #Let's loop through RelVals
650  for infi in phase2samples_noPU:
651  samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").replace("__DQMIO.root","")
652  index_file.write( '<h2> %s </h2> \n' %(_sampleName[infi.name()]) )
653  #table here with summary objects
654  index_file.write('<table> \n')
655  index_file.write(' <tr>\n')
656  #This is the row with the headers. So, the objects for us.
657  for obj in _summobj:
658  index_file.write(' <th>%s</th>\n' %(_pageNameMap[obj]) )
659  index_file.write(' </tr>\n')
660 
661  for i, row in df.iterrows():
662  index_file.write(' <tr>\n')
663  for j, column in row.iteritems():
664  print(column)
665  index_file.write(' <td>\n')
666  index_file.write(' <ul>\n')
667 
668 # if df[obj][ind] == None:
669  if column == None:
670  index_file.write(' </ul>\n')
671  index_file.write(' </td>\n')
672  continue
673  #index_file.write(' \n')
674  else:
675  #print(df[obj][ind])
676  print(j)
677  #index_file.write(' <li><a href="plots_%s_%s">%s</a></li> \n' %(samplename, df[obj][ind], df[obj][ind].partition("/")[2] ))
678  index_file.write(' <li><a href="../HGCValid_%s_Plots/plots_%s_%s">%s</a></li> \n' %(j, samplename, column, column.partition("/")[2] ))
679 
680  index_file.write(' </ul>\n')
681  index_file.write(' </td>\n')
682 
683  index_file.write(' </tr>\n')
684 
685  index_file.write(' </table>\n')
686  index_file.write(' <br/>\n' )
687  index_file.write(' <br/>\n' )
688  index_file.write(' <br/>\n' )
689 
690  #Writing postamble"
691  index_file.write(' </body>\n')
692  index_file.write('</html>\n')
693  index_file.close()
694 
695  objects = opt.GATHER.split(",")
696 
697  localoutputdir = ""
698  if "raw" in NotNormalRelease and "raw" in NotNormalRefRelease:
699  localoutputdir = NewRelease + "_raw1100" + "_vs_" + RefRelease + "_raw1100"
700  elif "raw" in NotNormalRelease and "normal" in NotNormalRefRelease:
701  localoutputdir = NewRelease + "_raw1100" + "_vs_" + RefRelease
702  elif "normal" in NotNormalRelease and "normal" in NotNormalRefRelease:
703  localoutputdir = NewRelease + "_vs_" + RefRelease
704  else:
705  localoutputdir = NewRelease
706 
707  #make the structure to hold the objects
708  for obj in objects:
709  #This is where we will save the final output per campaing:
710  if (not os.path.isdir('%s/standalone' %(localoutputdir))) :
711  processCmd('mkdir -p %s/standalone' %(localoutputdir))
712  if (obj!="standalone"): processCmd('mv HGCValid_%s_Plots %s'%(obj, localoutputdir) )
713  else :
714  processCmd('mv hgcalSimHitStudy %s/standalone/.'%(localoutputdir) )
715  processCmd('mv hgcalDigiStudy %s/standalone/.'%(localoutputdir) )
716  processCmd('mv hgcalRecHitStudy %s/standalone/.'%(localoutputdir) )
717  processCmd('cp %s/../public/index.php %s/standalone/.'%(opt.WWWAREA, localoutputdir) )
718 
719  '''
720  #Let's also copy to the summary folder what we need.
721  for infi in phase2samples_noPU:
722  samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").replace("__DQMIO.root","")
723  for obj in _summobj:
724  #print obj
725  #if obj == "hitValidation" : samplename = samplename + infi.pileup()
726  #else : samplename = infi.filename(NewRelease).replace("DQM_V0001_R000000001__","").replace("__DQMIO.root","")
727  for ind in df.index:
728  if df[obj][ind] == None: continue
729  else: processCmd('cp -r %s/HGCValid_%s_Plots/plots_%s_%s %s/HGCValid_summary_Plots ' %(NewRelease, obj, samplename, df[obj][ind].partition("/")[0], NewRelease ) )
730  '''
731 
732  index_file = open('%s/index.html'%(localoutputdir),'w')
733  #Write preamble
734  index_file.write('<html>\n')
735  index_file.write(' <head>\n')
736  index_file.write(' <title> <h2> HGCal validation results for %s </h2> </title>\n' %(localoutputdir) )
737  index_file.write(' </head>\n')
738  index_file.write(' <body>\n')
739  index_file.write(' HGCal validation results for %s \n' %(localoutputdir) )
740 
741  for obj in objects:
742  print(obj)
743  if (obj!="standalone"):
744  index_file.write(' <br/>\n' )
745  index_file.write(' <ul>\n' )
746  index_file.write(' <li><a href="HGCValid_%s_Plots/index.html">%s</a></li>\n' %(obj, _pageNameMap[obj] ) )
747  index_file.write(' </ul>\n' )
748  index_file.write(' <br/>\n' )
749  else :
750  index_file.write(' <br/>\n' )
751  index_file.write(' <ul>\n' )
752  index_file.write(' <li><a href="%s/index.php">%s</a></li>\n' %(obj, _pageNameMap[obj] ) )
753  index_file.write(' </ul>\n' )
754  index_file.write(' <br/>\n' )
755 
756 
757  #Writing postamble
758  index_file.write(' </body>\n')
759  index_file.write('</html>\n')
760  index_file.close()
761 
762  #We choose to zip in uncompressed form all the files for two reasons:
763  #1. Copying to eos so many files is really slow. It is faster to
764  # create one uncompressed file, copy that and unzip there.
765  #2. Inevitably, we will have to do some cleanup of the older campaigns,
766  # since we will reach the number of files limit quite easily.
767  # It will be easier to have already save the zip file and just delete
768  # the directory content, leaving inside only the zip file.
769 
770  # This will take some time.
771  processCmd('zip -0 -r %s.zip %s' %(localoutputdir,localoutputdir) )
772  processCmd('cp %s.zip %s/.' %(localoutputdir,opt.WWWAREA) )
773  processCmd('cd %s' %(opt.WWWAREA) )
774  processCmd('unzip %s.zip' %(localoutputdir) )
775  processCmd('mv %s.zip %s/.' %(localoutputdir,localoutputdir) )
776  processCmd('cd -')
777 
778 
779 
geometryDiff.transpose
def transpose(a)
Definition: geometryDiff.py:39
hgcalPerformanceValidation.parseOptions
def parseOptions()
Definition: hgcalPerformanceValidation.py:25
submitPVValidationJobs.split
def split(sequence, size)
Definition: submitPVValidationJobs.py:352
str
#define str(s)
Definition: TestProcessor.cc:51
hgcalPerformanceValidation.processCmd
def processCmd(cmd, quite=0)
Definition: hgcalPerformanceValidation.py:48
hgcalPerformanceValidation.putype
putype
Definition: hgcalPerformanceValidation.py:294
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:46
beamvalidation.exit
def exit(msg="")
Definition: beamvalidation.py:53
python.rootplot.root2matplotlib.replace
def replace(string, replacements)
Definition: root2matplotlib.py:444
Generated for CMSSW Reference Manual by doxygen 1.8.16