CMS 3D CMS Logo
TkMap_script_phase1.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 from __future__ import print_function
4 import sys
5 from ROOT import *
6 import os
7 from subprocess import call
8 import os.path
9 import shutil
10 import subprocess
11 import codecs
12 import re
13 import errno
14 from getGTfromDQMFile_V2 import getGTfromDQMFile
15 
16 def setRunDirectory(runNumber):
17  # Don't forget to add an entry when there is a new era
18  dirDict = { 325799:['Data2018', 'HIRun2018'],\
19  315252:['Data2018', 'Run2018'],\
20  308336:['Data2018', 'Commissioning2018'],\
21  294644:['Data2017', 'Run2017'],\
22  290123:['Data2017', 'Commissioning2017'],\
23  284500:['Data2016', 'PARun2016'],\
24  271024:['Data2016', 'Run2016'],\
25  264200:['Data2016', 'Commissioning2016'],\
26  246907:['Data2015', 'Run2015'],\
27  232881:['Data2015', 'Commissioning2015'],\
28  211658:['Data2013', 'Run2013'],\
29  209634:['Data2013', 'HIRun2013'],\
30  190450:['Data2012', 'Run2012']}
31  runKey=0
32  for key in sorted(dirDict):
33  if runNumber > key:
34  runKey=key
35  return dirDict[runKey]
36 
37 def downloadOfflineDQMhisto(run, Run_type,rereco):
38  runDirval=setRunDirectory(run)
39  DataLocalDir=runDirval[0]
40  DataOfflineDir=runDirval[1]
41  nnn=run/100
42  print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
43  File_Name = ''
44  print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
45  url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
46  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
47  f=codecs.open("index.html", 'r')
48  index = f.readlines()
49  if any(str(Run_Number[i]) in s for s in index):
50  for s in index:
51  if rereco:
52  if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s) and ("17Sep2018" in s):
53  File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
54  else:
55  if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
56  File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
57 
58  else:
59  print('No DQM file available. Please check the Offline server')
60  sys.exit(0)
61 
62  print('Downloading DQM file:'+File_Name)
63  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
64 
65  return File_Name
66 
67 def downloadOfflinePCLhisto(run, Run_type):
68  runDirval=setRunDirectory(run)
69  DataLocalDir=runDirval[0]
70  DataOfflineDir=runDirval[1]
71  nnn=run/100
72  print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
73  File_Name = 'Temp'
74  print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
75  url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
76  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
77  f=codecs.open("index.html", 'r')
78  index = f.readlines()
79  if any(str(Run_Number[i]) in s for s in index):
80  for s in index:
81  if (str(Run_Number[i]) in s) and ("PromptCalibProdSiPixel-Express" in s) and ("__ALCAPROMPT.root" in s):
82  File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
83  else:
84  print('No DQM file available. Please check the Offline server')
85  sys.exit(0)
86  if File_Name!='Temp':
87  print('Downloading DQM file:'+File_Name)
88  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
89 
90  return File_Name
91 
92 
93 def downloadnlineDQMhisto(run, Run_type):
94  runDirval=setRunDirectory(run)
95  DataLocalDir=runDirval[0]
96  DataOfflineDir=runDirval[1]
97  nnn=run/100
98  nnnOnline = run/10000
99  File_Name_online=''
100  deadRocMap = False
101 
102  url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
103  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
104 
105  url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
106  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
107  f_online_backup=codecs.open("index_online_backup.html", 'r')
108  index_online_backup = f_online_backup.readlines()
109 
110  f_online=codecs.open("index_online.html", 'r')
111  index_online = f_online.readlines()
112  if any(str(run) in x for x in index_online):
113  for x in index_online:
114  if (str(run) in x) and ("_PixelPhase1_" in x):
115  File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
116  deadRocMap = True
117  else:
118  print("Can't find any file in offline server, trying the online server")
119  if any(str(run) in y for y in index_online_backup):
120  for y in index_online:
121  if (str(run) in y) and ("_PixelPhase1_" in y):
122  File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
123  deadRocMap = True
124  else:
125  print('No Online DQM file available. Skip dead roc map')
126  deadRocMap = False
127 
128  print('Downloading DQM file:'+File_Name_online)
129 
130 
131  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
132 
133  os.remove('index_online.html')
134  os.remove('index_online_backup.html')
135  return deadRocMap, File_Name_online;
136 
137 
138 def getGT(DQMfile, RunNumber, globalTagVar):
139  globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
140  print("Global Tag: " + globalTag_v0)
141  globalTag = globalTag_v0
142 
143  for z in range(len(globalTag_v0)-2):#clean up the garbage string in the GT
144  if (globalTag_v0[z].isdigit()) and (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
145  globalTag = globalTag_v0[z:]
146  break
147  if globalTag == "":
148  print(" No GlobalTag found: trying from DAS.... ")
149  globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
150  if globalTag == "":
151  print(" No GlobalTag found for run: "+str(Run_Number[i]))
152 
153  return globalTag
154 
155 
156 Run_type = sys.argv[1]
157 Run_Number = [int(x) for x in sys.argv[2:]]
158 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
159 rereco=False
160 
161 
162 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or Run_type == 'ZeroBias' or Run_type == 'StreamHIExpress' or Run_type == 'HIMinimumBias1' or re.match('ZeroBias([0-9]+?)',Run_type) or re.match('HIMinimumBias([0-9]+?)',Run_type):
163  print(Run_type)
164 elif Run_type == 'ReReco':
165  rereco=True
166  Run_type='ZeroBias'
167 else:
168  print("please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ")
169  sys.exit(0)
170 
171 for i in range(len(Run_Number)):
172 
173  print("Downloading File!!")
174  nnnOut = Run_Number[i]/1000
175 
176  filepath = '/tmp/'
177  File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type, rereco)
178  if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
179  File_Name_PCL = downloadOfflinePCLhisto(Run_Number[i], Run_type)
180  deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
181 
182  runDirval=setRunDirectory(Run_Number[i])
183  DataLocalDir=runDirval[0]
184  #DataOfflineDir=runDirval[1]
185 
186 
187 
188  print("get the run status from DQMFile")
189 
190 
191  check_command = 'check_runcomplete '+filepath+File_Name
192  Check_output = subprocess.call(check_command, shell=True)
193 
194 
195  if Check_output == 0:
196  print('Using DQM file: '+File_Name)
197  else:
198  print('*****************Warning: DQM file is not ready************************')
199  input_var = raw_input("DQM file is incompleted, do you want to continue? (y/n): ")
200  if (input_var == 'y') or (input_var == 'Y'):
201  print('Using DQM file: '+File_Name)
202  else:
203  sys.exit(0)
204  if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
205  if File_Name_PCL=='Temp':
206  print(' ')
207  print(' ')
208  print('*****************Warning: PCL file is not ready************************')
209  input_var = raw_input("PCL file is not ready, you will need to re-run the script later for PCL plots, do you want to continue? (y/n): ")
210  if (input_var == 'y') or (input_var == 'Y'):
211  print('--------> Remember to re-run the script later!!!!!')
212  else:
213  sys.exit(0)
214 
215 
216 
217  checkfolder = os.path.exists(str(Run_Number[i]))
218  if checkfolder == True:
219  shutil.rmtree(str(Run_Number[i]))
220  os.makedirs(str(Run_Number[i])+'/'+Run_type)
221  else:
222  os.makedirs(str(Run_Number[i])+'/'+Run_type)
223 
224 
226  globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
227 
228 
229 
230 
231  print(" Creating the TrackerMap.... ")
232 
233  detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
234  workPath = os.popen('pwd').readline().strip()
235 
236  os.chdir(str(Run_Number[i])+'/'+Run_type)
237 
238 
239  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
240  os.system('rm -f *svg')
241 
242  sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
243  shutil.move('QTBadModules.log',sefile)
244 
245 
246 
247 # PLEASE UNCOMMENT THE LINES BELOW TO GET THE LEGEND ON THE QT TkMAP (it will work only on vocms061)
248 # os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/LegendToQT.py QTestAlarm.png /data/users/cctrack/FinalLegendTrans.png')
249 # shutil.move('result.png', 'QTestAlarm.png')
250 
251 
252 
253  if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
254  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
255  elif Run_type == "StreamExpress":
256  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_Express_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
257  else:
258  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
259 
260  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
261  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
262 
263  print(" Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder")
264 
265  output =[]
266  output.append(os.popen("/bin/ls ").readline().strip())
267  print(output)
268 
269 
270  print(" Creating the list of bad modules ")
271 
272  os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
273 
274 
277 
278 
279 
280 
281 # print(" Creating the lumisection certification:")
282 
283 # if (Run_type.startswith("ZeroBias")) or (Run_type == "StreamExpress"):
284 # os.system('ls_cert 0.95 0.95 '+filepath+'/'+File_Name)
285 
286 
290 
291 
292  os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
293 
294 
295 
296 
297  dest='Beam'
298 
299  if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
300  dest="Cosmics"
301 
302 
303 
304  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
305  shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
306 
307  os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
308 
309 
310  shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
311 
312  os.remove(detIdInfoFileName)
313  os.remove('MergedBadComponentsTkMap_Canvas.root')
314  os.remove('MergedBadComponentsTkMap.root')
315 
316  print("countig dead pixel ROCs" )
317  if (Run_Number[i] < 290124) :
318 
319  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
320  else:
321  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
322 
323  if rereco:
324  os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco 2> /dev/null')
325  else:
326  os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
327 
328  shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
329 
330 
331 
332 
333 
334 
335  if deadRocMap == True:
336 
337  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
338  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
339  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
340  os.remove('MaskedROC_sum.txt')
341  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
342  else:
343  print('No Online DQM file available, Dead ROC maps will not be produced')
344  print('No Online DQM file available, inefficient DC list will also not be produced')
345 
346 
347 
348  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
349 
350 
351  if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
352  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLDeadROC.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
353  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLFedErr.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
354  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PCLOthers.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
355 
356 
357  strip_files = os.listdir('.')
358  for file_name in strip_files:
359  full_stripfile_name = os.path.join('.', file_name)
360  if (os.path.isfile(full_stripfile_name)):
361  if rereco:
362  shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
363  else:
364  shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
365 
366 
367 
368 
369 
370  os.chdir(workPath)
371  shutil.rmtree(str(Run_Number[i]))
372  os.remove('index.html')
373 
374  # produce pixel phase1 TH2Poly maps
375 # os.chdir(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/scripts/PhaseIMaps/')
376  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
377  shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
378 
379 
380 
381  pixel_files = os.listdir('./OUT')
382  for file_name in pixel_files:
383  full_pixelfile_name = os.path.join('./OUT/', file_name)
384  if (os.path.isfile(full_pixelfile_name)):
385  if rereco:
386  shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
387  else:
388  shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
389 
390 
391  shutil.rmtree('OUT')
392 
393 
394  # produce pixel phase1 tree for Offline TkCommissioner
395  pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
396  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
397 
398  shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
399  os.remove(pixelTreeFileName)
400  if File_Name:
401  os.remove(filepath+File_Name)
402  if File_Name_online:
403  os.remove(filepath+File_Name_online)
404  os.chdir(workPath)
FastTimerService_cff.range
range
Definition: FastTimerService_cff.py:34
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
TkMap_script_phase1.downloadOfflinePCLhisto
def downloadOfflinePCLhisto(run, Run_type)
Definition: TkMap_script_phase1.py:67
getGTfromDQMFile
Definition: getGTfromDQMFile.py:1
any
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:38
TkMap_script_phase1.setRunDirectory
def setRunDirectory(runNumber)
Definition: TkMap_script_phase1.py:16
submitPVValidationJobs.split
def split(sequence, size)
Definition: submitPVValidationJobs.py:352
str
#define str(s)
Definition: TestProcessor.cc:51
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:46
createfilelist.int
int
Definition: createfilelist.py:10
TkMap_script_phase1.downloadOfflineDQMhisto
def downloadOfflineDQMhisto(run, Run_type, rereco)
Definition: TkMap_script_phase1.py:37
TkMap_script_phase1.downloadnlineDQMhisto
def downloadnlineDQMhisto(run, Run_type)
Definition: TkMap_script_phase1.py:93
readEcalDQMStatus.read
read
Definition: readEcalDQMStatus.py:38
TkMap_script_phase1.getGT
def getGT(DQMfile, RunNumber, globalTagVar)
Definition: TkMap_script_phase1.py:138
Generated for CMSSW Reference Manual by doxygen 1.8.16