CMS 3D CMS Logo
TkMap_script_phase1.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 
3 from __future__ import print_function
4 import sys
5 from ROOT import *
6 import os
7 from subprocess import call
8 import os.path
9 import shutil
10 import subprocess
11 import codecs
12 import re
13 import errno
14 from getGTfromDQMFile_V2 import getGTfromDQMFile
15 
16 def setRunDirectory(runNumber):
17  # Don't forget to add an entry when there is a new era
18  dirDict = { 315252:['Data2018', 'Run2018'],\
19  308336:['Data2018', 'Commissioning2018'],\
20  294644:['Data2017', 'Run2017'],\
21  290123:['Data2017', 'Commissioning2017'],\
22  284500:['Data2016', 'PARun2016'],\
23  271024:['Data2016', 'Run2016'],\
24  264200:['Data2016', 'Commissioning2016'],\
25  246907:['Data2015', 'Run2015'],\
26  232881:['Data2015', 'Commissioning2015'],\
27  211658:['Data2013', 'Run2013'],\
28  209634:['Data2013', 'HIRun2013'],\
29  190450:['Data2012', 'Run2012']}
30  runKey=0
31  for key in sorted(dirDict):
32  if runNumber > key:
33  runKey=key
34  return dirDict[runKey]
35 
36 def downloadOfflineDQMhisto(run, Run_type):
37  runDirval=setRunDirectory(run)
38  DataLocalDir=runDirval[0]
39  DataOfflineDir=runDirval[1]
40  nnn=run/100
41  print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
42  File_Name = ''
43  print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
44  url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
45  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
46  f=codecs.open("index.html", 'r')
47  index = f.readlines()
48  if any(str(Run_Number[i]) in s for s in index):
49  for s in index:
50  if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
51  File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
52  else:
53  print('No DQM file available. Please check the Offline server')
54  sys.exit(0)
55 
56  print('Downloading DQM file:'+File_Name)
57  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
58 
59  return File_Name
60 
61 def downloadnlineDQMhisto(run, Run_type):
62  runDirval=setRunDirectory(run)
63  DataLocalDir=runDirval[0]
64  DataOfflineDir=runDirval[1]
65  nnn=run/100
66  nnnOnline = run/10000
67  File_Name_online=''
68  deadRocMap = False
69  ##################online file########
70  url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
71  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
72 
73  url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
74  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
75  f_online_backup=codecs.open("index_online_backup.html", 'r')
76  index_online_backup = f_online_backup.readlines()
77 
78  f_online=codecs.open("index_online.html", 'r')
79  index_online = f_online.readlines()
80  if any(str(run) in x for x in index_online):
81  for x in index_online:
82  if (str(run) in x) and ("_PixelPhase1_" in x):
83  File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
84  deadRocMap = True
85  else:
86  print("Can't find any file in offline server, trying the online server")
87  if any(str(run) in y for y in index_online_backup):
88  for y in index_online:
89  if (str(run) in y) and ("_PixelPhase1_" in y):
90  File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
91  deadRocMap = True
92  else:
93  print('No Online DQM file available. Skip dead roc map')
94  deadRocMap = False
95 
96  print('Downloading DQM file:'+File_Name_online)
97 
98 
99  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
100 
101  os.remove('index_online.html')
102  os.remove('index_online_backup.html')
103  return deadRocMap, File_Name_online;
104 
105 
106 def getGT(DQMfile, RunNumber, globalTagVar):
107  globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
108  print("Global Tag: " + globalTag_v0)
109  globalTag = globalTag_v0
110 
111  for z in range(len(globalTag_v0)-2):#clean up the garbage string in the GT
112  if (globalTag_v0[z].isdigit()) and (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
113  globalTag = globalTag_v0[z:]
114  if globalTag == "":
115  print(" No GlobalTag found: trying from DAS.... ");
116  globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
117  if globalTag == "":
118  print(" No GlobalTag found for run: "+str(Run_Number[i]));
119  return globalTag
120 
121 
122 Run_type = sys.argv[1]
123 Run_Number = [int(x) for x in sys.argv[2:]]
124 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
125 
126 ###########Check if user enter the right run type######################
127 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or 'ZeroBias' or re.match('ZeroBias([0-9]+?)',Run_type):
128  print(Run_type)
129 else:
130  print("please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ");
131  sys.exit(0)
132 
133 for i in range(len(Run_Number)):
134 #################Downloading DQM file############################
135  print("Downloading File!!")
136  nnnOut = Run_Number[i]/1000
137 
138  filepath = '/tmp/'
139  File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type)
140  deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
141 
142  runDirval=setRunDirectory(Run_Number[i])
143  DataLocalDir=runDirval[0]
144  #DataOfflineDir=runDirval[1]
145 
146 ################Check if run is complete##############
147 
148  print("get the run status from DQMFile")
149 
150 
151  check_command = 'check_runcomplete '+filepath+File_Name
152  Check_output = subprocess.call(check_command, shell=True)
153 
154 
155  if Check_output == 0:
156  print('Using DQM file: '+File_Name)
157  else:
158  print('*****************Warning: DQM file is not ready************************');
159  input_var = raw_input("DQM file is incompleted, do you want to continue? (y/n): ")
160  if (input_var == 'y') or (input_var == 'Y'):
161  print('Using DQM file: '+File_Name)
162  else:
163  sys.exit(0)
164 
165 
166 ###################Start making TkMaps################
167 
168  checkfolder = os.path.exists(str(Run_Number[i]))
169  if checkfolder == True:
170  shutil.rmtree(str(Run_Number[i]))
171  os.makedirs(str(Run_Number[i])+'/'+Run_type)
172  else:
173  os.makedirs(str(Run_Number[i])+'/'+Run_type)
174 
175 #######Getting GT##############
176  ####After switch production to 10_X_X release, the clean up section need to be reviewed and modified ##########
177  globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
178  ####################################################
179 
180 
181 
182  print(" Creating the TrackerMap.... ")
183 
184  detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
185  workPath = os.popen('pwd').readline().strip()
186 
187  os.chdir(str(Run_Number[i])+'/'+Run_type)
188 
189 
190  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
191  os.system('rm -f *svg')
192 ####################### rename bad module list file######################
193  sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
194  shutil.move('QTBadModules.log',sefile)
195 
196 ################### put color legend in the TrackerMap###################
197 
198 # PLEASE UNCOMMENT THE LINES BELOW TO GET THE LEGEND ON THE QT TkMAP (it will work only on vocms061)
199 # os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/LegendToQT.py QTestAlarm.png /data/users/cctrack/FinalLegendTrans.png')
200 # shutil.move('result.png', 'QTestAlarm.png')
201 
202 ####################Copying the template html file to index.html########################
203 
204  if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
205  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
206  else:
207  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
208 
209  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
210  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
211 
212  print(" Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder")
213 
214  output =[]
215  output.append(os.popen("/bin/ls ").readline().strip())
216  print(output)
217 
218 ## Producing the list of bad modules
219  print(" Creating the list of bad modules ")
220 
221  os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
222 
223  ## if Run_type != "StreamExpress":
224  ## shutil.copyfile(sefile, checkdir+'/'+sefile)
225  ## os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/findBadModT9.py -p '+sefile+' -s /'+checkdir+'/'+sefile);
226 
227 
228 ## Producing the run certification by lumisection
229 
230 # print " Creating the lumisection certification:"
231 
232 # if (Run_type.startswith("ZeroBias")) or (Run_type == "StreamExpress"):
233 # os.system('ls_cert 0.95 0.95 '+filepath+'/'+File_Name)
234 
235 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
236 # if (Run_type != "Cosmics") and ( Run_type != "StreamExpress") and (Run_type != "StreamExpressCosmics"):
237 # print " Creating the BeamSpot Calibration certification summary:"
238 # os.system('lsbs_cert '+filepath+'/'+File_Name)
239 
240 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
241  os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
242 
243 
244 ## Producing the Module difference for ExpressStream
245 
246  dest='Beam'
247 
248  if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
249  dest="Cosmics"
250 
251 
252 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors) ignore for now
253  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
254  shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
255 
256  os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
257 
258 
259  shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
260 
261  os.remove(detIdInfoFileName)
262  os.remove('MergedBadComponentsTkMap_Canvas.root')
263  os.remove('MergedBadComponentsTkMap.root')
264 ##############counting dead pixel#######################
265  print("countig dead pixel ROCs")
266  if (Run_Number[i] < 290124) :
267 
268  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
269  else:
270  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
271 
272  os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
273 
274  shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
275 
276 
277 
278 
279 ######Counting Dead ROCs and Inefficient DC during the run#########################
280 
281  if deadRocMap == True:
282 
283  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
284  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
285  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
286  os.remove('MaskedROC_sum.txt')
287  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
288  else:
289  print('No Online DQM file available, Dead ROC maps will not be produced')
290  print('No Online DQM file available, inefficient DC list will also not be produced')
291 
292 #######Merge Dead ROCs and Occupoancy Plot ###################
293 
294  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
295 
296 ###################copy ouput files###################
297  strip_files = os.listdir('.')
298  for file_name in strip_files:
299  full_stripfile_name = os.path.join('.', file_name)
300  if (os.path.isfile(full_stripfile_name)):
301  shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
302 
303 
304 
305 
306 #########################Start making pixel maps#####################
307  os.chdir(workPath)
308  shutil.rmtree(str(Run_Number[i]))
309  os.remove('index.html')
310 
311  # produce pixel phase1 TH2Poly maps
312 # os.chdir(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/scripts/PhaseIMaps/')
313  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
314  shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
315 
316 
317 ###################copy ouput files##########
318  pixel_files = os.listdir('./OUT')
319  for file_name in pixel_files:
320  full_pixelfile_name = os.path.join('./OUT/', file_name)
321  if (os.path.isfile(full_pixelfile_name)):
322  shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
323 
324 
325  shutil.rmtree('OUT')
326 
327 
328  # produce pixel phase1 tree for Offline TkCommissioner
329  pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
330  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
331 
332  shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
333  os.remove(pixelTreeFileName)
334  if File_Name:
335  os.remove(filepath+File_Name)
336  if File_Name_online:
337  os.remove(filepath+File_Name_online)
338  os.chdir(workPath)
any
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:37
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:65
TkMap_script_phase1.downloadnlineDQMhisto
def downloadnlineDQMhisto(run, Run_type)
Definition: TkMap_script_phase1.py:61
TkMap_script_phase1.setRunDirectory
def setRunDirectory(runNumber)
Definition: TkMap_script_phase1.py:16
TkMap_script_phase1.getGT
def getGT(DQMfile, RunNumber, globalTagVar)
Definition: TkMap_script_phase1.py:106
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
str
#define str(s)
Definition: TestProcessor.cc:47
split
double split
Definition: MVATrainer.cc:139
TkMap_script_phase1.downloadOfflineDQMhisto
def downloadOfflineDQMhisto(run, Run_type)
Definition: TkMap_script_phase1.py:36
Generated for CMSSW Reference Manual by doxygen 1.8.11