CMS 3D CMS Logo
TkMap_script_phase1.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 
3 import sys
4 from ROOT import *
5 import os
6 from subprocess import call
7 import os.path
8 import shutil
9 import subprocess
10 import codecs
11 import re
12 import errno
13 
14 
15 
16 Run_type = sys.argv[1]
17 Run_Number = [int(x) for x in sys.argv[2:]]
18 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
19 
20 ###########Check if user enter the right run type######################
21 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or 'ZeroBias' or re.match('ZeroBias([0-9]+?)',Run_type):
22 
23  print Run_type
24 else:
25  print "please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ";
26  sys.exit(0)
27 
28 #########Checking Data taking period##########
29 for i in range(len(Run_Number)):
30 
31  if Run_Number[i] > 308336:
32  DataLocalDir='Data2018'
33  DataOfflineDir='Commissioning2018'
34 
35  elif Run_Number[i] > 294644:
36  DataLocalDir='Data2017'
37  DataOfflineDir='Run2017'
38 
39  elif Run_Number[i] > 290123:
40  DataLocalDir='Data2017'
41  DataOfflineDir='Commissioning2017'
42 
43 
44  elif Run_Number[i] > 284500:
45  DataLocalDir='Data2016';
46  DataOfflineDir='PARun2016';
47 
48 ##2016 data taking period run > 271024
49  elif Run_Number[i] > 271024:
50  DataLocalDir='Data2016';
51  DataOfflineDir='Run2016';
52 
53 #2016 - Commissioning period
54  elif Run_Number[i] > 264200:
55  DataLocalDir='Data2016';
56  DataOfflineDir='Commissioning2016';
57 
58 #Run2015A
59  elif Run_Number[i] > 246907:
60  DataLocalDir='Data2015';
61  DataOfflineDir='Run2015'
62 
63 #2015 Commissioning period (since January)
64  elif Run_Number[i] > 232881:
65  DataLocalDir='Data2015';
66  DataOfflineDir='Commissioning2015';
67 
68 #2013 pp run (2.76 GeV)
69  elif Run_Number[i] > 211658:
70  DataLocalDir='Data2013';
71  DataOfflineDir='Run2013';
72 
73 #2013 HI run
74  elif Run_Number[i] > 209634:
75  DataLocalDir='Data2013';
76  DataOfflineDir='HIRun2013';
77 
78  elif Run_Number[i] > 190450:
79  DataLocalDir='Data2012';
80  DataOfflineDir='Run2012';
81 
82  else:
83  print "Please enter vaild run numbers"
84  sys.exit(0)
85 
86 #################Downloading DQM file############################
87  nnn=Run_Number[i]/100
88  nnnOut = Run_Number[i]/1000
89  nnnOnline = Run_Number[i]/10000
90 
91  print 'Processing '+Run_type+ ' in '+DataOfflineDir+"..."
92 
93  File_Name = ''
94 
95  print 'Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
96  url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
97  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
98  f=codecs.open("index.html", 'r')
99  index = f.readlines()
100  if any(str(Run_Number[i]) in s for s in index):
101  for s in index:
102  print s
103  if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
104  File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
105  else:
106  print 'No DQM file available. Please check the Offline server'
107  sys.exit(0)
108 
109  print 'Downloading DQM file:'+File_Name
110 
111 
112  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
113 
114  filepath = '/tmp/'
115 
116 
117 
118 ##################online file########
119  url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
120  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
121 
122  url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
123  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
124  f_online_backup=codecs.open("index_online_backup.html", 'r')
125  index_online_backup = f_online_backup.readlines()
126 
127  f_online=codecs.open("index_online.html", 'r')
128  index_online = f_online.readlines()
129  if any(str(Run_Number[i]) in x for x in index_online):
130  for x in index_online:
131  if (str(Run_Number[i]) in x) and ("_PixelPhase1_" in x):
132  File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
133  deadRocMap = True
134  else:
135  print "Can't find any file in offline server, trying the online server"
136  if any(str(Run_Number[i]) in y for y in index_online_backup):
137  for y in index_online:
138  if (str(Run_Number[i]) in y) and ("_PixelPhase1_" in y):
139  File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
140  deadRocMap = True
141  else:
142  print 'No Online DQM file available. Skip dead roc map'
143  deadRocMap = False
144 
145  if deadRocMap == True:
146  print 'Downloading DQM file:'+File_Name_online
147 
148 
149  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
150 
151  os.remove('index_online.html')
152  os.remove('index_online_backup.html')
153 
154 
155 
156 
157 ################Check if run is complete##############
158 
159  print "get the run status from DQMFile"
160 
161 
162  check_command = 'check_runcomplete '+filepath+File_Name
163  Check_output = subprocess.call(check_command, shell=True)
164 
165 
166  if Check_output == 0:
167  print 'Using DQM file: '+File_Name
168  else:
169  print '*****************Warning: DQM file is not ready************************';
170  input_var = raw_input("DQM file is incompleted, do you want to continue? (y/n): ")
171  if (input_var == 'y') or (input_var == 'Y'):
172  print 'Using DQM file: '+File_Name
173  else:
174  sys.exit(0)
175 
176 
177 ###################Start making TkMaps################
178 
179  checkfolder = os.path.exists(str(Run_Number[i]))
180  if checkfolder == True:
181  shutil.rmtree(str(Run_Number[i]))
182  os.makedirs(str(Run_Number[i])+'/'+Run_type)
183  else:
184  os.makedirs(str(Run_Number[i])+'/'+Run_type)
185 
186 #######Getting GT##############
187  ####After switch production to 10_X_X release, the clean up section need to be reviewed and modified ##########
188  globalTag_v0 = os.popen('getGTfromDQMFile.py '+ filepath+File_Name+' ' +str(Run_Number[i])+' globalTag_Step1').readline().strip()
189  print "Global Tag: " + globalTag_v0
190  globalTag = globalTag_v0
191 
192  for z in range(len(globalTag_v0)-2):#clean up the garbage string in the GT
193  if (globalTag_v0[z].isdigit()) and (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and (globalTag_v0[z+3].isupper()):
194  globalTag = globalTag_v0[z:]
195 
196  ####################################################
197 
198  if globalTag == "":
199  print " No GlobalTag found: trying from DAS.... ";
200  globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
201  if globalTag == "":
202  print " No GlobalTag found for run: "+str(Run_Number[i]);
203 
204  print " Creating the TrackerMap.... "
205 
206  detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
207  workPath = os.popen('pwd').readline().strip()
208 
209  os.chdir(str(Run_Number[i])+'/'+Run_type)
210 
211 
212  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
213  os.system('rm -f *svg')
214 ####################### rename bad module list file######################
215  sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
216  shutil.move('QTBadModules.log',sefile)
217 
218 ################### put color legend in the TrackerMap###################
219 
220 # PLEASE UNCOMMENT THE LINES BELOW TO GET THE LEGEND ON THE QT TkMAP (it will work only on vocms061)
221 # os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/LegendToQT.py QTestAlarm.png /data/users/cctrack/FinalLegendTrans.png')
222 # shutil.move('result.png', 'QTestAlarm.png')
223 
224 ####################Copying the template html file to index.html########################
225 
226  if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
227  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
228  else:
229  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
230 
231  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
232  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
233 
234  print " Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder"
235 
236  output =[]
237  output.append(os.popen("/bin/ls ").readline().strip())
238  print output
239 
240 ## Producing the list of bad modules
241  print " Creating the list of bad modules "
242 
243  os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
244 
245  ## if Run_type != "StreamExpress":
246  ## shutil.copyfile(sefile, checkdir+'/'+sefile)
247  ## os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/findBadModT9.py -p '+sefile+' -s /'+checkdir+'/'+sefile);
248 
249 
250 ## Producing the run certification by lumisection
251 
252 # print " Creating the lumisection certification:"
253 
254 # if (Run_type.startswith("ZeroBias")) or (Run_type == "StreamExpress"):
255 # os.system('ls_cert 0.95 0.95 '+filepath+'/'+File_Name)
256 
257 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
258 # if (Run_type != "Cosmics") and ( Run_type != "StreamExpress") and (Run_type != "StreamExpressCosmics"):
259 # print " Creating the BeamSpot Calibration certification summary:"
260 # os.system('lsbs_cert '+filepath+'/'+File_Name)
261 
262 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
263  os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
264 
265 
266 ## Producing the Module difference for ExpressStream
267 
268  dest='Beam'
269 
270  if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
271  dest="Cosmics"
272 
273 
274 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors) ignore for now
275  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
276  shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
277 
278  os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
279 
280 
281  shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
282 
283  os.remove(detIdInfoFileName)
284  os.remove('MergedBadComponentsTkMap_Canvas.root')
285  os.remove('MergedBadComponentsTkMap.root')
286 ##############counting dead pixel#######################
287  print "countig dead pixel ROCs"
288  if (Run_Number[i] < 290124) :
289 
290  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
291  else:
292  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
293 
294  os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
295 
296  shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
297 
298 
299 
300 
301 ######Counting Dead ROCs during the run#########################
302 
303  if deadRocMap == True:
304 
305  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
306  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
307  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
308  os.remove('MaskedROC_sum.txt')
309  else:
310  print 'No Online DQM file available, Dead ROC maps will not be produced'
311 
312 
313 ########Counting Inefficient DC during the run #################
314 
315 
316  if deadRocMap == True:
317 
318  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
319 
320  else:
321  print 'No Online DQM file available, inefficient DC list will not be produced'
322 
323 
324 
325 
326 
327 ###################copy ouput files###################
328  strip_files = os.listdir('.')
329  for file_name in strip_files:
330  full_stripfile_name = os.path.join('.', file_name)
331  if (os.path.isfile(full_stripfile_name)):
332  shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
333 
334 
335 
336 
337 #########################Start making pixel maps#####################
338  os.chdir(workPath)
339  shutil.rmtree(str(Run_Number[i]))
340  os.remove('index.html')
341 
342  # produce pixel phase1 TH2Poly maps
343 # os.chdir(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/scripts/PhaseIMaps/')
344  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
345  shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
346 
347 
348 ###################copy ouput files##########
349  pixel_files = os.listdir('./OUT')
350  for file_name in pixel_files:
351  full_pixelfile_name = os.path.join('./OUT/', file_name)
352  if (os.path.isfile(full_pixelfile_name)):
353  shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
354 
355 
356  shutil.rmtree('OUT')
357 
358 
359  # produce pixel phase1 tree for Offline TkCommissioner
360  pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
361  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
362 
363  shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
364  os.remove(pixelTreeFileName)
365  os.remove(filepath+File_Name)
366  os.remove(filepath+File_Name_online)
367  os.chdir(workPath)
368 
any
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:37
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.11