00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014 import sys
00015 import os
00016 import string
00017 import urllib
00018 import time
00019 import datetime
00020 import getpass
00021
00022
00023
00024
00025 TD_shiftUTC = datetime.timedelta(hours = 2)
00026 INT_offset = 8
00027
00028 STR_p5 = 'cmsusr0.cern.ch'
00029 STR_wwwWBM = 'https://cmswbm/cmsdb/servlet'
00030 STR_SiStrip = 'SIST'
00031 STR_wwwDBSData = 'dbs_discovery/getData'
00032 LSTR_dbsInstances = ['cms_dbs_caf_analysis_01',
00033 'cms_dbs_prod_global' ]
00034 STR_headDatasets = 'datasets'
00035 STR_headFiles = 'available data files'
00036 DICT_tagsRunRegistry = {}
00037 DICT_tagsRunRegistry['GLOBAL_NAME'] = 'global name '
00038 DICT_tagsRunRegistry['STATUS'] = 'status '
00039 DICT_tagsRunRegistry['IN_DBS'] = 'in DBS '
00040 DICT_tagsRunRegistry['SUBSYSTEMS'] = 'subsystems '
00041 DICT_tagsRunRegistry['EVENTS'] = '# of triggers '
00042 DICT_tagsRunRegistry['START_TIME'] = 'start time (local) '
00043 DICT_tagsRunRegistry['END_TIME'] = 'end time (local) '
00044 DICT_tagsRunRegistry['L1KEY'] = 'L1 key '
00045 DICT_tagsRunRegistry['HLTKEY'] = 'HLT key '
00046 DICT_tagsRunRegistry['L1SOURCES'] = 'L1 sources '
00047 DICT_tagsRunRegistry['RUN_RATE'] = 'event rate (Hz) '
00048 DICT_tagsRunRegistry['STOP_REASON'] = 'stop reason '
00049 DICT_tagsRunRegistry['SHIFTER'] = 'DQM shifter '
00050 DICT_tagsRunRegistry['CREATE_USER'] = 'entry created by '
00051 DICT_tagsRunRegistry['CREATE_TIME'] = 'entry creation time '
00052 DICT_tagsRunRegistry['ONLINE_COMMENT'] = 'DQM online shifter\'s comment '
00053 DICT_tagsRunRegistry['OFFLINE_COMMENT'] = 'DQM offline shifter\'s comment '
00054 DICT_tagsRunRegistry['OFFLINE_COMMENT'] = 'DQM offline shifter\'s comment '
00055 DICT_tagsRunRegistry['BFIELD'] = 'magnetic field at run creation time'
00056 DICT_tagsRunRegistry['BFIELD_COMMENT'] = 'comment on magnetic field '
00057 STR_htlConfig = 'HLT Config ID'
00058 STR_runStart = 'START_TIME'
00059 STR_runEnd = 'STOP_TIME'
00060 DICT_keysRunSummary = {}
00061 DICT_keysRunSummary[STR_runStart] = 'start time '
00062 DICT_keysRunSummary[STR_runEnd] = 'end time '
00063 DICT_keysRunSummary['BField'] = 'magnetic field '
00064 DICT_keysRunSummary['HLT Version'] = 'HLT version '
00065 DICT_keysRunSummary['L1 Rate'] = 'L1 rate '
00066 DICT_keysRunSummary['HLT Rate'] = 'HLT rate '
00067 DICT_keysRunSummary['L1 Triggers'] = 'L1 triggers '
00068 DICT_keysRunSummary['HLT Triggers'] = 'HLT triggers '
00069 DICT_keysRunSummary['LHC Fill'] = 'LHC fill '
00070 DICT_keysRunSummary['LHC Energy'] = 'LHC energy '
00071 DICT_keysRunSummary['Initial Lumi'] = 'initial luminosity '
00072 DICT_keysRunSummary['Ending Lumi'] = 'ending luminosity '
00073 DICT_keysRunSummary['Run Lumi'] = 'run luminosity '
00074 DICT_keysRunSummary['Run Live Lumi'] = 'run live luminosity'
00075 DICT_keysRunSummaryTrigger = {}
00076 DICT_keysRunSummaryTrigger['L1 Key'] = 'L1 key '
00077 DICT_keysRunSummaryTrigger['HLT Key'] = 'HLT key '
00078 DICT_keysRunSummaryTrigger[STR_htlConfig] = 'HLT config ID '
00079
00080
00081
00082 global Str_passwd
00083 global Str_userID
00084 global Str_run
00085 global Dict_runRegistry
00086 global Float_magneticField
00087 global Dict_wbmRunSummary
00088 global Lstr_hltPaths
00089 global DictDict_dbsDatasets
00090 global DictDict_dbsEvents
00091 global Dict_dbsDatasets
00092 global Dict_maxLenDbsDatasets
00093
00094 Str_run = sys.argv[1]
00095 Dict_runRegistry = {}
00096 Float_magneticField = -999.0
00097 Dict_wbmRunSummary = {}
00098 Lstr_hltPaths = []
00099 DictDict_dbsDatasets = {}
00100 DictDict_dbsEvents = {}
00101 Dict_dbsDatasets = {}
00102 Dict_maxLenDbsDatasets = {}
00103
00104
00105
00106
00107
00108
00109 def Func_GetHtmlTags(str_text):
00110 """ Func_GetHtmlTags(str_text):
00111 Gets HTML tags from a string
00112 """
00113 dict_tags = {}
00114
00115 lstr_split = str_text.split('</')
00116 for str_split in lstr_split[1:]:
00117 str_key = str_split.split('>')[0]
00118 dict_tags[str_key] = str_key in dict_tags
00119
00120 lstr_split = str_text.split('/>')
00121 for str_split in lstr_split[:-1]:
00122 str_key = str_split.split('<')[-1].split()[0]
00123 dict_tags[str_key] = str_key in dict_tags
00124 return dict_tags
00125
00126
00127
00128
00129 def Func_GetHtmlTagValue(str_tag, str_text, int_index = 1):
00130 """ Func_GetHtmlTagValue(str_tag, str_text):
00131 Gets the value of the n-th oocurence a given HTML tag from a string
00132 """
00133 if int_index > str_text.count('<'+str_tag):
00134 return ''
00135 str_1 = str_text.split('<'+str_tag)[int_index]
00136 if str_1[0] != '>':
00137 if str_1.split('>')[0][-1] == '/':
00138 return ''
00139 return str_1.split('>',1)[1].split('</'+str_tag+'>')[0]
00140
00141
00142
00143
00144 def Func_GetHtmlTagValues(str_text):
00145 """ Func_GetHtmlTagValues(str_text):
00146 Gets HTML tag values from a string
00147 """
00148 lstr_split = str_text.split('</')
00149 lstr_values = []
00150 for str_split in lstr_split[:-1]:
00151 lstr_values.append(str_split.split('>')[-1])
00152 return lstr_values
00153
00154
00155
00156
00157 def Func_GetHtmlTagValueAttr(str_value, str_text):
00158 """ Func_GetHtmlTagValueAttr(str_value, str_text):
00159 Gets the (last) attributes of a given HTML tag value from a string
00160 """
00161 return str_text.split('\">'+str_value+'<')[0].split('=\"')[-1]
00162
00163
00164
00165
00166 def Func_MakeShellWord(str_python):
00167 """ Func_MakeShellWord(str_python)
00168 Adds shell escape charakters to Python strings
00169 """
00170 return str_python.replace('?','\\?').replace('=','\\=').replace(' ','\\ ').replace('&','\\&').replace(':','\\:')
00171
00172
00173
00174
00175 def Func_GetWBMInfo(str_name, str_path):
00176 """ Func_GetWBMInfo(str_name, str_path)
00177 Logs in on cmsusr0, retrieves WBM information and stores it locally
00178 """
00179 pid, fd = os.forkpty()
00180 if pid == 0:
00181 os.execv('/usr/bin/ssh', ['/usr/bin/ssh', '-l', Str_userID, STR_p5] + ['rm', '-f', '\"'+str_name + '\" && ' + 'wget', '\"'+str_path+'/'+str_name+'\"'])
00182 else:
00183 time.sleep(1)
00184 os.read(fd, 1000)
00185 time.sleep(1)
00186 os.write(fd, Str_passwd)
00187 time.sleep(1)
00188 c = 0
00189 s = os.read(fd, 1)
00190 while s:
00191 c += 1
00192 s = os.read(fd, 1)
00193 if c >= 2:
00194 break
00195
00196
00197
00198
00199 def Func_CopyWBMInfo(str_name):
00200 """ Func_CopyWBMInfo(str_name)
00201 Logs in on cmsusr0 and copies file from there
00202 """
00203 pid, fd = os.forkpty()
00204 if pid == 0:
00205 os.execv('/usr/bin/scp', ['/usr/bin/scp', Str_userID+'@'+STR_p5+':~/'+str_name, '.'])
00206 else:
00207 time.sleep(1)
00208 os.read(fd, 1000)
00209 time.sleep(1)
00210 os.write(fd, Str_passwd)
00211 time.sleep(1)
00212 c = 0
00213 s = os.read(fd, 1)
00214 while s:
00215 c += 1
00216 s = os.read(fd, 1)
00217 if c >= 163:
00218 break
00219
00220
00221
00222
00223 def Func_FillInfoRunRegistry():
00224 """ Func_FillInfoRunRegistry():
00225 Retrieves run info from RunRegistry and fills it into containers
00226 """
00227 str_runRegistry = urllib.urlencode({'format':'xml', 'intpl':'xml', 'qtype':'RUN_NUMBER', 'sortname':'RUN_NUMBER'})
00228 file_runRegistry = urllib.urlopen("https://pccmsdqm04.cern.ch/runregistry/runregisterdata", str_runRegistry)
00229 str_runRegistryLong = ''
00230 for str_runRegistry in file_runRegistry.readlines():
00231 str_runRegistryLong += str_runRegistry.splitlines()[0]
00232 bool_foundRun = False
00233 str_runRunRegistry = ''
00234 for int_runIndex in range(1,int(str_runRegistryLong.split('<RUNS')[1].split('>')[0].split('total=\"')[1].split('\"')[0])):
00235 str_runRunRegistry = Func_GetHtmlTagValue('RUN', str_runRegistryLong, int_runIndex)
00236 if Func_GetHtmlTagValue('NUMBER', str_runRunRegistry) == Str_run:
00237 bool_foundRun = True
00238 break
00239 if not bool_foundRun:
00240 print '> getRunInfo.py > run %s not found in run registry' %(Str_run)
00241 return False
00242 dict_tagsRunRegistry = Func_GetHtmlTags(str_runRunRegistry)
00243 for str_tagRunRegistry in dict_tagsRunRegistry.keys():
00244 if dict_tagsRunRegistry[str_tagRunRegistry] == False:
00245 Dict_runRegistry[str_tagRunRegistry] = Func_GetHtmlTagValue(str_tagRunRegistry, str_runRunRegistry)
00246 if Dict_runRegistry['SUBSYSTEMS'].find(STR_SiStrip) < 0:
00247 print '> getRunInfo.py > SiStrip was not in this run'
00248 return False
00249 return True
00250
00251
00252
00253
00254 def Func_FillInfoRunSummary():
00255 """ Func_FillInfoRunSummary():
00256 Retrieves run info from RunSummary and fills it into containers
00257 """
00258 str_nameRunSummary = 'RunSummary?RUN=' + Str_run
00259 Func_GetWBMInfo(str_nameRunSummary, STR_wwwWBM)
00260 Func_CopyWBMInfo(Func_MakeShellWord(str_nameRunSummary))
00261 file_wbmRunSummary = file(str_nameRunSummary, 'r')
00262 bool_table = False
00263 int_tableHeader = 0
00264 int_tableItem = 0
00265 int_startItem = 0
00266 int_endItem = 0
00267 for str_wbmRunSummary in file_wbmRunSummary.readlines():
00268 if str_wbmRunSummary.find('<TABLE CLASS="params"><THEAD><TR>') >= 0:
00269 bool_table = True
00270 if str_wbmRunSummary.find('</TBODY></TABLE>') >= 0:
00271 bool_table = False
00272 if bool_table:
00273 if str_wbmRunSummary.startswith('<TH>'):
00274 int_tableHeader += 1
00275 if str_wbmRunSummary.find(STR_runStart) >= 0:
00276 int_startItem = int_tableHeader
00277 if str_wbmRunSummary.find(STR_runEnd) >= 0:
00278 int_endItem = int_tableHeader
00279 if str_wbmRunSummary.startswith('<TD'):
00280 int_tableItem += 1
00281 if int_tableItem == int_startItem:
00282 Dict_wbmRunSummary[STR_runStart] = str_wbmRunSummary.split(' </TD>')[0].split('<TD>')[-1]
00283 if int_tableItem == int_endItem:
00284 Dict_wbmRunSummary[STR_runEnd] = str_wbmRunSummary.split(' </TD>')[0].split('<TD>')[-1]
00285 continue
00286 for str_keyRunSummary in DICT_keysRunSummary.keys():
00287 if str_wbmRunSummary.find(str_keyRunSummary) >= 0:
00288 Dict_wbmRunSummary[str_keyRunSummary] = str_wbmRunSummary.split('</TD></TR>')[0].split('>')[-1]
00289 break
00290 for str_summaryKeysTrigger in DICT_keysRunSummaryTrigger.keys():
00291 if str_wbmRunSummary.find(str_summaryKeysTrigger) >= 0:
00292 Dict_wbmRunSummary[str_summaryKeysTrigger] = str_wbmRunSummary.split('</A></TD></TR>')[0].split('>')[-1]
00293 if str_summaryKeysTrigger == 'HLT Key':
00294 Dict_wbmRunSummary[STR_htlConfig] = str_wbmRunSummary.split('HLTConfiguration?KEY=')[1].split('>')[0]
00295 file_wbmRunSummary.close()
00296 os.remove(str_nameRunSummary)
00297
00298
00299
00300
00301 def Func_FillInfoMagnetHistory(str_timeStart, str_timeEnd):
00302 """ Func_FillInfoMagnetHistory():
00303 Retrieves run info from MagnetHistory and fills it into containers
00304 """
00305 str_nameMagnetHistory = 'MagnetHistory?TIME_BEGIN=' + str_timeStart + '&TIME_END=' + str_timeEnd
00306 Func_GetWBMInfo(str_nameMagnetHistory, STR_wwwWBM)
00307 Func_CopyWBMInfo(Func_MakeShellWord(str_nameMagnetHistory))
00308 file_wbmMagnetHistory = file(str_nameMagnetHistory, 'r')
00309 float_avMagMeasure = Float_magneticField
00310 for str_wbmMagnetHistory in file_wbmMagnetHistory.readlines():
00311 if str_wbmMagnetHistory.find('BFIELD, Tesla') >= 0:
00312 float_avMagMeasure = float(str_wbmMagnetHistory.split('</A>')[0].split('>')[-1])
00313 file_wbmMagnetHistory.close()
00314 os.remove(str_nameMagnetHistory)
00315 return float_avMagMeasure
00316
00317
00318
00319
00320 def Func_FillInfoHlt():
00321 """ Func_FillInfoHlt():
00322 Retrieves run info from Hlt and fills it into containers
00323 """
00324 str_nameHlt = 'HLTConfiguration?KEY=' + Dict_wbmRunSummary[STR_htlConfig]
00325 Func_GetWBMInfo(str_nameHlt, STR_wwwWBM)
00326 Func_CopyWBMInfo(Func_MakeShellWord(str_nameHlt))
00327 file_wbmHlt = file(str_nameHlt, 'r')
00328 bool_foundPaths = False
00329 bool_foundPath = False
00330 for str_wbmHlt in file_wbmHlt.readlines():
00331 if str_wbmHlt.find('<H3>Paths</H3>') >= 0:
00332 bool_foundPaths = True
00333 if bool_foundPaths and str_wbmHlt.find('<HR><H3>') >= 0:
00334 bool_foundPaths = False
00335 if bool_foundPaths and str_wbmHlt.startswith('<TR><TD ALIGN=RIGHT>'):
00336 Lstr_hltPaths.append(str_wbmHlt.split('</TD>')[1].split('<TD>')[-1])
00337 file_wbmHlt.close()
00338 os.remove(str_nameHlt)
00339 return (len(Lstr_hltPaths)>0)
00340
00341
00342
00343
00344 def Func_FillInfoDBS(str_dbsInstance):
00345 """ Func_FillInfoDBS(str_dbsInstance)
00346 Retrieves run info from DBS and fills it into containers
00347 """
00348 str_dbsRuns = urllib.urlencode({'ajax':'0', '_idx':'0', 'pagerStep':'0', 'userMode':'user', 'release':'Any', 'tier':'Any', 'dbsInst':str_dbsInstance, 'primType':'Any', 'primD':'Any', 'minRun':Str_run, 'maxRun':Str_run})
00349 file_dbsRuns = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getRunsFromRange", str_dbsRuns)
00350 lstr_dbsRuns = []
00351 lstr_dbsDatasets = []
00352 dict_dbsDatasets = {}
00353 dict_dbsEvents = {}
00354 for str_dbsRuns in file_dbsRuns.readlines():
00355 lstr_dbsRuns.append(str_dbsRuns)
00356 if str_dbsRuns.find(STR_wwwDBSData) >= 0:
00357 if str_dbsRuns.split('&proc=')[1].find('&') >= 0:
00358 lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1].split('&')[0])
00359 else:
00360 lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1])
00361 int_maxLenDbsDatasets = 0
00362 for str_dbsDataset in lstr_dbsDatasets:
00363 str_dbsLFN = urllib.urlencode({'dbsInst':str_dbsInstance, 'blockName':'*', 'dataset':str_dbsDataset, 'userMode':'user', 'run':Str_run})
00364 file_dbsLFN = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getLFNlist", str_dbsLFN)
00365 lstr_dbsLFN = []
00366 int_events = 0
00367 for str_dbsLFN in file_dbsLFN.readlines():
00368 lstr_dbsLFN.append(str_dbsLFN)
00369 if str_dbsLFN.find('contians') >= 0 and str_dbsLFN.find('file(s)'):
00370 dict_dbsDatasets[str_dbsDataset] = str_dbsLFN.split()[1]
00371 if str_dbsLFN.startswith('/store/data/'):
00372 int_events += int(Func_GetHtmlTagValue('td' ,lstr_dbsLFN[len(lstr_dbsLFN)-4]))
00373 dict_dbsEvents[str_dbsDataset] = str(int_events)
00374 if len(str_dbsDataset) > int_maxLenDbsDatasets:
00375 int_maxLenDbsDatasets = len(str_dbsDataset)
00376 DictDict_dbsDatasets[str_dbsInstance] = dict_dbsDatasets
00377 DictDict_dbsEvents[str_dbsInstance] = dict_dbsEvents
00378 Dict_dbsDatasets[str_dbsInstance] = lstr_dbsDatasets
00379 Dict_maxLenDbsDatasets[str_dbsInstance] = int_maxLenDbsDatasets
00380
00381
00382
00383 print
00384 print '> getRunInfo.py > information on run \t*** %s ***' %(Str_run)
00385 print
00386
00387
00388
00389 Str_userID = getpass.getuser()
00390 Str_passwd = getpass.getpass('> getRunInfo.py > '+Str_userID+'@'+STR_p5+'\'s password: ') + '\n'
00391
00392
00393
00394 bool_runRegistry = Func_FillInfoRunRegistry()
00395
00396
00397
00398 if bool_runRegistry:
00399 print
00400 print '> getRunInfo.py > * information from run registry *'
00401 print
00402 for str_htmlTag in DICT_tagsRunRegistry.keys():
00403 if str_htmlTag in Dict_runRegistry:
00404 print '> getRunInfo.py > %s: %s' %(DICT_tagsRunRegistry[str_htmlTag],Dict_runRegistry[str_htmlTag])
00405
00406
00407
00408 Func_FillInfoRunSummary()
00409
00410
00411
00412 print
00413 print '> getRunInfo.py > * information from run summary *'
00414 print
00415 for str_key in DICT_keysRunSummary.keys():
00416 if str_key in Dict_wbmRunSummary:
00417 print '> getRunInfo.py > %s: %s' %(DICT_keysRunSummary[str_key],Dict_wbmRunSummary[str_key])
00418 for str_key in DICT_keysRunSummaryTrigger.keys():
00419 if str_key in Dict_wbmRunSummary:
00420 print '> getRunInfo.py > %s: %s' %(DICT_keysRunSummaryTrigger[str_key],Dict_wbmRunSummary[str_key])
00421
00422
00423
00424 if Dict_wbmRunSummary.has_key(STR_runStart) and Dict_wbmRunSummary.has_key(STR_runEnd):
00425 Float_magneticField = Func_FillInfoMagnetHistory(Dict_wbmRunSummary[STR_runStart],Dict_wbmRunSummary[STR_runEnd])
00426
00427
00428
00429 if Float_magneticField >= 0.0:
00430 print
00431 print '> getRunInfo.py > * information from magnet history *'
00432 print
00433 print '> getRunInfo.py > (average) magnetic field: %s T' %(str(Float_magneticField))
00434
00435
00436
00437 bool_hlt = False
00438 if Dict_wbmRunSummary.has_key(STR_htlConfig):
00439 bool_hlt = Func_FillInfoHlt()
00440
00441
00442
00443 if bool_hlt:
00444 print
00445 print '> getRunInfo.py > * information from HLT configuration %s *' %(Dict_wbmRunSummary[STR_htlConfig])
00446 print
00447 print '> getRunInfo.py > HLT paths included:'
00448 print '> -----------------------------------'
00449 for str_hltPaths in Lstr_hltPaths:
00450 if str_hltPaths.find('CandHLTTrackerCosmics') >= 0 or str_hltPaths.find('HLT_TrackerCosmics') >= 0 or str_hltPaths.find('HLTTrackerCosmics') >= 0:
00451 print ' %s \t<====== FOR SURE!' %(str_hltPaths)
00452 elif str_hltPaths.find('Tracker') >= 0:
00453 print ' %s \t<====== maybe?' %(str_hltPaths)
00454 else:
00455 print ' %s' %(str_hltPaths)
00456
00457
00458
00459 for str_dbsInstance in LSTR_dbsInstances:
00460 Func_FillInfoDBS(str_dbsInstance)
00461
00462
00463
00464 print
00465 print '> getRunInfo.py > * information from DBS *'
00466 for str_dbsInstance in LSTR_dbsInstances:
00467 print
00468 print '> getRunInfo.py > DBS instance: %s' %(str_dbsInstance)
00469 if str_dbsInstance == LSTR_dbsInstances[0]:
00470 print ' (This is the instance used at CAF!)'
00471 str_print = '> getRunInfo.py > ' + STR_headDatasets
00472 for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(STR_headDatasets)):
00473 str_print += ' '
00474 str_print += ' '
00475 int_length = len(str_print)
00476 print '%s%s' %(str_print,STR_headFiles)
00477 str_print = ' '
00478 for int_i in range(int_length-16+len(STR_headFiles)/2+INT_offset+8):
00479 str_print += '-'
00480 print str_print
00481 for str_dbsDataset in Dict_dbsDatasets[str_dbsInstance]:
00482 str_print = ' ' + str_dbsDataset
00483 for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(str_dbsDataset)):
00484 str_print += ' '
00485 str_print += ' '
00486 for int_i in range(len(STR_headFiles)/2-len(DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset])):
00487 str_print += ' '
00488 str_print += DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset] + ' ('
00489 for int_i in range(INT_offset-len(DictDict_dbsEvents[str_dbsInstance][str_dbsDataset])):
00490 str_print += ' '
00491 print '%s%s events)' %(str_print,DictDict_dbsEvents[str_dbsInstance][str_dbsDataset])
00492
00493 print
1.7.3