00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014 import sys
00015 import os
00016 import os.path
00017 import commands
00018 import shutil
00019 import string
00020 import math
00021 import urllib
00022 import time
00023 import datetime
00024 import smtplib
00025
00026
00027
00028
00029 OCT_rwx_r_r = 0744
00030 LFLOAT_valueMagField = [0.0,2.0,3.0,3.5,3.8,4.0]
00031 TD_shiftUTC = datetime.timedelta(hours = 2)
00032
00033 LSTR_true = ['1','TRUE' ,'True' ,'true' ]
00034 LSTR_false = ['0','FALSE','False','false']
00035 STR_default = 'DEFAULT'
00036 STR_none = 'None'
00037 LSTR_auto = ['AUTO','Auto','auto']
00038 STR_nameCmsswPackage = 'DQM/SiStripMonitorClient'
00039 STR_textUsage = """ CMSSW/DQM/SiStripMonitorClient/scripts/submitDQMOfflineCAF.py
00040
00041 This script submits batch jobs to the CAF in order to process the full
00042 granularity SiStrip offline DQM.
00043 Questions and comments to: volker.adler@cern.ch
00044
00045 Func_Usage(): submitDQMOfflineCAF.py (-s, --submit | -c, --create |
00046 -h, --help)
00047 [-r, --run]
00048 [-C, --CRAB]
00049 [-S, --server]
00050 [-e, --email]
00051 [-j, --jobs]
00052 [-f, --filter]
00053 [-d, --dataset]
00054 [-o, --outpath]
00055 [-m, --mergepath]
00056
00057 Function letters: One of the following options m u s t be used.
00058
00059 -s, --submit
00060 create jobs and submit them to CAF;
00061 requires option '-r'
00062
00063 -c, --create
00064 create jobs, but do not submit them;
00065 requires option '-r'
00066
00067 -h, --help
00068 print this message
00069
00070 Other options:
00071
00072 -r, --run RUNNUMBER
00073 number of run to process;
00074 required by funtion letters '-s' and '-c'
00075
00076 -d, --dataset PRIMARY_DATASET
00077 specify dataset for DBS query;
00078 required by funtion letters '-s' and '-c'
00079
00080 -C, --CRAB TRUE/FALSE
00081 submit or submit not using CRAB;
00082 default: TRUE
00083
00084 NOTE: This script runs only with CRAB 2.4.0 or higher.
00085
00086 -S, --server CRAB_SERVER
00087 CRAB server to use;
00088 available: None (default)
00089 caf (works, but slow)
00090 bari (CRAB version >= 2.4.1,
00091 s. https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#Server_available_for_users)
00092
00093 NOTE: CRAB server submission is disabled at the moment.
00094
00095 -e, --email EMAIL_ADDRESS
00096 where the CRAB server should send its messages;
00097 default: volker.adler@cern.ch
00098
00099 -j, --jobs NUMBER
00100 number of jobs to create;
00101 default: 10
00102
00103 -g, --global-tag GLOBAL_TAG
00104 global tag to be used;
00105 default: CRAFT_V4P::All
00106
00107 -M, --magnetic-field FIELD
00108 specification of field to be used;
00109 can be:
00110 - average field during run in Tesla, given as float (e.g.: 3.8),
00111 - specification referring to existing configuration files (e.g.: 38T)
00112 (s. Configuration/StandardSequences/python/MagneticField_[FIELD]_cff.py),
00113 - usage of automatic average field determination from CMS WBM by using "auto"
00114 default: 38T
00115
00116 NOTE: "auto" is disabled for the moment due to unavailablity of CMS WBM outside '.cms' network.
00117
00118 -f, --filter TRUE/FALSE
00119 use or use not HLT filters to select events to process;
00120 default: FALSE
00121
00122 -o, --outpath PATH
00123 path to copy job output *.root files to;
00124 currently (almost) no check performed;
00125 must be in AFS or CASTOR
00126 default: /castor/cern.ch/user/c/cctrack/DQM
00127
00128 -m, --mergepath PATH
00129 path to merge the job output *.root files;
00130 currently (almost) no check performed;
00131 must be in AFS or on local computer (e.g. /tmp/[user])
00132 default: /afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged
00133 """
00134 LSTR_datatiers = ['RECO','RAW']
00135
00136 LSTR_wordArgument = sys.argv[1:]
00137
00138 BOOL_CRAB = True
00139 LSTR_server = [STR_none,'caf','bari']
00140 STR_server = LSTR_server[0]
00141 STR_email = 'volker.adler@cern.ch'
00142 INT_jobs = 10
00143 STR_globalTag = 'CRAFT_V4P::All'
00144 STR_magField = '38T'
00145 BOOL_magFieldAuto = False
00146 BOOL_filter = False
00147 STR_outpath = '/castor/cern.ch/user/c/cctrack/DQM'
00148 BOOL_useCastor = True
00149 STR_mergepath = '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged'
00150
00151 LSTR_functionLetters = ['-s','-c','-h']
00152 DICT_functionLetters = {'--submit':LSTR_functionLetters[0],
00153 '--create':LSTR_functionLetters[1],
00154 '--help' :LSTR_functionLetters[2]}
00155 LSTR_optionLetters = ['-r','-C','-S','-e','-j','-M','-g','-f','-d','-o','-m']
00156 DICT_optionLetters = {'--run' :LSTR_optionLetters[0],
00157 '--CRAB' :LSTR_optionLetters[1],
00158 '--server' :LSTR_optionLetters[2],
00159 '--email' :LSTR_optionLetters[3],
00160 '--jobs' :LSTR_optionLetters[4],
00161 '--magnetic-field':LSTR_optionLetters[5],
00162 '--global-tag' :LSTR_optionLetters[6],
00163 '--filter' :LSTR_optionLetters[7],
00164 '--dataset' :LSTR_optionLetters[8],
00165 '--outpath' :LSTR_optionLetters[9],
00166 '--mergepath' :LSTR_optionLetters[10]}
00167 STR_mailSmtp = 'localhost'
00168 STR_mailServer = '@mail.cern.ch'
00169 STR_mailTextOpener = """Dear """ + os.getenv('USER').capitalize() + """,
00170
00171 on """ + str(time.ctime()) + """, you have submitted run """
00172 STR_mailText = """
00173 for SiStrip offline DQM at the CAF.
00174 Unfortunately, this needed to be done from your private account. So, only you
00175 are able to finalize this submission -- even after the end of your shift.
00176 To do so, please forward all emails from the LSF batch system referring to the
00177 respective jobs to the list t h i s message was sent to.
00178 -- and then your shift is r e a l l y done :-)
00179
00180 We are very sorry for the inconvenience.
00181 Thanks a lot!
00182
00183 Best regards,
00184 your SiStrip DQM team
00185
00186 P.S.:
00187 To reply to this email, simply use the "Reply to all" function of your email
00188 client.
00189 """
00190
00191
00192
00193
00194 global Dict_arguments
00195 global Str_run
00196 global Bool_CRAB
00197 global Str_server
00198 global Str_email
00199 global Int_jobs
00200 global Str_globalTag
00201 global Bool_filter
00202 global Str_dataset
00203 global Str_datatier
00204 global Str_magField
00205 global Float_magField
00206 global Bool_magFieldAuto
00207 global Str_outpath
00208 global Bool_useCastor
00209 global Str_mergepath
00210
00211 global Str_pathCurrentDir
00212 global Str_pathCmsswBase
00213 global Str_nameCmsswRel
00214 global Str_pathCmsswBasePackage
00215 global Str_nameRun
00216 global Str_pathRunIncludeDir
00217 global Str_pathInputFilesCAFCff
00218 global Int_jobsNew
00219 global Str_magField
00220
00221 Dict_arguments = {}
00222 Bool_CRAB = BOOL_CRAB
00223 Str_server = STR_server
00224 Str_email = STR_email
00225 Int_jobs = INT_jobs
00226 Str_globalTag = STR_globalTag
00227 Bool_filter = BOOL_filter
00228 Str_magField = STR_magField
00229 Float_magField = float(Str_magField[:-1])/10.
00230 Bool_magFieldAuto = BOOL_magFieldAuto
00231 Str_outpath = STR_outpath
00232 Bool_useCastor = BOOL_useCastor
00233 Str_mergepath = STR_mergepath
00234
00235
00236
00237
00238 def Func_Usage():
00239 """ Function Func_Usage():
00240 Displays usage of the script
00241 """
00242 print STR_textUsage
00243
00244
00245
00246
00247 def Func_Exit():
00248 """ Function Func_Exit():
00249 Exit after error
00250 """
00251 print ' exit'
00252 print
00253 sys.exit(1)
00254
00255
00256
00257
00258 def Func_ExitUsage():
00259 """ Function Func_ExitUsage():
00260 Exit after wrong invocation of script
00261 """
00262 print ' exit'
00263 print
00264 Func_Usage()
00265 sys.exit(1)
00266
00267
00268
00269
00270 def Func_ExitBool(int_index):
00271 """ Function Func_ExitBool():
00272 Exit after wrong assignment of bool option
00273 """
00274 print '> submitDQMOfflineCAF.py > option %s expects 0/1, FALSE/TRUE, False/True or false/true' %(DICT_optionLetters.items()[int_index])
00275 Func_Exit()
00276
00277
00278
00279
00280 def Func_MkDir(str_path):
00281 """ Function Func_MkDir():
00282 Create new directory
00283 """
00284 shutil.rmtree(str_path, True)
00285 os.mkdir(str_path)
00286
00287
00288
00289
00290 def Func_MagConfig(float_magFieldMeasured):
00291 """ Func_MagConfig(float_magFieldMeasured):
00292 Determine configuration to be used for a given magnetic field
00293 """
00294 float_magField = 0.0
00295 for float_valueMagField in LFLOAT_valueMagField:
00296 if math.fabs(float_valueMagField-float_magFieldMeasured) < math.fabs(float_magField-float_magFieldMeasured):
00297 float_magField = float_valueMagField
00298 return float_magField
00299
00300
00301
00302 print
00303
00304
00305
00306 Str_pathCurrentDir = os.getcwd()
00307 Str_pathCmsswBase = os.getenv('CMSSW_BASE')
00308 if not Str_pathCmsswBase:
00309 print '> submitDQMOfflineCAF.py > CMSSW environment not set properly;'
00310 print ' first do'
00311 print
00312 print ' $ cd [your/CMSSW/release/area]/src'
00313 print ' $ cmsenv'
00314 print
00315 Func_Exit()
00316 Str_nameCmsswRel = os.getenv('CMSSW_VERSION')
00317 Str_pathCmsswBasePackage = Str_pathCmsswBase + '/src/' + STR_nameCmsswPackage
00318 str_suffixShell = 'csh'
00319 if not os.getenv('SHELL')[-3:] == str_suffixShell:
00320 str_suffixShell = 'sh'
00321
00322
00323
00324 if len(LSTR_wordArgument) == 0:
00325 Func_ExitUsage()
00326 int_nFunctionLetters = 0
00327 for str_argument in LSTR_wordArgument:
00328 if str_argument in LSTR_functionLetters or\
00329 DICT_functionLetters.has_key(str_argument) :
00330 int_nFunctionLetters += 1
00331 if int_nFunctionLetters == 0:
00332 print '> submitDQMOfflineCAF.py > no or unknown function letter used'
00333 Func_ExitUsage()
00334 elif int_nFunctionLetters > 1:
00335 print '> submitDQMOfflineCAF.py > too many function letter used'
00336 Func_ExitUsage()
00337
00338
00339
00340 str_argumentFormer = ''
00341 bool_standBy = False
00342 for str_argument in LSTR_wordArgument:
00343 if not ( str_argument in LSTR_functionLetters or DICT_functionLetters.has_key(str_argument) or\
00344 str_argument in LSTR_optionLetters or DICT_optionLetters.has_key(str_argument) ):
00345 if str_argument[0] == '-':
00346 print '> submitDQMOfflineCAF.py > unknown option used'
00347 Func_ExitUsage()
00348 if not bool_standBy:
00349 print '> submitDQMOfflineCAF.py > value without option used'
00350 Func_ExitUsage()
00351 Dict_arguments[str_argumentFormer] = str_argument
00352 bool_standBy = False
00353 else:
00354 if bool_standBy:
00355 Dict_arguments[str_argumentFormer] = STR_default
00356 if str_argumentFormer in LSTR_optionLetters or\
00357 DICT_optionLetters.has_key(str_argumentFormer):
00358 print '> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer)
00359 print ' default used'
00360 print
00361 bool_standBy = not ( str_argument in LSTR_functionLetters or\
00362 DICT_functionLetters.has_key(str_argument) )
00363 if not bool_standBy:
00364 Dict_arguments[str_argument] = STR_default
00365 str_argumentFormer = str_argument
00366 if bool_standBy:
00367 Dict_arguments[str_argumentFormer] = STR_default
00368 if str_argumentFormer in LSTR_optionLetters or\
00369 DICT_optionLetters.has_key(str_argumentFormer) :
00370 print '> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer)
00371 print ' default used'
00372 print
00373
00374
00375
00376 dict_arguments = Dict_arguments
00377 for str_key, str_value in dict_arguments.items():
00378 if str_key in DICT_functionLetters.keys():
00379 del Dict_arguments[str_key]
00380 Dict_arguments[DICT_functionLetters[str_key]] = str_value
00381 if str_key in DICT_optionLetters.keys():
00382 del Dict_arguments[str_key]
00383 Dict_arguments[DICT_optionLetters[str_key]] = str_value
00384
00385
00386
00387 if Dict_arguments.has_key(LSTR_functionLetters[2]):
00388 Func_Usage()
00389 sys.exit(0)
00390
00391
00392
00393
00394 if Dict_arguments.has_key(LSTR_optionLetters[0]) and\
00395 Dict_arguments[LSTR_optionLetters[0]] != STR_default :
00396 Str_run = Dict_arguments[LSTR_optionLetters[0]]
00397 else:
00398 print '> submitDQMOfflineCAF.py > no run number given'
00399 Func_Exit()
00400
00401 if Dict_arguments.has_key(LSTR_optionLetters[1]) and\
00402 Dict_arguments[LSTR_optionLetters[1]] != STR_default :
00403 if Dict_arguments[LSTR_optionLetters[1]] in LSTR_true:
00404 Bool_CRAB = True
00405 elif Dict_arguments[LSTR_optionLetters[1]] in LSTR_false:
00406 Bool_CRAB = False
00407 else:
00408 Func_ExitBool(1)
00409
00410 if Dict_arguments.has_key(LSTR_optionLetters[2]) and\
00411 Dict_arguments[LSTR_optionLetters[2]] != STR_default :
00412 Str_server = Dict_arguments[LSTR_optionLetters[2]]
00413
00414 if Dict_arguments.has_key(LSTR_optionLetters[3]) and\
00415 Dict_arguments[LSTR_optionLetters[3]] != STR_default :
00416 Str_email = Dict_arguments[LSTR_optionLetters[3]]
00417
00418 if Dict_arguments.has_key(LSTR_optionLetters[4]) and\
00419 Dict_arguments[LSTR_optionLetters[4]] != STR_default :
00420 Int_jobs = int(Dict_arguments[LSTR_optionLetters[4]])
00421
00422 if Dict_arguments.has_key(LSTR_optionLetters[5]) and\
00423 Dict_arguments[LSTR_optionLetters[5]] != STR_default :
00424 Str_magField = Dict_arguments[LSTR_optionLetters[5]]
00425 if Str_magField in LSTR_auto:
00426
00427 print '> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment'
00428 Func_Exit()
00429
00430 if Dict_arguments.has_key(LSTR_optionLetters[6]) and\
00431 Dict_arguments[LSTR_optionLetters[6]] != STR_default :
00432 Str_globalTag = Dict_arguments[LSTR_optionLetters[6]]
00433
00434 if Dict_arguments.has_key(LSTR_optionLetters[7]) and\
00435 Dict_arguments[LSTR_optionLetters[7]] != STR_default :
00436 if Dict_arguments[LSTR_optionLetters[7]] in LSTR_true:
00437 Bool_filter = True
00438 elif Dict_arguments[LSTR_optionLetters[7]] in LSTR_false:
00439 Bool_filter = False
00440 else:
00441 Func_ExitBool(7)
00442
00443 if Dict_arguments.has_key(LSTR_optionLetters[8]) and\
00444 Dict_arguments[LSTR_optionLetters[8]] != STR_default :
00445 Str_dataset = Dict_arguments[LSTR_optionLetters[8]]
00446 else:
00447 print '> submitDQMOfflineCAF.py > no primary dataset given'
00448 Func_Exit()
00449
00450 if Dict_arguments.has_key(LSTR_optionLetters[9]) and\
00451 Dict_arguments[LSTR_optionLetters[9]] != STR_default :
00452 Str_outpath = Dict_arguments[LSTR_optionLetters[9]]
00453
00454 if Dict_arguments.has_key(LSTR_optionLetters[10]) and\
00455 Dict_arguments[LSTR_optionLetters[10]] != STR_default :
00456 Str_mergepath = Dict_arguments[LSTR_optionLetters[10]]
00457
00458
00459
00460
00461 if Bool_CRAB:
00462 str_buffer = commands.getoutput('which crab')
00463 if str_buffer.find('which: no crab in') >= 0:
00464 str_suffixShell = 'csh'
00465 if not os.getenv('SHELL')[-3:] == str_suffixShell:
00466 str_suffixShell = 'sh'
00467 print '> submitDQMOfflineCAF.py > CRAB environment not set properly;'
00468 print ' please use'
00469 print
00470 print ' $ source /afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.%s' %(str_suffixShell)
00471 print
00472 Func_Exit()
00473
00474 if not Str_server in LSTR_server:
00475 print '> submitDQMOfflineCAF.py > CRAB server "%s" not available' %(Str_server)
00476 Func_Exit()
00477
00478 if Int_jobs == 0:
00479 Int_jobs = 1
00480 print '> submitDQMOfflineCAF.py > number of requested jobs was 0'
00481 print ' set to 1'
00482
00483 if Str_magField in LSTR_auto:
00484
00485 print '> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment'
00486 Func_Exit()
00487 elif Str_magField[-1] == 'T':
00488 bool_foundField = False
00489 for float_valueMagField in LFLOAT_valueMagField:
00490 if str(int(float_valueMagField*10)) == Str_magField[:-1]:
00491 Float_magField = float_valueMagField
00492 bool_foundField = True
00493 break
00494 if not bool_foundField:
00495 print '> submitDQMOfflineCAF.py > no magnet configuration for \'%s\' available' %(Str_magField)
00496 Func_Exit()
00497 else:
00498 Float_magField = float(Str_magField)
00499
00500
00501 str_datatier = Str_dataset.split('/')[-1]
00502 if str_datatier == 'RAW-RECO':
00503 Str_datatier = LSTR_datatiers[1]
00504 else:
00505 Str_datatier = str_datatier
00506 Str_datatier = Str_dataset.split('/')[-1]
00507 if not Str_datatier in LSTR_datatiers:
00508 print '> submitDQMOfflineCAF.py > datatier "%s" not processable' %(Str_datatier)
00509 Func_Exit()
00510
00511
00512 if Str_outpath.split('/')[1] == 'afs':
00513 Bool_useCastor = False
00514 elif Str_outpath.split('/')[1] != 'castor':
00515 print '> submitDQMOfflineCAF.py > output path not accepted'
00516 Func_ExitUsage()
00517 str_castorCp = 'cp'
00518 if Bool_useCastor:
00519 str_castorCp = 'rfcp'
00520
00521 if Str_mergepath.split('/')[1] != 'afs':
00522 print '> submitDQMOfflineCAF.py > merge path not accepted'
00523 Func_ExitUsage()
00524
00525
00526
00527
00528 Str_nameRun = 'R' + Str_run.zfill(9)
00529
00530 Func_MkDir(Str_nameRun)
00531 Str_pathRunIncludeDir = Str_pathCmsswBasePackage + '/python/' + Str_nameRun
00532 Func_MkDir(Str_pathRunIncludeDir)
00533 str_nameInputFilesFile = Str_nameRun + '/' + Str_nameRun + '.txt'
00534 str_nameRunIncludeDir = STR_nameCmsswPackage.replace('/','.') + '.' + Str_nameRun
00535
00536
00537
00538
00539 int_nInputFiles = 0
00540 file_inputFilesCff = file(str_nameInputFilesFile, 'w')
00541
00542 str_dbsParams = urllib.urlencode({'dbsInst':'cms_dbs_prod_global', 'blockName':'*', 'dataset':Str_dataset, 'userMode':'user', 'run':Str_run, 'what':'py'})
00543 file_dbsOutput = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getLFN_txt", str_dbsParams)
00544 for str_iLine in file_dbsOutput.readlines():
00545 lstr_wordsLine = str_iLine.split("/")
00546 if len(lstr_wordsLine) >= 5:
00547 if lstr_wordsLine[1] == 'store' and\
00548 lstr_wordsLine[2] == 'data' and\
00549 Str_dataset.find(lstr_wordsLine[3]) >= 0 and\
00550 Str_dataset.find(lstr_wordsLine[4]) >= 0 and\
00551 lstr_wordsLine[5] == str_datatier :
00552 int_nInputFiles += 1
00553 file_inputFilesCff.write(str_iLine)
00554 if int_nInputFiles == 0:
00555 print '> submitDQMOfflineCAF.py > no input files found in DBS for run %s in dataset %s' %(Str_run,Str_dataset)
00556 Func_Exit()
00557 file_inputFilesCff.close()
00558 print '> submitDQMOfflineCAF.py > input files for run %s: %i' %(Str_run,int_nInputFiles)
00559 if int_nInputFiles < Int_jobs:
00560 Int_jobs = int_nInputFiles
00561 print ' number of requested jobs reduced accordingly'
00562 Int_jobsNew = Int_jobs
00563 print
00564
00565
00566 if Bool_magFieldAuto:
00567
00568 str_cmsmonParams = urllib.urlencode({'RUN':Str_run})
00569 file_cmsmonOutput = urllib.urlopen("https://cmsmon.cern.ch/cmsdb/servlet/RunSummary", str_cmsmonParams)
00570 str_timeBegin = ''
00571 str_timeEnd = ''
00572 for str_cmsmonOutput in file_cmsmonOutput.readlines():
00573 if str_cmsmonOutput.find('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER') >= 0:
00574 lstr_timeQuery = str_cmsmonOutput.split('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER&')[1].split('>TRACKER')[0].split('&')
00575 for str_timeQuery in lstr_timeQuery:
00576 str_nameStamp = str_timeQuery.split('=')[0]
00577 lstr_timeDate = str_timeQuery.split('=')[1].split('_')[0].split('.')
00578 lstr_timeTime = str_timeQuery.split('=')[1].split('_')[1].split(':')
00579 dt_stampOld = datetime.datetime(int(lstr_timeDate[0]),int(lstr_timeDate[1]),int(lstr_timeDate[2]),int(lstr_timeTime[0]),int(lstr_timeTime[1]),int(lstr_timeTime[2]))
00580 dt_stampNew = dt_stampOld - TD_shiftUTC
00581 str_timeStamp = str(dt_stampNew).replace('-','.')
00582 if str_nameStamp == 'TIME_BEGIN':
00583 str_timeBegin = str_timeStamp
00584 elif str_nameStamp == 'TIME_END':
00585 str_timeEnd = str_timeStamp
00586
00587 str_cmsmonParams = urllib.urlencode({'TIME_BEGIN':str_timeBegin, 'TIME_END':str_timeEnd})
00588 file_cmsmonOutput = urllib.urlopen("https://cmsmon.cern.ch/cmsdb/servlet/MagnetHistory", str_cmsmonParams)
00589 bool_foundField = False
00590 for str_cmsmonOutput in file_cmsmonOutput.readlines():
00591 if str_cmsmonOutput.find('BFIELD, Tesla') >= 0:
00592 Float_magField = float(str_cmsmonOutput.split('</A>')[0].split('>')[-1])
00593 bool_foundField = True
00594 break
00595 if not bool_foundField:
00596 print '> submitDQMOfflineCAF.py > could not extract magnetic field'
00597 print ' please provide value'
00598 Func_Exit()
00599
00600 float_magField = Func_MagConfig(Float_magField)
00601 Str_magField = str(int(float_magField*10)) + 'T'
00602 print '> submitDQMOfflineCAF.py > (average) magnetic field in run %s: %s T' %(Str_run,Float_magField)
00603 print ' using %s T for configuration' %(float_magField)
00604 print
00605
00606
00607
00608 int_nLinesRead = 0
00609 file_inputFilesCff = file(str_nameInputFilesFile)
00610 lstr_linesInput = file_inputFilesCff.readlines()
00611 file_inputFilesCff.close()
00612
00613
00614 str_sedCommand = 'sed '
00615 str_sedCommand += '-e \"s#xMAG_FIELDx#' + Str_magField + '#g\" '
00616 str_sedCommand += '-e \"s#xGLOBAL_TAGx#' + Str_globalTag + '#g\" '
00617 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
00618 str_sedCommand += '-e \"s#xMERGE_PATHx#' + Str_mergepath + '#g\" '
00619 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripCAFHarvest_template_cfg.py > ' + Str_nameRun + '/SiStripCAFHarvest_cfg.py'
00620 os.system(str_sedCommand)
00621 str_sedCommand = 'sed '
00622 str_sedCommand += '-e \"s#xCMSSW_BASEx#' + Str_pathCmsswBase + '#g\" '
00623 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
00624 str_sedCommand += '-e \"s#xMERGE_PATHx#' + Str_mergepath + '#g\" '
00625 str_sedCommand += '-e \"s#xCURRENT_DIRx#' + Str_pathCurrentDir + '#g\" '
00626 str_sedCommand += '-e \"s#xDATA_TIERx#' + Str_datatier + '#g\" '
00627 str_sedCommand += '-e \"s#xCMSSW_VERSIONx#' + Str_nameCmsswRel + '#g\" '
00628 str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMCAFHarvest_template.job > ' + Str_nameRun + '/SiStripCAFHarvest.job'
00629 os.system(str_sedCommand)
00630
00631 Str_pathInputFilesCAFCff = Str_pathRunIncludeDir + '/inputFilesCAF_cff.py'
00632 file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
00633 file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n processingMode = cms.untracked.string( \'Runs\' ),\n fileNames = cms.untracked.vstring(\n')
00634 for int_iJob in range(1,Int_jobs+1):
00635 str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
00636 if Bool_useCastor:
00637 str_lineInput = 'rfio:' + str_lineInput
00638 str_lineInput = ' \'' + str_lineInput + '\''
00639 if int_iJob == Int_jobs:
00640 str_lineInput += '\n'
00641 file_inputFilesCAFCff.write(str_lineInput)
00642 break
00643 str_lineInput += ',\n'
00644 file_inputFilesCAFCff.write(str_lineInput)
00645 file_inputFilesCAFCff.write(' )\n)\n')
00646 file_inputFilesCAFCff.close()
00647
00648 str_sedCommandCommon = 'sed '
00649 if Bool_filter:
00650 str_sedCommandCommon += '-e \"s#xHLT_FILTERx# #g\" '
00651 else:
00652 str_sedCommandCommon += '-e \"s#xHLT_FILTERx#\# #g\" '
00653 if Str_datatier == 'RECO':
00654 str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx#\# #g\" '
00655 str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx#\# #g\" '
00656 else:
00657 str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx# #g\" '
00658 str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx# #g\" '
00659 str_sedCommandCommon += '-e \"s#xMAG_FIELDx#' + Str_magField + '#g\" '
00660 str_sedCommandCommon += '-e \"s#xGLOBAL_TAGx#' + Str_globalTag + '#g\" '
00661 str_sedCommandCommon += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
00662
00663 if Bool_CRAB:
00664 os.chdir(Str_nameRun)
00665 str_outputDir = '.'
00666
00667 str_sedCommand = str_sedCommandCommon
00668 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
00669 str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#' + str_outputDir + '#g\" '
00670 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
00671 os.system(str_sedCommand)
00672
00673 str_pathInputFilesJobCff = Str_pathRunIncludeDir + '/inputFiles_cff.py'
00674 file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
00675 file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n fileNames = cms.untracked.vstring (\n')
00676 nLines = 0
00677 for str_linesInput in lstr_linesInput:
00678 nLines += 1
00679 str_correctedLine1 = str_linesInput.replace(') );',',')
00680 str_correctedLine = str_correctedLine1.replace(' ] );',',')
00681 if nLines == len(lstr_linesInput):
00682 str_actualLine = str_correctedLine.replace(',','\n )\n)\n')
00683 elif nLines%255 == 0:
00684 str_actualLine = str_correctedLine.replace(',','\n )\n)\nsource.fileNames.extend(\n (')
00685 else:
00686 str_actualLine = str_correctedLine
00687 file_inputFilesJobCff.write(str_actualLine)
00688 file_inputFilesJobCff.close()
00689
00690 lstr_outpath = Str_outpath.split('/', 3)
00691 str_outpath = lstr_outpath[0] + '/' + lstr_outpath[1] + '/' + lstr_outpath[2]
00692 str_sedCommand = 'sed '
00693 str_sedCommand += '-e \"s#xSERVER_NAMEx#' + Str_server + '#g\" '
00694 str_sedCommand += '-e \"s#xDATASETPATHx#' + Str_dataset + '#g\" '
00695 str_sedCommand += '-e \"s#xRUNSELECTIONx#' + Str_run + '#g\" '
00696 str_sedCommand += '-e \"s#xNUMBER_OF_JOBSx#' + str(Int_jobs) + '#g\" '
00697 str_sedCommand += '-e \"s#xEMAILx#' + Str_email + '#g\" '
00698 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
00699 str_sedCommand += '-e \"s#xSTORAGE_PATHx#' + str_outpath + '#g\" '
00700 str_sedCommand += '-e \"s#xLFNx#' + lstr_outpath[3] + '#g\" '
00701 str_sedCommand += '-e \"s#xCOPY_DATAx#' + str(int(Bool_useCastor)) + '#g\" '
00702 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineCAF_template.crab > crab.cfg'
00703 os.system(str_sedCommand)
00704 os.chdir(Str_pathCurrentDir)
00705 else:
00706
00707 int_nInputFilesJob = int(int_nInputFiles/Int_jobs) + 1
00708 if int_nInputFiles%Int_jobs == 0:
00709 int_nInputFilesJob -= 1
00710 if int_nInputFiles == int_nInputFilesJob*(Int_jobs-1) and Int_jobs > 1:
00711 Int_jobs -= 1
00712
00713 for int_iJob in range(1,Int_jobs+1):
00714 str_nameJob = Str_nameRun + '_' + str(int_iJob).zfill(4)
00715
00716 str_nameJobDir = Str_nameRun + "/" + str_nameJob
00717 str_outputDir = '/tmp/' + os.getenv('USER') + '/' + str_nameJobDir
00718 str_pathJobIncludeDir = Str_pathRunIncludeDir + '/' + str_nameJob
00719 str_nameJobIncludeDir = STR_nameCmsswPackage.replace('/','.') + '.' + str_nameJobDir.replace('/','.')
00720 os.mkdir(str_nameJobDir)
00721 os.chdir(str_nameJobDir)
00722
00723 str_sedCommand = 'sed '
00724 str_sedCommand += '-e \"s#xCMSSW_BASEx#' + Str_pathCmsswBase + '#g\" '
00725 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
00726 str_sedCommand += '-e \"s#xJOB_NAMEx#' + str_nameJob + '#g\" '
00727 str_sedCommand += '-e \"s#xCURRENT_DIRx#' + Str_pathCurrentDir + '#g\" '
00728 str_sedCommand += '-e \"s#xCOPYx#' + str_castorCp + '#g\" '
00729 str_sedCommand += '-e \"s#xOUTPUT_DIRx#' + Str_outpath + '#g\" '
00730 str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMOfflineCAF_template.job > SiStripDQMOfflineCAF.job'
00731 os.system(str_sedCommand)
00732
00733 str_sedCommand = str_sedCommandCommon
00734 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameJobIncludeDir + '#g\" '
00735 str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#' + str_outputDir + '#g\" '
00736 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
00737 os.system(str_sedCommand)
00738
00739 os.mkdir(str_pathJobIncludeDir)
00740
00741 str_pathInputFilesJobCff = str_pathJobIncludeDir + '/inputFiles_cff.py'
00742 file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
00743 file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n fileNames = cms.untracked.vstring (\n')
00744 for n_iActualLine in range(int_nLinesRead, min(int_nLinesRead+int_nInputFilesJob, int_nInputFiles)):
00745 str_linesInput = lstr_linesInput[n_iActualLine].replace(') );',',')
00746
00747 str_actualLine = str_linesInput
00748 if (n_iActualLine+1)%int_nInputFilesJob == 0 or int_nLinesRead == int_nInputFiles-1:
00749 str_actualLine = str_linesInput.split(',')[0] + '\n'
00750 file_inputFilesJobCff.write(str_actualLine)
00751 int_nLinesRead += 1
00752 file_inputFilesJobCff.write(' )\n)\n')
00753 file_inputFilesJobCff.close()
00754
00755 os.chdir(Str_pathCurrentDir)
00756
00757 if int_nLinesRead >= int_nInputFiles:
00758 Int_jobsNew = int_iJob
00759 break
00760
00761
00762
00763 os.chdir(Str_pathRunIncludeDir+'/..')
00764 os.system('scramv1 b python')
00765 os.chdir(Str_pathCurrentDir)
00766 print
00767
00768
00769
00770 if Bool_CRAB:
00771 os.chdir(Str_nameRun)
00772 os.system('crab -create')
00773 print
00774
00775 bool_found = False
00776 file_logCRAB = file('crab'+Str_nameRun+'/log/crab.log')
00777 for str_iLine in file_logCRAB.readlines():
00778 if str_iLine.startswith('Total of ') and str_iLine.endswith(' jobs created.\n'):
00779 bool_found = True
00780 Int_jobsNew = int(str_iLine.split()[2])
00781 break
00782 file_logCRAB.close()
00783 os.chdir(Str_pathCurrentDir)
00784 if not bool_found:
00785 print '> submitDQMOfflineCAF.py > could not extract number of jobs created by CRAB; check'
00786 print ' %s' %(Str_pathInputFilesCAFCff)
00787 print ' and modify manually, if necessary'
00788 print
00789
00790
00791
00792 print '> submitDQMOfflineCAF.py > number of created jobs: %i' %(Int_jobsNew)
00793 print
00794 if Int_jobsNew != Int_jobs:
00795 file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
00796 file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n processingMode = cms.untracked.string( \'Runs\' ),\n fileNames = cms.untracked.vstring(\n')
00797 for int_iJob in range(1,Int_jobsNew+1):
00798 str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
00799 if Bool_useCastor:
00800 str_lineInput = 'rfio:' + str_lineInput
00801 str_lineInput = ' \'' + str_lineInput + '\''
00802 if int_iJob == Int_jobsNew:
00803 str_lineInput += '\n'
00804 file_inputFilesCAFCff.write(str_lineInput)
00805 break
00806 str_lineInput += ',\n'
00807 file_inputFilesCAFCff.write(str_lineInput)
00808 file_inputFilesCAFCff.write(' )\n)\n')
00809 file_inputFilesCAFCff.close()
00810
00811
00812
00813 if Dict_arguments.has_key(LSTR_functionLetters[0]):
00814 os.chdir(Str_nameRun)
00815 if Bool_CRAB:
00816 str_crabCommand = 'crab -submit -c crab' + Str_nameRun
00817 print '> submitDQMOfflineCAF.py >'
00818 print ' %s : %s' %(os.getcwd(),str_crabCommand)
00819 os.system(str_crabCommand)
00820 print
00821 time.sleep(5)
00822 os.system('crab -status -c crab' + Str_nameRun)
00823 else:
00824 for int_iJob in range(Int_jobs):
00825 str_nameJobDir = Str_nameRun + '_' + str(int_iJob).zfill(4)
00826 os.chdir(str_nameJobDir)
00827 os.chmod('SiStripDQMOfflineCAF.job',OCT_rwx_r_r)
00828 str_batchCommand = 'bsub -q cmscaf SiStripDQMOfflineCAF.job'
00829 print '> submitDQMOfflineCAF.py >'
00830 print ' %s : %s' %(os.getcwd(),str_batchCommand)
00831 os.system(str_batchCommand)
00832 print
00833 os.chdir('../')
00834 time.sleep(5)
00835 os.system('bjobs -q cmscaf')
00836 os.chmod('SiStripCAFHarvest.job',OCT_rwx_r_r)
00837 os.chdir(Str_pathCurrentDir)
00838
00839
00840
00841 if Dict_arguments.has_key(LSTR_functionLetters[0]) and not Bool_CRAB:
00842 str_mailFrom = os.getenv('USER') + STR_mailServer
00843 str_mailTo = [str_mailFrom,
00844 'volker.adler@cern.ch',
00845 'suchandra.dutta@cern.ch',
00846 'domenico.giordano@cern.ch',
00847 'vitaliano.ciulli@cern.ch']
00848 str_mailSubject = 'Your SiStrip offline DQM shift on ' + str(datetime.date.today()) + ', run ' + Str_run
00849 str_mailText = STR_mailTextOpener + Str_run + STR_mailText
00850 str_mailMessage = """From: %s
00851 To: %s
00852 Subject: %s
00853
00854 %s
00855 """ % (str_mailFrom, ", ".join(str_mailTo), str_mailSubject, str_mailText)
00856 server = smtplib.SMTP(STR_mailSmtp)
00857 server.sendmail(str_mailFrom, str_mailTo, str_mailMessage)
00858 server.quit()
1.7.3