CMSSW Reference Manual

00001 #!/usr/bin/env python
00002 
00003 #
00004 # $Id: submitDQMOfflineCAF.py,v 1.24 2008/11/27 17:36:56 vadler Exp $
00005 #
00006 
00007 ## CMSSW/DQM/SiStripMonitorClient/scripts/submitDQMOfflineCAF.py
00008 #
00009 #  This script submits CRAB/LSF jobs to the CAF in order to process the full
00010 #  granularity SiStrip offline DQM.
00011 #  Questions and comments to: volker.adler@cern.ch
00012 
00013 
00014 import sys
00015 import os
00016 import os.path
00017 import commands
00018 import shutil
00019 import string
00020 import math
00021 import urllib
00022 import time
00023 import datetime
00024 import smtplib
00025 
00026 # Constants
00027 
00028 # numbers
00029 OCT_rwx_r_r          = 0744
00030 LFLOAT_valueMagField = [0.0,2.0,3.0,3.5,3.8,4.0]
00031 TD_shiftUTC          = datetime.timedelta(hours = 2) # positive for timezones with later time than UTC
00032 # strings
00033 LSTR_true                = ['1','TRUE' ,'True' ,'true' ]
00034 LSTR_false               = ['0','FALSE','False','false']
00035 STR_default              = 'DEFAULT'
00036 STR_none                 = 'None'
00037 LSTR_auto                = ['AUTO','Auto','auto']
00038 STR_nameCmsswPackage     = 'DQM/SiStripMonitorClient'
00039 STR_textUsage            = """ CMSSW/DQM/SiStripMonitorClient/scripts/submitDQMOfflineCAF.py
00040  
00041  This script submits batch jobs to the CAF in order to process the full
00042  granularity SiStrip offline DQM.
00043  Questions and comments to: volker.adler@cern.ch
00044  
00045  Func_Usage(): submitDQMOfflineCAF.py (-s, --submit | -c, --create |
00046                                   -h, --help)
00047                                  [-r, --run]
00048                                  [-C, --CRAB]
00049                                  [-S, --server]
00050                                  [-e, --email]
00051                                  [-j, --jobs]
00052                                  [-f, --filter]
00053                                  [-d, --dataset]
00054                                  [-o, --outpath]
00055                                  [-m, --mergepath]                               
00056                                
00057    Function letters: One of the following options  m u s t  be used.
00058    
00059      -s, --submit
00060          create jobs and submit them to CAF;
00061          requires option '-r'
00062          
00063      -c, --create
00064          create jobs, but do not submit them;
00065          requires option '-r'
00066          
00067      -h, --help
00068          print this message
00069          
00070    Other options:
00071    
00072      -r, --run RUNNUMBER
00073          number of run to process;
00074          required by funtion letters '-s' and '-c'
00075       
00076      -d, --dataset PRIMARY_DATASET
00077          specify dataset for DBS query;
00078          required by funtion letters '-s' and '-c'
00079    
00080      -C, --CRAB TRUE/FALSE
00081          submit or submit not using CRAB;
00082          default: TRUE
00083          
00084          NOTE: This script runs only with CRAB 2.4.0 or higher.
00085    
00086      -S, --server CRAB_SERVER
00087          CRAB server to use;
00088          available: None (default)
00089                     caf  (works, but slow)
00090                     bari (CRAB version >= 2.4.1,
00091                           s. httpss://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#Server_available_for_users)
00092                     
00093          NOTE: CRAB server submission is disabled at the moment.
00094          
00095      -e, --email EMAIL_ADDRESS
00096          where the CRAB server should send its messages;
00097          default: volker.adler@cern.ch
00098          
00099      -j, --jobs NUMBER
00100          number of jobs to create;
00101          default: 10
00102          
00103      -g, --global-tag GLOBAL_TAG
00104          global tag to be used;
00105          default: CRAFT_V4P::All
00106          
00107      -M, --magnetic-field FIELD
00108          specification of field to be used;
00109          can be:
00110          - average field during run in Tesla, given as float (e.g.: 3.8),
00111          - specification referring to existing configuration files (e.g.: 38T)
00112            (s. Configuration/StandardSequences/python/MagneticField_[FIELD]_cff.py),
00113          - usage of automatic average field determination from CMS WBM by using "auto"
00114          default: 38T
00115                     
00116          NOTE: "auto" is disabled for the moment due to unavailablity of CMS WBM outside '.cms' network.
00117          
00118      -f, --filter TRUE/FALSE
00119          use or use not HLT filters to select events to process;
00120          default: FALSE
00121                     
00122      -o, --outpath PATH
00123          path to copy job output *.root files to;
00124          currently (almost) no check performed;
00125          must be in AFS or CASTOR
00126          default: /castor/cern.ch/user/c/cctrack/DQM
00127          
00128      -m, --mergepath PATH
00129          path to merge the job output *.root files;
00130          currently (almost) no check performed;
00131          must be in AFS or on local computer (e.g. /tmp/[user])
00132          default: /afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged
00133 """                        
00134 LSTR_datatiers = ['RECO','RAW']
00135 # argument vector
00136 LSTR_wordArgument = sys.argv[1:]
00137 # default arguments
00138 BOOL_CRAB         = True
00139 LSTR_server       = [STR_none,'caf','bari']
00140 STR_server        = LSTR_server[0]
00141 STR_email         = 'volker.adler@cern.ch'
00142 INT_jobs          = 10
00143 STR_globalTag     = 'CRAFT_V4P::All'
00144 STR_magField      = '38T'
00145 BOOL_magFieldAuto = False
00146 BOOL_filter       = False
00147 STR_outpath       = '/castor/cern.ch/user/c/cctrack/DQM'
00148 BOOL_useCastor    = True
00149 STR_mergepath     = '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged'
00150 # option lists
00151 LSTR_functionLetters = ['-s','-c','-h']
00152 DICT_functionLetters = {'--submit':LSTR_functionLetters[0],
00153                         '--create':LSTR_functionLetters[1],
00154                         '--help'  :LSTR_functionLetters[2]}
00155 LSTR_optionLetters   = ['-r','-C','-S','-e','-j','-M','-g','-f','-d','-o','-m']
00156 DICT_optionLetters   = {'--run'           :LSTR_optionLetters[0],  
00157                         '--CRAB'          :LSTR_optionLetters[1],  
00158                         '--server'        :LSTR_optionLetters[2],  
00159                         '--email'         :LSTR_optionLetters[3],  
00160                         '--jobs'          :LSTR_optionLetters[4],
00161                         '--magnetic-field':LSTR_optionLetters[5],
00162                         '--global-tag'    :LSTR_optionLetters[6],
00163                         '--filter'        :LSTR_optionLetters[7],
00164                         '--dataset'       :LSTR_optionLetters[8],
00165                         '--outpath'       :LSTR_optionLetters[9],
00166                         '--mergepath'     :LSTR_optionLetters[10]}
00167 STR_mailSmtp        = 'localhost'
00168 STR_mailServer      = '@mail.cern.ch'
00169 STR_mailTextOpener  = """Dear """ + os.getenv('USER').capitalize() + """,
00170 
00171 on """ + str(time.ctime()) + """, you have submitted run """
00172 STR_mailText = """
00173 for SiStrip offline DQM at the CAF.
00174 Unfortunately, this needed to be done from your private account. So, only you
00175 are able to finalize this submission -- even after the end of your shift.
00176 To do so, please forward all emails from the LSF batch system referring to the
00177 respective jobs to the list  t h i s  message was sent to.
00178 -- and then your shift is  r e a l l y  done :-)
00179 
00180 We are very sorry for the inconvenience.
00181 Thanks a lot!
00182 
00183 Best regards,
00184 your SiStrip DQM team
00185 
00186 P.S.:
00187 To reply to this email, simply use the "Reply to all" function of your email
00188 client.
00189 """
00190                         
00191 # Globals
00192 
00193 # arguments
00194 global Dict_arguments
00195 global Str_run
00196 global Bool_CRAB 
00197 global Str_server
00198 global Str_email
00199 global Int_jobs     
00200 global Str_globalTag
00201 global Bool_filter
00202 global Str_dataset   
00203 global Str_datatier
00204 global Str_magField
00205 global Float_magField   
00206 global Bool_magFieldAuto   
00207 global Str_outpath
00208 global Bool_useCastor
00209 global Str_mergepath
00210 # others
00211 global Str_pathCurrentDir
00212 global Str_pathCmsswBase
00213 global Str_nameCmsswRel
00214 global Str_pathCmsswBasePackage
00215 global Str_nameRun
00216 global Str_pathRunIncludeDir
00217 global Str_pathInputFilesCAFCff
00218 global Int_jobsNew
00219 global Str_magField
00220 # initialize arguments
00221 Dict_arguments    = {}
00222 Bool_CRAB         = BOOL_CRAB
00223 Str_server        = STR_server
00224 Str_email         = STR_email
00225 Int_jobs          = INT_jobs
00226 Str_globalTag     = STR_globalTag
00227 Bool_filter       = BOOL_filter
00228 Str_magField      = STR_magField
00229 Float_magField    = float(Str_magField[:-1])/10.
00230 Bool_magFieldAuto = BOOL_magFieldAuto
00231 Str_outpath       = STR_outpath
00232 Bool_useCastor    = BOOL_useCastor
00233 Str_mergepath     = STR_mergepath
00234 
00235 ## Function Func_Usage()
00236 #
00237 #  Displays usage of the script
00238 def Func_Usage():
00239   """ Function Func_Usage():
00240   Displays usage of the script
00241   """
00242   print STR_textUsage
00243 
00244 ## Function Func_Exit()
00245 #
00246 #  Exit after error
00247 def Func_Exit():
00248   """ Function Func_Exit():
00249   Exit after error
00250   """
00251   print '                           exit'
00252   print
00253   sys.exit(1)
00254 
00255 ## Function Func_ExitUsage()
00256 #
00257 #  Exit after wrong invocation of script
00258 def Func_ExitUsage():
00259   """ Function Func_ExitUsage():
00260   Exit after wrong invocation of script
00261   """
00262   print '                           exit'
00263   print
00264   Func_Usage()
00265   sys.exit(1)
00266 
00267 ## Function Func_ExitBool()
00268 #
00269 #  Exit after wrong assignment of bool option
00270 def Func_ExitBool(int_index):
00271   """ Function Func_ExitBool():
00272   Exit after wrong assignment of bool option
00273   """
00274   print '> submitDQMOfflineCAF.py > option %s expects 0/1, FALSE/TRUE, False/True or false/true' %(DICT_optionLetters.items()[int_index])
00275   Func_Exit()
00276 
00277 ## Function Func_MkDir()
00278 #
00279 #  Create new directory
00280 def Func_MkDir(str_path):
00281   """ Function Func_MkDir():
00282   Create new directory
00283   """
00284   shutil.rmtree(str_path, True)
00285   os.mkdir(str_path)
00286   
00287 ## Function Func_MagConfig(float_magFieldMeasured)
00288 #
00289 # Determine configuration to be used for a given magnetic field
00290 def Func_MagConfig(float_magFieldMeasured):
00291   """ Func_MagConfig(float_magFieldMeasured):
00292   Determine configuration to be used for a given magnetic field
00293   """
00294   float_magField = 0.0
00295   for float_valueMagField in LFLOAT_valueMagField:
00296     if math.fabs(float_valueMagField-float_magFieldMeasured) < math.fabs(float_magField-float_magFieldMeasured):
00297       float_magField = float_valueMagField
00298   return float_magField
00299   
00300 ## Main program
00301 
00302 print
00303   
00304 # Current environment
00305 
00306 Str_pathCurrentDir       = os.getcwd()
00307 Str_pathCmsswBase        = os.getenv('CMSSW_BASE')
00308 if not Str_pathCmsswBase:
00309   print '> submitDQMOfflineCAF.py > CMSSW environment not set properly;'
00310   print '                           first do'
00311   print
00312   print '                           $ cd [your/CMSSW/release/area]/src'
00313   print '                           $ cmsenv'
00314   print
00315   Func_Exit()
00316 Str_nameCmsswRel         = os.getenv('CMSSW_VERSION')
00317 Str_pathCmsswBasePackage = Str_pathCmsswBase + '/src/' + STR_nameCmsswPackage
00318 str_suffixShell          = 'csh'
00319 if not os.getenv('SHELL')[-3:] == str_suffixShell:
00320   str_suffixShell = 'sh'
00321 
00322 # Check function letters
00323 
00324 if len(LSTR_wordArgument) == 0:
00325   Func_ExitUsage()
00326 int_nFunctionLetters = 0
00327 for str_argument in LSTR_wordArgument:
00328   if str_argument in LSTR_functionLetters       or\
00329      DICT_functionLetters.has_key(str_argument)   :
00330     int_nFunctionLetters += 1
00331 if int_nFunctionLetters == 0:
00332   print '> submitDQMOfflineCAF.py > no or unknown function letter used'
00333   Func_ExitUsage()
00334 elif int_nFunctionLetters > 1:
00335   print '> submitDQMOfflineCAF.py > too many function letter used'
00336   Func_ExitUsage()
00337     
00338 # Check options
00339 
00340 str_argumentFormer = ''
00341 bool_standBy       = False
00342 for str_argument in LSTR_wordArgument:
00343   if not ( str_argument in LSTR_functionLetters or DICT_functionLetters.has_key(str_argument) or\
00344            str_argument in LSTR_optionLetters   or DICT_optionLetters.has_key(str_argument)     ):
00345     if str_argument[0] == '-':
00346       print '> submitDQMOfflineCAF.py > unknown option used'
00347       Func_ExitUsage()
00348     if not bool_standBy:
00349       print '> submitDQMOfflineCAF.py > value without option used'
00350       Func_ExitUsage()
00351     Dict_arguments[str_argumentFormer] = str_argument
00352     bool_standBy                       = False
00353   else:
00354     if bool_standBy:
00355       Dict_arguments[str_argumentFormer] = STR_default
00356       if str_argumentFormer in LSTR_optionLetters or\
00357          DICT_optionLetters.has_key(str_argumentFormer):
00358         print '> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer)
00359         print '                           default used'
00360         print
00361     bool_standBy = not ( str_argument in LSTR_functionLetters       or\
00362                          DICT_functionLetters.has_key(str_argument)   )
00363     if not bool_standBy:
00364       Dict_arguments[str_argument] = STR_default
00365   str_argumentFormer = str_argument
00366 if bool_standBy:
00367   Dict_arguments[str_argumentFormer] = STR_default
00368   if str_argumentFormer in LSTR_optionLetters       or\
00369      DICT_optionLetters.has_key(str_argumentFormer)   :
00370     print '> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer)
00371     print '                           default used'
00372     print
00373     
00374 # Correct arguments' dictionary
00375 
00376 dict_arguments = Dict_arguments
00377 for str_key, str_value in dict_arguments.items():
00378   if str_key in DICT_functionLetters.keys():
00379     del Dict_arguments[str_key]
00380     Dict_arguments[DICT_functionLetters[str_key]] = str_value
00381   if str_key in DICT_optionLetters.keys():
00382     del Dict_arguments[str_key]
00383     Dict_arguments[DICT_optionLetters[str_key]] = str_value
00384     
00385 # Help (exit)
00386 
00387 if Dict_arguments.has_key(LSTR_functionLetters[2]):
00388   Func_Usage()
00389   sys.exit(0)
00390   
00391 # Check and assign arguments
00392 
00393 # run number
00394 if Dict_arguments.has_key(LSTR_optionLetters[0])        and\
00395    Dict_arguments[LSTR_optionLetters[0]] != STR_default    :
00396   Str_run = Dict_arguments[LSTR_optionLetters[0]]
00397 else:   
00398   print '> submitDQMOfflineCAF.py > no run number given'
00399   Func_Exit()
00400 # use CRAB
00401 if Dict_arguments.has_key(LSTR_optionLetters[1])        and\
00402    Dict_arguments[LSTR_optionLetters[1]] != STR_default    :
00403   if Dict_arguments[LSTR_optionLetters[1]] in LSTR_true:
00404     Bool_CRAB = True
00405   elif Dict_arguments[LSTR_optionLetters[1]] in LSTR_false:  
00406     Bool_CRAB = False
00407   else:
00408     Func_ExitBool(1)
00409 # name of CRAB server
00410 if Dict_arguments.has_key(LSTR_optionLetters[2])        and\
00411    Dict_arguments[LSTR_optionLetters[2]] != STR_default    :
00412   Str_server = Dict_arguments[LSTR_optionLetters[2]]
00413 # email address to be used by CRAB server
00414 if Dict_arguments.has_key(LSTR_optionLetters[3])        and\
00415    Dict_arguments[LSTR_optionLetters[3]] != STR_default    :
00416   Str_email = Dict_arguments[LSTR_optionLetters[3]]
00417 # number of jobs to create
00418 if Dict_arguments.has_key(LSTR_optionLetters[4])        and\
00419    Dict_arguments[LSTR_optionLetters[4]] != STR_default    :
00420   Int_jobs  = int(Dict_arguments[LSTR_optionLetters[4]])
00421 # magnetic field
00422 if Dict_arguments.has_key(LSTR_optionLetters[5])        and\
00423    Dict_arguments[LSTR_optionLetters[5]] != STR_default    :
00424   Str_magField = Dict_arguments[LSTR_optionLetters[5]]
00425 if Str_magField in LSTR_auto:
00426 #   Bool_magFieldAuto = True
00427   print '> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment'
00428   Func_Exit()
00429 # global tag
00430 if Dict_arguments.has_key(LSTR_optionLetters[6])        and\
00431    Dict_arguments[LSTR_optionLetters[6]] != STR_default    :
00432   Str_globalTag  = Dict_arguments[LSTR_optionLetters[6]]
00433 # use HLT to filter events
00434 if Dict_arguments.has_key(LSTR_optionLetters[7])        and\
00435    Dict_arguments[LSTR_optionLetters[7]] != STR_default    :
00436   if Dict_arguments[LSTR_optionLetters[7]] in LSTR_true:
00437     Bool_filter = True
00438   elif Dict_arguments[LSTR_optionLetters[7]] in LSTR_false:  
00439     Bool_filter = False
00440   else:
00441     Func_ExitBool(7)
00442 # primary dataset
00443 if Dict_arguments.has_key(LSTR_optionLetters[8])        and\
00444    Dict_arguments[LSTR_optionLetters[8]] != STR_default    :
00445   Str_dataset = Dict_arguments[LSTR_optionLetters[8]]
00446 else:   
00447   print '> submitDQMOfflineCAF.py > no primary dataset given'
00448   Func_Exit()
00449 # path for job output
00450 if Dict_arguments.has_key(LSTR_optionLetters[9])        and\
00451    Dict_arguments[LSTR_optionLetters[9]] != STR_default    :
00452   Str_outpath = Dict_arguments[LSTR_optionLetters[9]]
00453 # path for merged output
00454 if Dict_arguments.has_key(LSTR_optionLetters[10])        and\
00455    Dict_arguments[LSTR_optionLetters[10]] != STR_default    :
00456   Str_mergepath = Dict_arguments[LSTR_optionLetters[10]]
00457   
00458 # React on arguments
00459 
00460 # on use CRAB
00461 if Bool_CRAB:
00462   str_buffer  = commands.getoutput('which crab')
00463   if str_buffer.find('which: no crab in') >= 0:
00464     str_suffixShell          = 'csh'
00465     if not os.getenv('SHELL')[-3:] == str_suffixShell:
00466       str_suffixShell = 'sh'
00467     print '> submitDQMOfflineCAF.py > CRAB environment not set properly;'
00468     print '                           please use'
00469     print
00470     print '                           $ source /afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.%s' %(str_suffixShell)
00471     print
00472     Func_Exit()
00473 # on name of CRAB server
00474 if not Str_server in LSTR_server:
00475   print '> submitDQMOfflineCAF.py > CRAB server "%s" not available' %(Str_server)
00476   Func_Exit()
00477 # on number of jobs
00478 if Int_jobs == 0:
00479   Int_jobs = 1
00480   print '> submitDQMOfflineCAF.py > number of requested jobs was 0'
00481   print '                           set to 1'
00482 # on magnetic field
00483 if Str_magField in LSTR_auto:
00484 #   Bool_magFieldAuto = True
00485   print '> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment'
00486   Func_Exit()
00487 elif Str_magField[-1] == 'T':
00488   bool_foundField = False
00489   for float_valueMagField in LFLOAT_valueMagField:
00490     if str(int(float_valueMagField*10)) == Str_magField[:-1]:
00491       Float_magField = float_valueMagField
00492       bool_foundField = True
00493       break
00494   if not bool_foundField:
00495     print '> submitDQMOfflineCAF.py > no magnet configuration for \'%s\' available' %(Str_magField)
00496     Func_Exit()
00497 else:
00498   Float_magField = float(Str_magField) # FIXME protect better from wrong user input
00499 # on primary dataset
00500 # data tier
00501 Str_datatier = Str_dataset.split('/')[-1]
00502 if not Str_datatier in LSTR_datatiers:
00503   print '> submitDQMOfflineCAF.py > datatier "%s" not processable' %(Str_datatier)
00504   Func_Exit()
00505 # on path for job output
00506 # use CASTOR
00507 if Str_outpath.split('/')[1] == 'afs':
00508   Bool_useCastor = False
00509 elif Str_outpath.split('/')[1] != 'castor':
00510   print '> submitDQMOfflineCAF.py > output path not accepted'
00511   Func_ExitUsage()
00512 str_castorCp = 'cp'
00513 if Bool_useCastor:
00514   str_castorCp = 'rfcp'
00515 # on path for merged output
00516 if Str_mergepath.split('/')[1] != 'afs':
00517   print '> submitDQMOfflineCAF.py > merge path not accepted'
00518   Func_ExitUsage()
00519   
00520 # Prepare work area
00521 
00522 # string identifying run (run name)  
00523 Str_nameRun = 'R' + Str_run.zfill(9)
00524 #  directories
00525 Func_MkDir(Str_nameRun)
00526 Str_pathRunIncludeDir = Str_pathCmsswBasePackage + '/python/' + Str_nameRun
00527 Func_MkDir(Str_pathRunIncludeDir)
00528 str_nameInputFilesFile = Str_nameRun + '/' + Str_nameRun + '.txt'
00529 str_nameRunIncludeDir  = STR_nameCmsswPackage.replace('/','.') + '.' + Str_nameRun
00530 
00531 # Retrieving information from the web
00532 
00533 # input files
00534 int_nInputFiles    = 0
00535 file_inputFilesCff = file(str_nameInputFilesFile, 'w')
00536 # DBS query for list of input files
00537 str_dbsParams  = urllib.urlencode({'dbsInst':'cms_dbs_prod_global', 'blockName':'*', 'dataset':Str_dataset, 'userMode':'user', 'run':Str_run, 'what':'py'})
00538 file_dbsOutput = urllib.urlopen("httpss://cmsweb.cern.ch/dbs_discovery/getLFN_txt", str_dbsParams)
00539 for str_iLine in file_dbsOutput.readlines():
00540   lstr_wordsLine = str_iLine.split("/")
00541   if len(lstr_wordsLine) >= 5:
00542     # Necessary and (hopefully) temporary fix due to wrongly assigned dataset
00543 #     if                  lstr_wordsLine[1]  == 'store'      and\
00544 #                         lstr_wordsLine[2]  == 'data'       and\
00545 #        Str_dataset.find(lstr_wordsLine[3]) >= 0            and\
00546 #        Str_dataset.find(lstr_wordsLine[4]) >= 0            and\
00547 #                         lstr_wordsLine[5]  == Str_datatier and\
00548 #        Str_dataset.find(lstr_wordsLine[6]) >= 0            and\
00549 #                     len(lstr_wordsLine[7]) == 3               :
00550     if                  lstr_wordsLine[1]  == 'store'      and\
00551                         lstr_wordsLine[2]  == 'data'       and\
00552        Str_dataset.find(lstr_wordsLine[3]) >= 0            and\
00553        Str_dataset.find(lstr_wordsLine[4]) >= 0            and\
00554                         lstr_wordsLine[5]  == Str_datatier and\
00555                     len(lstr_wordsLine[7]) == 3               :
00556       int_nInputFiles += 1
00557       file_inputFilesCff.write(str_iLine)
00558 if int_nInputFiles == 0:
00559   print '> submitDQMOfflineCAF.py > no input files found in DBS for run %s in dataset %s' %(Str_run,Str_dataset)
00560   Func_Exit()
00561 file_inputFilesCff.close()
00562 print '> submitDQMOfflineCAF.py > input files for run %s:   %i' %(Str_run,int_nInputFiles)
00563 if int_nInputFiles < Int_jobs:
00564   Int_jobs = int_nInputFiles
00565   print '                           number of requested jobs reduced accordingly'
00566 Int_jobsNew = Int_jobs
00567 print
00568 
00569 # magnetic field
00570 if Bool_magFieldAuto:
00571   # extract time stamps of the run
00572   str_cmsmonParams  = urllib.urlencode({'RUN':Str_run})
00573   file_cmsmonOutput = urllib.urlopen("https://cmsmon.cern.ch/cmsdb/servlet/RunSummary", str_cmsmonParams)
00574   str_timeBegin     = ''
00575   str_timeEnd       = ''
00576   for str_cmsmonOutput in file_cmsmonOutput.readlines():
00577     if str_cmsmonOutput.find('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER') >= 0:
00578       lstr_timeQuery = str_cmsmonOutput.split('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER&')[1].split('>TRACKER')[0].split('&')
00579       for str_timeQuery in lstr_timeQuery:
00580         str_nameStamp = str_timeQuery.split('=')[0]
00581         lstr_timeDate = str_timeQuery.split('=')[1].split('_')[0].split('.')
00582         lstr_timeTime = str_timeQuery.split('=')[1].split('_')[1].split(':')
00583         dt_stampOld   = datetime.datetime(int(lstr_timeDate[0]),int(lstr_timeDate[1]),int(lstr_timeDate[2]),int(lstr_timeTime[0]),int(lstr_timeTime[1]),int(lstr_timeTime[2]))
00584         dt_stampNew   = dt_stampOld - TD_shiftUTC
00585         str_timeStamp = str(dt_stampNew).replace('-','.') 
00586         if str_nameStamp == 'TIME_BEGIN':
00587           str_timeBegin = str_timeStamp
00588         elif str_nameStamp == 'TIME_END':
00589           str_timeEnd = str_timeStamp
00590   # get magnetic field itself
00591   str_cmsmonParams  = urllib.urlencode({'TIME_BEGIN':str_timeBegin, 'TIME_END':str_timeEnd})
00592   file_cmsmonOutput = urllib.urlopen("https://cmsmon.cern.ch/cmsdb/servlet/MagnetHistory", str_cmsmonParams)
00593   bool_foundField = False
00594   for str_cmsmonOutput in file_cmsmonOutput.readlines():
00595     if str_cmsmonOutput.find('BFIELD, Tesla') >= 0:
00596       Float_magField = float(str_cmsmonOutput.split('</A>')[0].split('>')[-1])
00597       bool_foundField = True
00598       break
00599   if not bool_foundField:
00600     print  '> submitDQMOfflineCAF.py > could not extract magnetic field'
00601     print  '                           please provide value'
00602     Func_Exit()
00603 # determine corresponding configuration file to be included
00604 float_magField = Func_MagConfig(Float_magField)
00605 Str_magField   = str(int(float_magField*10)) + 'T'
00606 print '> submitDQMOfflineCAF.py > (average) magnetic field in run %s:   %s T' %(Str_run,Float_magField)
00607 print '                           using %s T for configuration' %(float_magField)
00608 print
00609 
00610 # Create scripts
00611 
00612 int_nLinesRead     = 0
00613 file_inputFilesCff = file(str_nameInputFilesFile)
00614 lstr_linesInput    = file_inputFilesCff.readlines()
00615 file_inputFilesCff.close()
00616 
00617 # create harvesting config file and job script
00618 str_sedCommand  = 'sed '
00619 str_sedCommand += '-e \"s#xMAG_FIELDx#'         + Str_magField          + '#g\" '
00620 str_sedCommand += '-e \"s#xGLOBAL_TAGx#'        + Str_globalTag         + '#g\" '
00621 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
00622 str_sedCommand += '-e \"s#xMERGE_PATHx#'        + Str_mergepath         + '#g\" '
00623 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripCAFHarvest_template_cfg.py > ' + Str_nameRun + '/SiStripCAFHarvest_cfg.py'
00624 os.system(str_sedCommand)
00625 str_sedCommand  = 'sed '
00626 str_sedCommand += '-e \"s#xCMSSW_BASEx#'    + Str_pathCmsswBase  + '#g\" '
00627 str_sedCommand += '-e \"s#xRUN_NAMEx#'      + Str_nameRun        + '#g\" '
00628 str_sedCommand += '-e \"s#xMERGE_PATHx#'    + Str_mergepath      + '#g\" '
00629 str_sedCommand += '-e \"s#xCURRENT_DIRx#'   + Str_pathCurrentDir + '#g\" '
00630 str_sedCommand += '-e \"s#xDATA_TIERx#'     + Str_datatier       + '#g\" '
00631 str_sedCommand += '-e \"s#xCMSSW_VERSIONx#' + Str_nameCmsswRel   + '#g\" '
00632 str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMCAFHarvest_template.job > ' + Str_nameRun + '/SiStripCAFHarvest.job'
00633 os.system(str_sedCommand)
00634 # create included CAF input files list (mainly for compilation)
00635 Str_pathInputFilesCAFCff = Str_pathRunIncludeDir + '/inputFilesCAF_cff.py'
00636 file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
00637 file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n    processingMode = cms.untracked.string( \'Runs\' ),\n    fileNames      = cms.untracked.vstring(\n')
00638 for int_iJob in range(1,Int_jobs+1):
00639   str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
00640   if Bool_useCastor:
00641     str_lineInput = 'rfio:' + str_lineInput
00642   str_lineInput = '        \'' + str_lineInput + '\''
00643   if int_iJob == Int_jobs:
00644     str_lineInput += '\n'
00645     file_inputFilesCAFCff.write(str_lineInput)
00646     break
00647   str_lineInput += ',\n'
00648   file_inputFilesCAFCff.write(str_lineInput)
00649 file_inputFilesCAFCff.write('    )\n)\n')
00650 file_inputFilesCAFCff.close()
00651 
00652 str_sedCommandCommon = 'sed '
00653 if Bool_filter:
00654   str_sedCommandCommon += '-e \"s#xHLT_FILTERx#    #g\" '
00655 else:
00656   str_sedCommandCommon += '-e \"s#xHLT_FILTERx#\#     #g\" '
00657 if Str_datatier == 'RECO':
00658   str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx#\#     #g\" '
00659   str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx#\#     #g\" '
00660 else:
00661   str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx#    #g\" '
00662   str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx#    #g\" '
00663 str_sedCommandCommon += '-e \"s#xMAG_FIELDx#'  + Str_magField  + '#g\" '
00664 str_sedCommandCommon += '-e \"s#xGLOBAL_TAGx#' + Str_globalTag + '#g\" '
00665 str_sedCommandCommon += '-e \"s#xRUN_NAMEx#'   + Str_nameRun   + '#g\" '
00666 
00667 if Bool_CRAB:
00668   os.chdir(Str_nameRun)     
00669   str_outputDir = '.'
00670   # create main configuration file
00671   str_sedCommand = str_sedCommandCommon
00672   str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
00673   str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#'  + str_outputDir         + '#g\" '
00674   str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
00675   os.system(str_sedCommand)
00676   # create included input files list
00677   str_pathInputFilesJobCff = Str_pathRunIncludeDir + '/inputFiles_cff.py'
00678   file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
00679   file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n    fileNames = cms.untracked.vstring (\n')
00680   nLines = 0
00681   for str_linesInput in lstr_linesInput:
00682     nLines += 1
00683     str_correctedLine = str_linesInput.replace(') );',',')
00684     if nLines == len(lstr_linesInput):
00685       str_actualLine = str_correctedLine.replace(',','\n    )\n)\n')
00686     elif nLines%255 == 0: # FIXME add this check also to LSF
00687       str_actualLine = str_correctedLine.replace(',','\n    )\n)\nsource.fileNames.extend(\n    (')
00688     else:
00689       str_actualLine = str_correctedLine
00690     file_inputFilesJobCff.write(str_actualLine)
00691   file_inputFilesJobCff.close()
00692   # create CRAB configuration file
00693   lstr_outpath = Str_outpath.split('/', 3)
00694   str_outpath  = lstr_outpath[0] + '/' + lstr_outpath[1] + '/' + lstr_outpath[2] 
00695   str_sedCommand  = 'sed '
00696   str_sedCommand += '-e \"s#xSERVER_NAMEx#'    + Str_server               + '#g\" '
00697   str_sedCommand += '-e \"s#xDATASETPATHx#'    + Str_dataset              + '#g\" '
00698   str_sedCommand += '-e \"s#xRUNSELECTIONx#'   + Str_run                  + '#g\" '
00699   str_sedCommand += '-e \"s#xNUMBER_OF_JOBSx#' + str(Int_jobs)            + '#g\" '
00700   str_sedCommand += '-e \"s#xEMAILx#'          + Str_email                + '#g\" '
00701   str_sedCommand += '-e \"s#xRUN_NAMEx#'       + Str_nameRun              + '#g\" '
00702   str_sedCommand += '-e \"s#xSTORAGE_PATHx#'   + str_outpath              + '#g\" '
00703   str_sedCommand += '-e \"s#xLFNx#'            + lstr_outpath[3]          + '#g\" '
00704   str_sedCommand += '-e \"s#xCOPY_DATAx#'      + str(int(Bool_useCastor)) + '#g\" '
00705   str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineCAF_template.crab > crab.cfg'
00706   os.system(str_sedCommand)
00707   os.chdir(Str_pathCurrentDir)
00708 else:
00709   # FIXME: following calculation has to be reviewed
00710   int_nInputFilesJob = int(int_nInputFiles/Int_jobs) + 1
00711   if int_nInputFiles%Int_jobs == 0:
00712     int_nInputFilesJob -= 1
00713   if int_nInputFiles == int_nInputFilesJob*(Int_jobs-1) and Int_jobs > 1:
00714     Int_jobs -= 1
00715   # loop over single jobs
00716   for int_iJob in range(1,Int_jobs+1):
00717     str_nameJob = Str_nameRun + '_' + str(int_iJob).zfill(4)
00718     # prepare job dir
00719     str_nameJobDir        = Str_nameRun + "/" + str_nameJob
00720     str_outputDir         = '/tmp/' + os.getenv('USER') + '/' + str_nameJobDir
00721     str_pathJobIncludeDir = Str_pathRunIncludeDir + '/' + str_nameJob
00722     str_nameJobIncludeDir = STR_nameCmsswPackage.replace('/','.') + '.' + str_nameJobDir.replace('/','.')
00723     os.mkdir(str_nameJobDir)
00724     os.chdir(str_nameJobDir)     
00725     # create job script
00726     str_sedCommand = 'sed '
00727     str_sedCommand += '-e \"s#xCMSSW_BASEx#'  + Str_pathCmsswBase  + '#g\" '
00728     str_sedCommand += '-e \"s#xRUN_NAMEx#'    + Str_nameRun        + '#g\" '
00729     str_sedCommand += '-e \"s#xJOB_NAMEx#'    + str_nameJob        + '#g\" '
00730     str_sedCommand += '-e \"s#xCURRENT_DIRx#' + Str_pathCurrentDir + '#g\" '
00731     str_sedCommand += '-e \"s#xCOPYx#'        + str_castorCp       + '#g\" '
00732     str_sedCommand += '-e \"s#xOUTPUT_DIRx#'  + Str_outpath        + '#g\" '
00733     str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMOfflineCAF_template.job > SiStripDQMOfflineCAF.job'
00734     os.system(str_sedCommand)
00735     # create main configuration file
00736     str_sedCommand = str_sedCommandCommon
00737     str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameJobIncludeDir + '#g\" '
00738     str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#'  + str_outputDir         + '#g\" '
00739     str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
00740     os.system(str_sedCommand)
00741     # prepare job include dir
00742     os.mkdir(str_pathJobIncludeDir)
00743     # create included input files list
00744     str_pathInputFilesJobCff = str_pathJobIncludeDir + '/inputFiles_cff.py'
00745     file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
00746     file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n    fileNames = cms.untracked.vstring (\n')
00747     for n_iActualLine in range(int_nLinesRead, min(int_nLinesRead+int_nInputFilesJob, int_nInputFiles)):
00748       str_linesInput = lstr_linesInput[n_iActualLine].replace(') );',',')
00749       # fix commata and end of line
00750       str_actualLine = str_linesInput
00751       if (n_iActualLine+1)%int_nInputFilesJob == 0 or int_nLinesRead == int_nInputFiles-1:
00752         str_actualLine = str_linesInput.split(',')[0] + '\n'
00753       file_inputFilesJobCff.write(str_actualLine)
00754       int_nLinesRead += 1
00755     file_inputFilesJobCff.write('    )\n)\n')
00756     file_inputFilesJobCff.close()
00757     # finalize job creation
00758     os.chdir(Str_pathCurrentDir)
00759     # FIXME: This protection is currently needed. Review calculations again!
00760     if int_nLinesRead >= int_nInputFiles:
00761       Int_jobsNew = int_iJob
00762       break
00763 
00764 # Compile
00765 
00766 os.chdir(Str_pathRunIncludeDir+'/..')
00767 os.system('scramv1 b python')
00768 os.chdir(Str_pathCurrentDir)
00769 print
00770 
00771 # Create CRAB
00772 
00773 if Bool_CRAB:
00774   os.chdir(Str_nameRun)     
00775   os.system('crab -create')
00776   print
00777   # extract number of really created jobs
00778   bool_found = False
00779   file_logCRAB = file('crab'+Str_nameRun+'/log/crab.log')
00780   for str_iLine in file_logCRAB.readlines():
00781     if str_iLine.startswith('Total of ') and str_iLine.endswith(' jobs created.\n'):
00782       bool_found = True
00783       Int_jobsNew = int(str_iLine.split()[2])
00784       break
00785   file_logCRAB.close()
00786   os.chdir(Str_pathCurrentDir)
00787   if not bool_found:
00788     print '> submitDQMOfflineCAF.py > could not extract number of jobs created by CRAB; check'
00789     print '                           %s' %(Str_pathInputFilesCAFCff)
00790     print '                           and modify manually, if necessary'
00791     print
00792   
00793 # recreate included CAF input files list according to number of created jobs, if necessary
00794 
00795 print '> submitDQMOfflineCAF.py > number of created jobs: %i' %(Int_jobsNew)
00796 print
00797 if Int_jobsNew != Int_jobs:
00798   file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
00799   file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n    processingMode = cms.untracked.string( \'Runs\' ),\n    fileNames      = cms.untracked.vstring(\n')
00800   for int_iJob in range(1,Int_jobsNew+1): # FIXME use number of CRAB jobs created
00801     str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
00802     if Bool_useCastor:
00803       str_lineInput = 'rfio:' + str_lineInput
00804     str_lineInput = '        \'' + str_lineInput + '\''
00805     if int_iJob == Int_jobsNew:
00806       str_lineInput += '\n'
00807       file_inputFilesCAFCff.write(str_lineInput)
00808       break
00809     str_lineInput += ',\n'
00810     file_inputFilesCAFCff.write(str_lineInput)
00811   file_inputFilesCAFCff.write('    )\n)\n')
00812   file_inputFilesCAFCff.close()
00813 
00814 # Submit jobs
00815 
00816 if Dict_arguments.has_key(LSTR_functionLetters[0]):
00817   os.chdir(Str_nameRun)
00818   if Bool_CRAB:
00819     str_crabCommand = 'crab -submit -c crab' + Str_nameRun
00820     print '> submitDQMOfflineCAF.py >'
00821     print '  %s : %s' %(os.getcwd(),str_crabCommand)
00822     os.system(str_crabCommand)
00823     print
00824     time.sleep(5)
00825     os.system('crab -status -c crab' + Str_nameRun)
00826   else:
00827     for int_iJob in range(Int_jobs):
00828       str_nameJobDir = Str_nameRun + '_' + str(int_iJob).zfill(4)
00829       os.chdir(str_nameJobDir)     
00830       os.chmod('SiStripDQMOfflineCAF.job',OCT_rwx_r_r)
00831       str_batchCommand = 'bsub -q cmscaf SiStripDQMOfflineCAF.job'
00832       print '> submitDQMOfflineCAF.py >'
00833       print '  %s : %s' %(os.getcwd(),str_batchCommand)
00834       os.system(str_batchCommand)
00835       print
00836       os.chdir('../')
00837     time.sleep(5)
00838     os.system('bjobs -q cmscaf')
00839   os.chmod('SiStripCAFHarvest.job',OCT_rwx_r_r)
00840   os.chdir(Str_pathCurrentDir)
00841 
00842 # Send reminder email to submitter (not needed for CRAB)
00843     
00844 if Dict_arguments.has_key(LSTR_functionLetters[0]) and not Bool_CRAB:
00845   str_mailFrom    = os.getenv('USER') + STR_mailServer
00846   str_mailTo      = [str_mailFrom,
00847                      'volker.adler@cern.ch',
00848                      'suchandra.dutta@cern.ch',
00849                      'domenico.giordano@cern.ch',
00850                      'vitaliano.ciulli@cern.ch']
00851   str_mailSubject = 'Your SiStrip offline DQM shift on ' + str(datetime.date.today()) + ', run ' + Str_run
00852   str_mailText    = STR_mailTextOpener + Str_run + STR_mailText
00853   str_mailMessage = """From: %s
00854 To: %s
00855 Subject: %s
00856 
00857 %s
00858 """  % (str_mailFrom, ", ".join(str_mailTo), str_mailSubject, str_mailText)   
00859   server = smtplib.SMTP(STR_mailSmtp)
00860   server.sendmail(str_mailFrom, str_mailTo, str_mailMessage)
00861   server.quit()