CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
cmsBatch Namespace Reference

Classes

class  CmsBatchException
 
class  MyBatchManager
 

Functions

def batchScriptCCIN2P3
 
def batchScriptCERN
 
def batchScriptLocal
 
def rootfiles_to_eos_script
 

Variables

tuple batchManager = MyBatchManager()
 
tuple cfgFile = open(batchManager.outputDir_+'/base_cfg.py','w')
 
list cfgFileName = args[1]
 
tuple cfo = imp.load_source("pycfg", cfgFileName, handle)
 
string default = "cmsRun"
 
 doCVSTag = options.tagPackages
 
tuple file = open('cmsBatch.txt', 'w')
 
tuple fullSource = process.source.clone()
 
 generator = False
 
tuple grouping = int(args[0])
 
tuple handle = open(cfgFileName, 'r')
 
string help = "program to run on your cfg file"
 batchManager.parser_.add_option("-b", "--batch", dest="batch", help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.", default="bsub -q 8nh < .batchScript.sh") More...
 
list listOfValues = [i+1 for i in range( nJobs )]
 
tuple log = logger( logDir )
 
string logDir = 'Logger'
 
tuple nFiles = len(process.source.fileNames)
 
 nJobs = grouping
 
tuple oldPwd = os.getcwd()
 
 process = cfo.process
 
 prog = options.prog
 
 pycfg_params = options.cmdargs
 
 runningMode = None
 
 trueArgv = sys.argv
 
int waitingTime = 1
 

Function Documentation

def cmsBatch.batchScriptCCIN2P3 ( )

Definition at line 19 of file cmsBatch.py.

19 
20 def batchScriptCCIN2P3():
21  script = """!/usr/bin/env bash
22 #PBS -l platform=LINUX,u_sps_cmsf,M=2000MB,T=2000000
23 # sets the queue
24 #PBS -q T
25 #PBS -eo
26 #PBS -me
27 #PBS -V
28 
29 source $HOME/.bash_profile
30 
31 echo '***********************'
32 
33 ulimit -v 3000000
34 
35 # coming back to submission dir do setup the env
36 cd $PBS_O_WORKDIR
37 eval `scramv1 ru -sh`
38 
39 
40 # back to the worker
41 cd -
42 
43 # copy job dir here
44 cp -r $PBS_O_WORKDIR .
45 
46 # go inside
47 jobdir=`ls`
48 echo $jobdir
49 
50 cd $jobdir
51 
52 cat > sysinfo.sh <<EOF
53 #! env bash
54 echo '************** ENVIRONMENT ****************'
55 
56 env
57 
58 echo
59 echo '************** WORKER *********************'
60 echo
61 
62 free
63 cat /proc/cpuinfo
64 
65 echo
66 echo '************** START *********************'
67 echo
68 EOF
69 
70 source sysinfo.sh > sysinfo.txt
71 
72 cmsRun run_cfg.py
73 
74 # copy job dir do disk
75 cd -
76 cp -r $jobdir $PBS_O_WORKDIR
77 """
78  return script
79 
80 
def batchScriptCCIN2P3
Definition: cmsBatch.py:19
def cmsBatch.batchScriptCERN (   remoteDir,
  index 
)
prepare the LSF version of the batch script, to run on LSF

Definition at line 94 of file cmsBatch.py.

References rootfiles_to_eos_script().

Referenced by cmsBatch.MyBatchManager.PrepareJobUser().

94 
95 def batchScriptCERN( remoteDir, index ):
96  '''prepare the LSF version of the batch script, to run on LSF'''
97  script = """#!/bin/bash
98 # sets the queue
99 #BSUB -q 8nm
100 
101 echo 'environment:'
102 echo
103 env
104 ulimit -v 3000000
105 echo 'copying job dir to worker'
106 cd $CMSSW_BASE/src
107 eval `scramv1 ru -sh`
108 cd -
109 cp -rf $LS_SUBCWD .
110 ls
111 cd `find . -type d | grep /`
112 echo 'running'
113 {prog} run_cfg.py
114 if [ $? != 0 ]; then
115  echo wrong exit code! removing all root files
116  rm *.root
117  exit 1
118 fi
119 echo 'sending the job directory back'
120 """.format(prog=prog)
121 
122  if remoteDir != '':
123  script += rootfiles_to_eos_script(index, remoteDir)
124 
125  script += 'cp -rf * $LS_SUBCWD\n'
126 
127  return script
def rootfiles_to_eos_script
Definition: cmsBatch.py:81
def batchScriptCERN
Definition: cmsBatch.py:94
def cmsBatch.batchScriptLocal (   remoteDir,
  index 
)
prepare a local version of the batch script, to run using nohup

Definition at line 128 of file cmsBatch.py.

References rootfiles_to_eos_script().

Referenced by cmsBatch.MyBatchManager.PrepareJobUser().

129 def batchScriptLocal( remoteDir, index ):
130  '''prepare a local version of the batch script, to run using nohup'''
131 
132  script = """#!/bin/bash
133 echo 'running'
134 {prog} run_cfg.py
135 if [ $? != 0 ]; then
136  echo wrong exit code! removing all root files
137  rm *.root
138  exit 1
139 fi
140 echo 'sending the job directory back'
141 """.format(prog=prog)
142 
143  if remoteDir != '':
144  script += rootfiles_to_eos_script(index, remoteDir)
145 
146  return script
147 
def batchScriptLocal
Definition: cmsBatch.py:128
def rootfiles_to_eos_script
Definition: cmsBatch.py:81
def cmsBatch.rootfiles_to_eos_script (   index,
  remoteDir 
)

Definition at line 81 of file cmsBatch.py.

References eostools.eosToLFN().

Referenced by batchScriptCERN(), and batchScriptLocal().

81 
82 def rootfiles_to_eos_script(index, remoteDir):
83  remoteDir = eostools.eosToLFN(remoteDir)
84  return """
85 for file in *.root; do
86 newFileName=`echo $file | sed -r -e 's/\./_{index}\./'`
87 fullFileName={remoteDir}/$newFileName
88 {eos} cp $file /eos/cms/$fullFileName
89 {eos} chmod 755 /eos/cms/$fullFileName
90 rm *.root
91 done
92 """.format(index=index, remoteDir=remoteDir, eos=eostools.eos_select)
93 
def eosToLFN
Definition: eostools.py:66
def rootfiles_to_eos_script
Definition: cmsBatch.py:81

Variable Documentation

tuple cmsBatch.batchManager = MyBatchManager()

Definition at line 200 of file cmsBatch.py.

tuple cmsBatch.cfgFile = open(batchManager.outputDir_+'/base_cfg.py','w')

Definition at line 321 of file cmsBatch.py.

list cmsBatch.cfgFileName = args[1]

Definition at line 273 of file cmsBatch.py.

tuple cmsBatch.cfo = imp.load_source("pycfg", cfgFileName, handle)

Definition at line 287 of file cmsBatch.py.

string cmsBatch.default = "cmsRun"

Definition at line 242 of file cmsBatch.py.

cmsBatch.doCVSTag = options.tagPackages

Definition at line 257 of file cmsBatch.py.

tuple cmsBatch.file = open('cmsBatch.txt', 'w')

Definition at line 203 of file cmsBatch.py.

tuple cmsBatch.fullSource = process.source.clone()

Definition at line 296 of file cmsBatch.py.

cmsBatch.generator = False

Definition at line 297 of file cmsBatch.py.

tuple cmsBatch.grouping = int(args[0])

Definition at line 271 of file cmsBatch.py.

tuple cmsBatch.handle = open(cfgFileName, 'r')

Definition at line 286 of file cmsBatch.py.

string cmsBatch.help = "program to run on your cfg file"

batchManager.parser_.add_option("-b", "--batch", dest="batch", help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.", default="bsub -q 8nh < .batchScript.sh")

Definition at line 241 of file cmsBatch.py.

list cmsBatch.listOfValues = [i+1 for i in range( nJobs )]

Definition at line 304 of file cmsBatch.py.

tuple cmsBatch.log = logger( logDir )

Definition at line 343 of file cmsBatch.py.

string cmsBatch.logDir = 'Logger'

Definition at line 341 of file cmsBatch.py.

tuple cmsBatch.nFiles = len(process.source.fileNames)

Definition at line 308 of file cmsBatch.py.

Referenced by SiStripSpyMonitorModule.analyze(), FFTJetPileupProcessor.mixExtraGrid(), SiStripSpyMonitorModule.SiStripSpyMonitorModule(), and SiStripSpyMonitorModule.~SiStripSpyMonitorModule().

int cmsBatch.nJobs = grouping

Definition at line 272 of file cmsBatch.py.

tuple cmsBatch.oldPwd = os.getcwd()

Definition at line 339 of file cmsBatch.py.

cmsBatch.process = cfo.process

Definition at line 288 of file cmsBatch.py.

cmsBatch.prog = options.prog

Definition at line 256 of file cmsBatch.py.

cmsBatch.pycfg_params = options.cmdargs

Definition at line 277 of file cmsBatch.py.

tuple cmsBatch.runningMode = None

Definition at line 264 of file cmsBatch.py.

cmsBatch.trueArgv = sys.argv

Definition at line 278 of file cmsBatch.py.

int cmsBatch.waitingTime = 1

Definition at line 327 of file cmsBatch.py.