CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
cmsBatch Namespace Reference

Classes

class  CmsBatchException
 
class  MyBatchManager
 

Functions

def batchScriptCCIN2P3
 
def batchScriptCERN
 
def batchScriptLocal
 
def rootfiles_to_eos_script
 

Variables

tuple batchManager = MyBatchManager()
 
tuple cfgFile = open(batchManager.outputDir_+'/base_cfg.py','w')
 
list cfgFileName = args[1]
 
tuple cfo = imp.load_source("pycfg", cfgFileName, handle)
 
string default = "cmsRun"
 
 doCVSTag = options.tagPackages
 
tuple file = open('cmsBatch.txt', 'w')
 
tuple fullSource = process.source.clone()
 
 generator = False
 
tuple grouping = int(args[0])
 
tuple handle = open(cfgFileName, 'r')
 
string help = "program to run on your cfg file"
 batchManager.parser_.add_option("-b", "--batch", dest="batch", help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.", default="bsub -q 8nh < .batchScript.sh") More...
 
list listOfValues = [i+1 for i in range( nJobs )]
 
tuple log = logger( logDir )
 
string logDir = 'Logger'
 
tuple nFiles = len(process.source.fileNames)
 
 nJobs = grouping
 
tuple oldPwd = os.getcwd()
 
 process = cfo.process
 
 prog = options.prog
 
 pycfg_params = options.cmdargs
 
 runningMode = None
 
 trueArgv = sys.argv
 
int waitingTime = 1
 

Function Documentation

def cmsBatch.batchScriptCCIN2P3 ( )

Definition at line 17 of file cmsBatch.py.

17 
18 def batchScriptCCIN2P3():
19  script = """!/usr/bin/env bash
20 #PBS -l platform=LINUX,u_sps_cmsf,M=2000MB,T=2000000
21 # sets the queue
22 #PBS -q T
23 #PBS -eo
24 #PBS -me
25 #PBS -V
26 
27 source $HOME/.bash_profile
28 
29 echo '***********************'
30 
31 ulimit -v 3000000
32 
33 # coming back to submission dir do setup the env
34 cd $PBS_O_WORKDIR
35 eval `scramv1 ru -sh`
36 
37 
38 # back to the worker
39 cd -
40 
41 # copy job dir here
42 cp -r $PBS_O_WORKDIR .
43 
44 # go inside
45 jobdir=`ls`
46 echo $jobdir
47 
48 cd $jobdir
49 
50 cat > sysinfo.sh <<EOF
51 #! env bash
52 echo '************** ENVIRONMENT ****************'
53 
54 env
55 
56 echo
57 echo '************** WORKER *********************'
58 echo
59 
60 free
61 cat /proc/cpuinfo
62 
63 echo
64 echo '************** START *********************'
65 echo
66 EOF
67 
68 source sysinfo.sh > sysinfo.txt
69 
70 cmsRun run_cfg.py
71 
72 # copy job dir do disk
73 cd -
74 cp -r $jobdir $PBS_O_WORKDIR
75 """
76  return script
77 
78 
def batchScriptCCIN2P3
Definition: cmsBatch.py:17
def cmsBatch.batchScriptCERN (   remoteDir,
  index 
)
prepare the LSF version of the batch script, to run on LSF

Definition at line 92 of file cmsBatch.py.

References rootfiles_to_eos_script().

Referenced by cmsBatch.MyBatchManager.PrepareJobUser().

92 
93 def batchScriptCERN( remoteDir, index ):
94  '''prepare the LSF version of the batch script, to run on LSF'''
95  script = """#!/bin/bash
96 # sets the queue
97 #BSUB -q 8nm
98 
99 echo 'environment:'
100 echo
101 env
102 ulimit -v 3000000
103 echo 'copying job dir to worker'
104 cd $CMSSW_BASE/src
105 eval `scramv1 ru -sh`
106 cd -
107 cp -rf $LS_SUBCWD .
108 ls
109 cd `find . -type d | grep /`
110 echo 'running'
111 {prog} run_cfg.py
112 if [ $? != 0 ]; then
113  echo wrong exit code! removing all root files
114  rm *.root
115  exit 1
116 fi
117 echo 'sending the job directory back'
118 """.format(prog=prog)
119 
120  if remoteDir != '':
121  script += rootfiles_to_eos_script(index, remoteDir)
122 
123  script += 'cp -rf * $LS_SUBCWD\n'
124 
125  return script
def rootfiles_to_eos_script
Definition: cmsBatch.py:79
def batchScriptCERN
Definition: cmsBatch.py:92
def cmsBatch.batchScriptLocal (   remoteDir,
  index 
)
prepare a local version of the batch script, to run using nohup

Definition at line 126 of file cmsBatch.py.

References rootfiles_to_eos_script().

Referenced by cmsBatch.MyBatchManager.PrepareJobUser().

127 def batchScriptLocal( remoteDir, index ):
128  '''prepare a local version of the batch script, to run using nohup'''
129 
130  script = """#!/bin/bash
131 echo 'running'
132 {prog} run_cfg.py
133 if [ $? != 0 ]; then
134  echo wrong exit code! removing all root files
135  rm *.root
136  exit 1
137 fi
138 echo 'sending the job directory back'
139 """.format(prog=prog)
140 
141  if remoteDir != '':
142  script += rootfiles_to_eos_script(index, remoteDir)
143 
144  return script
145 
def batchScriptLocal
Definition: cmsBatch.py:126
def rootfiles_to_eos_script
Definition: cmsBatch.py:79
def cmsBatch.rootfiles_to_eos_script (   index,
  remoteDir 
)

Definition at line 79 of file cmsBatch.py.

References eostools.eosToLFN().

Referenced by batchScriptCERN(), and batchScriptLocal().

79 
80 def rootfiles_to_eos_script(index, remoteDir):
81  remoteDir = eostools.eosToLFN(remoteDir)
82  return """
83 for file in *.root; do
84 newFileName=`echo $file | sed -r -e 's/\./_{index}\./'`
85 fullFileName={remoteDir}/$newFileName
86 {eos} cp $file /eos/cms/$fullFileName
87 {eos} chmod 755 /eos/cms/$fullFileName
88 rm *.root
89 done
90 """.format(index=index, remoteDir=remoteDir, eos=eostools.eos_select)
91 
def eosToLFN
Definition: eostools.py:65
def rootfiles_to_eos_script
Definition: cmsBatch.py:79

Variable Documentation

tuple cmsBatch.batchManager = MyBatchManager()

Definition at line 198 of file cmsBatch.py.

tuple cmsBatch.cfgFile = open(batchManager.outputDir_+'/base_cfg.py','w')

Definition at line 319 of file cmsBatch.py.

list cmsBatch.cfgFileName = args[1]

Definition at line 271 of file cmsBatch.py.

tuple cmsBatch.cfo = imp.load_source("pycfg", cfgFileName, handle)

Definition at line 285 of file cmsBatch.py.

string cmsBatch.default = "cmsRun"

Definition at line 240 of file cmsBatch.py.

cmsBatch.doCVSTag = options.tagPackages

Definition at line 255 of file cmsBatch.py.

tuple cmsBatch.file = open('cmsBatch.txt', 'w')

Definition at line 201 of file cmsBatch.py.

tuple cmsBatch.fullSource = process.source.clone()

Definition at line 294 of file cmsBatch.py.

cmsBatch.generator = False

Definition at line 295 of file cmsBatch.py.

tuple cmsBatch.grouping = int(args[0])

Definition at line 269 of file cmsBatch.py.

tuple cmsBatch.handle = open(cfgFileName, 'r')

Definition at line 284 of file cmsBatch.py.

string cmsBatch.help = "program to run on your cfg file"

batchManager.parser_.add_option("-b", "--batch", dest="batch", help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.", default="bsub -q 8nh < .batchScript.sh")

Definition at line 239 of file cmsBatch.py.

list cmsBatch.listOfValues = [i+1 for i in range( nJobs )]

Definition at line 302 of file cmsBatch.py.

tuple cmsBatch.log = logger( logDir )

Definition at line 341 of file cmsBatch.py.

string cmsBatch.logDir = 'Logger'

Definition at line 339 of file cmsBatch.py.

tuple cmsBatch.nFiles = len(process.source.fileNames)

Definition at line 306 of file cmsBatch.py.

Referenced by SiStripSpyMonitorModule.analyze(), FFTJetPileupProcessor.mixExtraGrid(), SiStripSpyMonitorModule.SiStripSpyMonitorModule(), and SiStripSpyMonitorModule.~SiStripSpyMonitorModule().

int cmsBatch.nJobs = grouping

Definition at line 270 of file cmsBatch.py.

tuple cmsBatch.oldPwd = os.getcwd()

Definition at line 337 of file cmsBatch.py.

cmsBatch.process = cfo.process

Definition at line 286 of file cmsBatch.py.

cmsBatch.prog = options.prog

Definition at line 254 of file cmsBatch.py.

cmsBatch.pycfg_params = options.cmdargs

Definition at line 275 of file cmsBatch.py.

tuple cmsBatch.runningMode = None

Definition at line 262 of file cmsBatch.py.

cmsBatch.trueArgv = sys.argv

Definition at line 276 of file cmsBatch.py.

int cmsBatch.waitingTime = 1

Definition at line 325 of file cmsBatch.py.