CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
heppy_batch Namespace Reference

Classes

class  MyBatchManager
 

Functions

def batchScriptCERN
 
def batchScriptLocal
 
def batchScriptPISA
 
def batchScriptPSI
 

Variables

tuple batchManager = MyBatchManager()
 
list cfgFileName = args[0]
 
tuple cfo = imp.load_source("pycfg", cfgFileName, handle)
 
tuple components = split( [comp for comp in config.components if len(comp.files)>0] )
 
 config = cfo.config
 
tuple handle = open(cfgFileName, 'r')
 
list listOfNames = [comp.name for comp in components]
 
tuple listOfValues = range(0, len(components))
 
float waitingTime = 0.1
 

Function Documentation

def heppy_batch.batchScriptCERN (   index,
  remoteDir = '' 
) 
prepare the LSF version of the batch script, to run on LSF

Definition at line 48 of file heppy_batch.py.

Referenced by heppy_batch.MyBatchManager.PrepareJobUser().

48 
49 def batchScriptCERN( index, remoteDir=''):
50  '''prepare the LSF version of the batch script, to run on LSF'''
51  script = """#!/bin/bash
52 #BSUB -q 8nm
53 echo 'environment:'
54 echo
55 env
56 # ulimit -v 3000000 # NO
57 echo 'copying job dir to worker'
58 cd $CMSSW_BASE/src
59 eval `scramv1 ru -sh`
60 # cd $LS_SUBCWD
61 # eval `scramv1 ru -sh`
62 cd -
63 cp -rf $LS_SUBCWD .
64 ls
65 cd `find . -type d | grep /`
66 echo 'running'
67 python $CMSSW_BASE/src/PhysicsTools/HeppyCore/python/framework/looper.py pycfg.py config.pck
68 echo
69 echo 'sending the job directory back'
70 cp -r Loop/* $LS_SUBCWD
71 """
72  return script
73 
heppy_batch.batchScriptCERN
def batchScriptCERN
Definition: heppy_batch.py:48
def heppy_batch.batchScriptLocal (   remoteDir,
  index 
) 
prepare a local version of the batch script, to run using nohup

Definition at line 166 of file heppy_batch.py.

Referenced by heppy_batch.MyBatchManager.PrepareJobUser().

166 
167 def batchScriptLocal( remoteDir, index ):
168  '''prepare a local version of the batch script, to run using nohup'''
169 
170  script = """#!/bin/bash
171 echo 'running'
172 python $CMSSW_BASE/src/PhysicsTools/HeppyCore/python/framework/looper.py pycfg.py config.pck echo
173 echo 'sending the job directory back'
174 mv Loop/* ./
175 """
176  return script
177 
178 
heppy_batch.batchScriptLocal
def batchScriptLocal
Definition: heppy_batch.py:166
def heppy_batch.batchScriptPISA (   index,
  remoteDir = '' 
) 
prepare the LSF version of the batch script, to run on LSF

Definition at line 15 of file heppy_batch.py.

Referenced by heppy_batch.MyBatchManager.PrepareJobUser().

15 
16 def batchScriptPISA( index, remoteDir=''):
17  '''prepare the LSF version of the batch script, to run on LSF'''
18  script = """#!/bin/bash
19 #BSUB -q cms
20 echo 'PWD:'
21 pwd
22 export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch
23 source $VO_CMS_SW_DIR/cmsset_default.sh
24 echo 'environment:'
25 echo
26 env > local.env
27 env
28 # ulimit -v 3000000 # NO
29 echo 'copying job dir to worker'
30 ###cd $CMSSW_BASE/src
31 eval `scramv1 runtime -sh`
32 #eval `scramv1 ru -sh`
33 # cd $LS_SUBCWD
34 # eval `scramv1 ru -sh`
35 ##cd -
36 ##cp -rf $LS_SUBCWD .
37 ls
38 echo `find . -type d | grep /`
39 echo 'running'
40 python $CMSSW_BASE/src/PhysicsTools/HeppyCore/python/framework/looper.py pycfg.py config.pck >& local.output
41 exit $?
42 #echo
43 #echo 'sending the job directory back'
44 #echo cp -r Loop/* $LS_SUBCWD
45 """
46  return script
47 
heppy_batch.batchScriptPISA
def batchScriptPISA
Definition: heppy_batch.py:15
def heppy_batch.batchScriptPSI (   index,
  jobDir,
  remoteDir = '' 
) 
prepare the SGE version of the batch script, to run on the PSI tier3 batch system

Definition at line 74 of file heppy_batch.py.

74 
75 def batchScriptPSI( index, jobDir, remoteDir=''):
76  '''prepare the SGE version of the batch script, to run on the PSI tier3 batch system'''
77 
78  cmssw_release = os.environ['CMSSW_BASE']
79  VO_CMS_SW_DIR = "/swshare/cms" # $VO_CMS_SW_DIR doesn't seem to work in the new SL6 t3wn
80 
81  if remoteDir=='':
82  cpCmd="""echo 'sending the job directory back'
83 cp -r Loop/* $SUBMISIONDIR"""
84  elif remoteDir.startswith("/pnfs/psi.ch"):
85  cpCmd="""echo 'sending root files to remote dir'
86 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/dcap/ # Fabio's workaround to fix gfal-tools
87 for f in Loop/treeProducerSusyFullHad/*.root
88 do
89 echo $f
90 ff=`basename $f | cut -d . -f 1`
91 echo $ff
92 gfal-mkdir {srm}
93 echo "gfal-copy file:///`pwd`/Loop/treeProducerSusyFullHad/$file.root {srm}/${{ff}}_{idx}.root"
94 gfal-copy file:///`pwd`/Loop/treeProducerSusyFullHad/$ff.root {srm}/${{ff}}_{idx}.root
95 done
96 rm Loop/treeProducerSusyFullHad/*.root
97 echo 'sending the logs back'
98 cp -r Loop/* $SUBMISIONDIR""".format(idx=index, srm='srm://t3se01.psi.ch'+remoteDir+jobDir[jobDir.rfind("/"):jobDir.find("_Chunk")])
99  else:
100  print "remote directory not supported yet: ", remoteDir
101  print 'path must start with "/pnfs/psi.ch"'
102  sys.exit(1)
103 
104 
105  script = """#!/bin/bash
106 shopt expand_aliases
107 ##### MONITORING/DEBUG INFORMATION ###############################
108 DATE_START=`date +%s`
109 echo "Job started at " `date`
110 cat <<EOF
111 ################################################################
112 ## QUEUEING SYSTEM SETTINGS:
113 HOME=$HOME
114 USER=$USER
115 JOB_ID=$JOB_ID
116 JOB_NAME=$JOB_NAME
117 HOSTNAME=$HOSTNAME
118 TASK_ID=$TASK_ID
119 QUEUE=$QUEUE
120 
121 EOF
122 echo "######## Environment Variables ##########"
123 env
124 echo "################################################################"
125 TOPWORKDIR=/scratch/`whoami`
126 JOBDIR=sgejob-$JOB_ID
127 WORKDIR=$TOPWORKDIR/$JOBDIR
128 SUBMISIONDIR={jdir}
129 if test -e "$WORKDIR"; then
130  echo "ERROR: WORKDIR ($WORKDIR) already exists! Aborting..." >&2
131  exit 1
132 fi
133 mkdir -p $WORKDIR
134 if test ! -d "$WORKDIR"; then
135  echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
136  exit 1
137 fi
138 
139 #source $VO_CMS_SW_DIR/cmsset_default.sh
140 source {vo}/cmsset_default.sh
141 export SCRAM_ARCH=slc6_amd64_gcc481
142 #cd $CMSSW_BASE/src
143 cd {cmssw}/src
144 shopt -s expand_aliases
145 cmsenv
146 cd $WORKDIR
147 cp -rf $SUBMISIONDIR .
148 ls
149 cd `find . -type d | grep /`
150 echo 'running'
151 #python $CMSSW_BASE/src/CMGTools/RootTools/python/fwlite/looper.py config.pck
152 python {cmssw}/src/CMGTools/RootTools/python/fwlite/looper.py pycfg.py config.pck
153 echo
154 {copy}
155 ###########################################################################
156 DATE_END=`date +%s`
157 RUNTIME=$((DATE_END-DATE_START))
158 echo "################################################################"
159 echo "Job finished at " `date`
160 echo "Wallclock running time: $RUNTIME s"
161 exit 0
162 """.format(jdir=jobDir, vo=VO_CMS_SW_DIR,cmssw=cmssw_release, copy=cpCmd)
163 
164  return script
165 
heppy_batch.batchScriptPSI
def batchScriptPSI
Definition: heppy_batch.py:74

Variable Documentation

tuple heppy_batch.batchManager = MyBatchManager()

Definition at line 213 of file heppy_batch.py.

list heppy_batch.cfgFileName = args[0]

Definition at line 223 of file heppy_batch.py.

tuple heppy_batch.cfo = imp.load_source("pycfg", cfgFileName, handle)

Definition at line 226 of file heppy_batch.py.

tuple heppy_batch.components = split( [comp for comp in config.components if len(comp.files)>0] )

Definition at line 230 of file heppy_batch.py.

heppy_batch.config = cfo.config

Definition at line 227 of file heppy_batch.py.

tuple heppy_batch.handle = open(cfgFileName, 'r')

Definition at line 225 of file heppy_batch.py.

list heppy_batch.listOfNames = [comp.name for comp in components]

Definition at line 232 of file heppy_batch.py.

tuple heppy_batch.listOfValues = range(0, len(components))

Definition at line 231 of file heppy_batch.py.

float heppy_batch.waitingTime = 0.1

Definition at line 235 of file heppy_batch.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5