heppy_batch Namespace Reference

Classes
class	MyBatchManager

Functions
def	batchScriptCERN
def	batchScriptIC
def	batchScriptLocal
def	batchScriptPADOVA
def	batchScriptPISA
def	batchScriptPSI

Variables
tuple	batchManager = MyBatchManager()
list	cfgFileName = args[0]
tuple	cfo = imp.load_source("pycfg", cfgFileName, handle)
tuple	components = split( [comp for comp in config.components if len(comp.files)>0] )
	config = cfo.config
tuple	handle = open(cfgFileName, 'r')
list	listOfNames = [comp.name for comp in components]
tuple	listOfValues = list(range(0, len(components)))
float	waitingTime = 0.1

Function Documentation

def heppy_batch.batchScriptCERN	(		jobDir,
			remoteDir = ''
	)

prepare the LSF version of the batch script, to run on LSF

Definition at line 78 of file heppy_batch.py.

References print(), python.rootplot.root2matplotlib.replace(), and digitizers_cfi.strip.

Referenced by heppy_batch.MyBatchManager.PrepareJobUser().

def batchScriptCERN( jobDir, remoteDir=''):
    '''prepare the LSF version of the batch script, to run on LSF'''

    dirCopy = """echo 'sending the logs back'  # will send also root files if copy failed
rm Loop/cmsswPreProcessing.root
cp -r Loop/* $LS_SUBCWD
if [ $? -ne 0 ]; then
   echo 'ERROR: problem copying job directory back'
else
   echo 'job directory copy succeeded'
fi"""

    if remoteDir=='':
        cpCmd=dirCopy
    elif  remoteDir.startswith("root://eoscms.cern.ch//eos/cms/store/"):
        cpCmd="""echo 'sending root files to remote dir'
export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH # 
for f in Loop/*/tree*.root
do
   rm Loop/cmsswPreProcessing.root
   ff=`echo $f | cut -d/ -f2`
   ff="${{ff}}_`basename $f | cut -d . -f 1`"
   echo $f
   echo $ff
   export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch
   source $VO_CMS_SW_DIR/cmsset_default.sh
   for try in `seq 1 3`; do
      echo "Stageout try $try"
      echo "/afs/cern.ch/project/eos/installation/pro/bin/eos.select mkdir {srm}"
      /afs/cern.ch/project/eos/installation/pro/bin/eos.select mkdir {srm}
      echo "/afs/cern.ch/project/eos/installation/pro/bin/eos.select cp `pwd`/$f {srm}/${{ff}}_{idx}.root"
      /afs/cern.ch/project/eos/installation/pro/bin/eos.select cp `pwd`/$f {srm}/${{ff}}_{idx}.root
      if [ $? -ne 0 ]; then
         echo "ERROR: remote copy failed for file $ff"
         continue
      fi
      echo "remote copy succeeded"
      remsize=$(/afs/cern.ch/project/eos/installation/pro/bin/eos.select find --size {srm}/${{ff}}_{idx}.root | cut -d= -f3) 
      locsize=$(cat `pwd`/$f | wc -c)
      ok=$(($remsize==$locsize))
      if [ $ok -ne 1 ]; then
         echo "Problem with copy (file sizes don't match), will retry in 30s"
         sleep 30
         continue
      fi
      echo "everything ok"
      rm $f
      echo root://eoscms.cern.ch/{srm}/${{ff}}_{idx}.root > $f.url
      break
   done
done
cp -r Loop/* $LS_SUBCWD
if [ $? -ne 0 ]; then
   echo 'ERROR: problem copying job directory back'
else
   echo 'job directory copy succeeded'
fi
""".format(
           idx = jobDir[jobDir.find("_Chunk")+6:].strip("/") if '_Chunk' in jobDir else 'all',
           srm = (""+remoteDir+jobDir[ jobDir.rfind("/") : (jobDir.find("_Chunk") if '_Chunk' in jobDir else len(jobDir)) ]).replace("root://eoscms.cern.ch/","")
           )
    else:
        print("chosen location not supported yet: ", remoteDir)
        print('path must start with /store/')
        sys.exit(1)

    script = """#!/bin/bash
#BSUB -q 8nm
echo 'environment:'
echo
env | sort
# ulimit -v 3000000 # NO
echo 'copying job dir to worker'
cd $CMSSW_BASE/src
eval `scramv1 ru -sh`
# cd $LS_SUBCWD
# eval `scramv1 ru -sh`
cd -
cp -rf $LS_SUBCWD .
ls
cd `find . -type d | grep /`
echo 'running'
python $CMSSW_BASE/src/PhysicsTools/HeppyCore/python/framework/looper.py pycfg.py config.pck --options=options.json
echo
{copy}
""".format(copy=cpCmd)

    return script