CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
batchmanager.BatchManager Class Reference

Public Member Functions

def __init__
 
def CheckBatchScript
 
def DefineOptions
 
def ManageOutputDir
 
def mkdir
 
def ParseOptions
 
def PrepareJob
 
def PrepareJobs
 
def PrepareJobUser
 
def RunningMode
 
def SubmitJob
 
def SubmitJobs
 

Public Attributes

 listOfJobs_
 
 outputDir_
 
 parser_
 
 remoteOutputDir_
 
 remoteOutputFile_
 

Detailed Description

This class manages batch jobs
Used in batch scripts
Colin Bernet 2008

Definition at line 13 of file batchmanager.py.

Constructor & Destructor Documentation

def batchmanager.BatchManager.__init__ (   self)

Definition at line 23 of file batchmanager.py.

References batchmanager.BatchManager.DefineOptions().

23 
24  def __init__(self):
25  self.DefineOptions()
26 

Member Function Documentation

def batchmanager.BatchManager.CheckBatchScript (   self,
  batchScript 
)

Definition at line 190 of file batchmanager.py.

References if().

190 
191  def CheckBatchScript( self, batchScript ):
192 
193  if batchScript == '':
194  return
195 
196  if( os.path.isfile(batchScript)== False ):
197  print 'file ',batchScript,' does not exist'
198  sys.exit(3)
199 
200  try:
201  ifile = open(batchScript)
202  except:
203  print 'cannot open input %s' % batchScript
204  sys.exit(3)
205  else:
206  for line in ifile:
207  p = re.compile("\s*cp.*\$jobdir\s+(\S+)$");
208  m=p.match(line)
209  if m:
210  if os.path.isdir( os.path.expandvars(m.group(1)) ):
211  print 'output directory ', m.group(1), 'already exists!'
212  print 'exiting'
213  sys.exit(2)
214  else:
215  if self.options_.negate==False:
216  os.mkdir( os.path.expandvars(m.group(1)) )
217  else:
218  print 'not making dir', self.options_.negate
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
def batchmanager.BatchManager.DefineOptions (   self)

Definition at line 27 of file batchmanager.py.

Referenced by batchmanager.BatchManager.__init__().

27 
28  def DefineOptions(self):
29  # define options and arguments ====================================
30  # how to add more doc to the help?
31  self.parser_ = OptionParser()
32  self.parser_.add_option("-o", "--output-dir", dest="outputDir",
33  help="Name of the local output directory for your jobs. This directory will be created automatically.",
34  default=None)
35  self.parser_.add_option("-r", "--remote-copy", dest="remoteCopy",
36  help="remote output directory for your jobs. Example: /store/cmst3/user/cbern/CMG/HT/Run2011A-PromptReco-v1/AOD/PAT_CMG/RA2. This directory *must* be provided as a logical file name (LFN). When this option is used, all root files produced by a job are copied to the remote directory, and the job index is appended to the root file name. The Logger directory is tarred and compressed into Logger.tgz, and sent to the remote output directory as well. Afterwards, use logger.py to access the information contained in Logger.tgz. For remote copy to PSI specify path like: '/pnfs/psi.ch/...'. Logs will be sent back to the submision directory.",
37  default=None)
38  self.parser_.add_option("-f", "--force", action="store_true",
39  dest="force", default=False,
40  help="Don't ask any questions, just over-write")
41  # this opt can be removed
42  self.parser_.add_option("-n", "--negate", action="store_true",
43  dest="negate", default=False,
44  help="create jobs, but does not submit the jobs.")
45  self.parser_.add_option("-b", "--batch", dest="batch",
46  help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.",
47  default="bsub -q 8nh < ./batchScript.sh")
48 
def batchmanager.BatchManager.ManageOutputDir (   self)

Definition at line 116 of file batchmanager.py.

116 
117  def ManageOutputDir( self ):
118 
119  #if the output dir is not specified, generate a name
120  #else
121  #test if the directory exists
122  #if yes, returns
123 
124  outputDir = self.options_.outputDir
125 
126  if outputDir==None:
127  today = datetime.today()
128  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M%S")
129  print 'output directory not specified, using %s' % outputDir
130 
131  self.outputDir_ = os.path.abspath(outputDir)
132 
133  if( os.path.isdir(self.outputDir_) == True ):
134  input = ''
135  if not self.options_.force:
136  while input != 'y' and input != 'n':
137  input = raw_input( 'The directory ' + self.outputDir_ + ' exists. Are you sure you want to continue? its contents will be overwritten [y/n]' )
138  if input == 'n':
139  sys.exit(1)
140  else:
141  os.system( 'rm -rf ' + self.outputDir_)
142 
143  self.mkdir( self.outputDir_ )
144 
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
def batchmanager.BatchManager.mkdir (   self,
  dirname 
)

Definition at line 220 of file batchmanager.py.

References if().

Referenced by batchmanager.BatchManager.PrepareJob().

220 
221  def mkdir( self, dirname ):
222  # there is probably a command for this in python
223  mkdir = 'mkdir -p %s' % dirname
224  ret = os.system( mkdir )
225  if( ret != 0 ):
226  print 'please remove or rename directory: ', dirname
227  sys.exit(4)
228 
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
def batchmanager.BatchManager.ParseOptions (   self)

Definition at line 49 of file batchmanager.py.

References reco::parser::MethodInvoker.args_, reco::parser::ExpressionQuaterOperator< Op >.args_, ExternalLHEProducer.args_, Json::Path.args_, pftools::CalibCompare.options_, and pftools::Exercises3.options_.

49 
50  def ParseOptions(self):
51  (self.options_,self.args_) = self.parser_.parse_args()
52  if self.options_.remoteCopy == None:
53  self.remoteOutputDir_ = ""
54  else:
55  # removing possible trailing slash
56  import CMGTools.Production.eostools as castortools
57  self.remoteOutputDir_ = self.options_.remoteCopy.rstrip('/')
58 
59  if "psi.ch" in self.remoteOutputDir_: # T3 @ PSI:
60  # overwriting protection to be improved
61  if self.remoteOutputDir_.startswith("/pnfs/psi.ch"):
62  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
63  outputDir = self.options_.outputDir
64  if outputDir==None:
65  today = datetime.today()
66  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M")
67  self.remoteOutputDir_+="/"+outputDir
68  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
69  else:
70  print "remote directory must start with /pnfs/psi.ch to send to the tier3 at PSI"
71  print self.remoteOutputDir_, "not valid"
72  sys.exit(1)
73  else: # assume EOS
74  if not castortools.isLFN( self.remoteOutputDir_ ):
75  print 'When providing an output directory, you must give its LFN, starting by /store. You gave:'
76  print self.remoteOutputDir_
77  sys.exit(1)
78  self.remoteOutputDir_ = castortools.lfnToEOS( self.remoteOutputDir_ )
79  dirExist = castortools.isDirectory( self.remoteOutputDir_ )
80  # nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
81  # dirExist = os.system( nsls )
82  if dirExist is False:
83  print 'creating ', self.remoteOutputDir_
84  if castortools.isEOSFile( self.remoteOutputDir_ ):
85  # the output directory is currently a file..
86  # need to remove it.
87  castortools.rm( self.remoteOutputDir_ )
88  castortools.createEOSDir( self.remoteOutputDir_ )
89  else:
90  # directory exists.
91  if self.options_.negate is False and self.options_.force is False:
92  #COLIN need to reimplement protectedRemove in eostools
93  raise ValueError( ' '.join(['directory ', self.remoteOutputDir_, ' already exists.']))
94  # if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
95  # the user does not want to delete the root files
96  self.remoteOutputFile_ = ""
97  self.ManageOutputDir()
98  return (self.options_, self.args_)
99 
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def batchmanager.BatchManager.PrepareJob (   self,
  value,
  dirname = None 
) 
Prepare a job for a given value.

calls PrepareJobUser, which should be overloaded by the user.

Definition at line 145 of file batchmanager.py.

References join(), TFileService.mkdir(), TFileDirectory.mkdir(), batchmanager.BatchManager.mkdir(), L1GtVhdlWriter.outputDir_, batchmanager.BatchManager.outputDir_, L1GtVhdlWriterCore.outputDir_, LaserSorter.outputDir_, and batchmanager.BatchManager.PrepareJobUser().

145 
146  def PrepareJob( self, value, dirname=None):
147  '''Prepare a job for a given value.
148 
149  calls PrepareJobUser, which should be overloaded by the user.
150  '''
151  print 'PrepareJob : %s' % value
152  dname = dirname
153  if dname is None:
154  dname = 'Job_{value}'.format( value=value )
155  jobDir = '/'.join( [self.outputDir_, dname])
156  print '\t',jobDir
157  self.mkdir( jobDir )
158  self.listOfJobs_.append( jobDir )
159  self.PrepareJobUser( jobDir, value )
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def batchmanager.BatchManager.PrepareJobs (   self,
  listOfValues,
  listOfDirNames = None 
)

Definition at line 100 of file batchmanager.py.

100 
101  def PrepareJobs(self, listOfValues, listOfDirNames=None):
102  print 'PREPARING JOBS ======== '
103  self.listOfJobs_ = []
104 
105  if listOfDirNames is None:
106  for value in listOfValues:
107  self.PrepareJob( value )
108  else:
109  for value, name in zip( listOfValues, listOfDirNames):
110  self.PrepareJob( value, name )
111  print "list of jobs:"
112  pp = pprint.PrettyPrinter(indent=4)
113  pp.pprint( self.listOfJobs_)
114 
archive.zip
tuple zip
Definition: archive.py:476
def batchmanager.BatchManager.PrepareJobUser (   self,
  value 
) 
Hook allowing user to define how one of his jobs should be prepared.

Definition at line 160 of file batchmanager.py.

Referenced by batchmanager.BatchManager.PrepareJob().

160 
161  def PrepareJobUser(self, value ):
162  '''Hook allowing user to define how one of his jobs should be prepared.'''
163  print '\to be customized'
164 
def batchmanager.BatchManager.RunningMode (   self,
  batch 
) 
Returns "LXPLUS", "PSI", "LOCAL", or None,

"LXPLUS" : batch command is bsub, and logged on lxplus
"PSI"    : batch command is qsub, and logged to t3uiXX
"IC"     : batch command is qsub, and logged to hep.ph.ic.ac.uk
"LOCAL"  : batch command is nohup.
In all other cases, a CmsBatchException is raised

Definition at line 229 of file batchmanager.py.

Referenced by heppy_batch.MyBatchManager.PrepareJobUser().

229 
230  def RunningMode(self, batch):
231  '''Returns "LXPLUS", "PSI", "LOCAL", or None,
232 
233  "LXPLUS" : batch command is bsub, and logged on lxplus
234  "PSI" : batch command is qsub, and logged to t3uiXX
235  "IC" : batch command is qsub, and logged to hep.ph.ic.ac.uk
236  "LOCAL" : batch command is nohup.
237  In all other cases, a CmsBatchException is raised
238  '''
239 
240  hostName = os.environ['HOSTNAME']
241  onLxplus = hostName.startswith('lxplus')
242  onPSI = hostName.startswith('t3ui' )
243  onPISA = re.match('.*gridui.*',hostName) or re.match('.*faiwn.*',hostName)
244  onIC = 'hep.ph.ic.ac.uk' in hostName
245  batchCmd = batch.split()[0]
246 
247  if batchCmd == 'bsub':
248  if not (onLxplus or onPISA) :
249  err = 'Cannot run %s on %s' % (batchCmd, hostName)
250  raise ValueError( err )
251  elif onPISA :
252  print 'running on LSF pisa : %s from %s' % (batchCmd, hostName)
253  return 'PISA'
254  else:
255  print 'running on LSF lxplus: %s from %s' % (batchCmd, hostName)
256  return 'LXPLUS'
257  elif batchCmd == "qsub":
258  #if not onPSI:
259  # err = 'Cannot run %s on %s' % (batchCmd, hostName)
260  # raise ValueError( err )
261 
262  if onIC:
263  print 'running on IC : %s from %s' % (batchCmd, hostName)
264  return 'IC'
265 
266  else:
267  if onPSI:
268  print 'running on SGE : %s from %s' % (batchCmd, hostName)
269  return 'PSI'
270 
271  elif batchCmd == 'nohup' or batchCmd == './batchScript.sh':
272  print 'running locally : %s on %s' % (batchCmd, hostName)
273  return 'LOCAL'
274  else:
275  err = 'unknown batch command: X%sX' % batchCmd
276  raise ValueError( err )
def batchmanager.BatchManager.SubmitJob (   self,
  jobDir 
) 
Hook for job submission.

Definition at line 184 of file batchmanager.py.

Referenced by batchmanager.BatchManager.SubmitJobs().

184 
185  def SubmitJob( self, jobDir ):
186  '''Hook for job submission.'''
187  print 'submitting (to be customized): ', jobDir
188  os.system( self.options_.batch )
189 
def batchmanager.BatchManager.SubmitJobs (   self,
  waitingTimeInSec = 0 
) 
Submit all jobs. Possibly wait between each job

Definition at line 165 of file batchmanager.py.

References if(), batchmanager.BatchManager.listOfJobs_, and batchmanager.BatchManager.SubmitJob().

165 
166  def SubmitJobs( self, waitingTimeInSec=0 ):
167  '''Submit all jobs. Possibly wait between each job'''
168 
169  if(self.options_.negate):
170  print '*NOT* SUBMITTING JOBS - exit '
171  return
172  print 'SUBMITTING JOBS ======== '
173  for jobDir in self.listOfJobs_:
174  root = os.getcwd()
175  # run it
176  print 'processing ', jobDir
177  os.chdir( jobDir )
178  self.SubmitJob( jobDir )
179  # and come back
180  os.chdir(root)
181  print 'waiting %s seconds...' % waitingTimeInSec
182  time.sleep( waitingTimeInSec )
183  print 'done.'
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58

Member Data Documentation

batchmanager.BatchManager.listOfJobs_

Definition at line 102 of file batchmanager.py.

Referenced by batchmanager.BatchManager.SubmitJobs().

batchmanager.BatchManager.outputDir_

Definition at line 130 of file batchmanager.py.

Referenced by batchmanager.BatchManager.PrepareJob(), and valtools.webpage.readCaptions().

batchmanager.BatchManager.parser_

Definition at line 30 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputDir_

Definition at line 52 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputFile_

Definition at line 95 of file batchmanager.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5