CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
batchmanager.BatchManager Class Reference

Public Member Functions

def __init__
 
def CheckBatchScript
 
def DefineOptions
 
def ManageOutputDir
 
def mkdir
 
def ParseOptions
 
def PrepareJob
 
def PrepareJobs
 
def PrepareJobUser
 
def RunningMode
 
def SubmitJob
 
def SubmitJobArray
 
def SubmitJobs
 

Public Attributes

 listOfJobs_
 
 outputDir_
 
 parser_
 
 remoteOutputDir_
 
 remoteOutputFile_
 

Detailed Description

This class manages batch jobs
Used in batch scripts
Colin Bernet 2008

Definition at line 14 of file batchmanager.py.

Constructor & Destructor Documentation

def batchmanager.BatchManager.__init__ (   self)

Definition at line 24 of file batchmanager.py.

References batchmanager.BatchManager.DefineOptions().

24 
25  def __init__(self):
26  self.DefineOptions()
27 

Member Function Documentation

def batchmanager.BatchManager.CheckBatchScript (   self,
  batchScript 
)

Definition at line 198 of file batchmanager.py.

References if().

198 
199  def CheckBatchScript( self, batchScript ):
200 
201  if batchScript == '':
202  return
203 
204  if( os.path.isfile(batchScript)== False ):
205  print 'file ',batchScript,' does not exist'
206  sys.exit(3)
207 
208  try:
209  ifile = open(batchScript)
210  except:
211  print 'cannot open input %s' % batchScript
212  sys.exit(3)
213  else:
214  for line in ifile:
215  p = re.compile("\s*cp.*\$jobdir\s+(\S+)$");
216  m=p.match(line)
217  if m:
218  if os.path.isdir( os.path.expandvars(m.group(1)) ):
219  print 'output directory ', m.group(1), 'already exists!'
220  print 'exiting'
221  sys.exit(2)
222  else:
223  if self.options_.negate==False:
224  os.mkdir( os.path.expandvars(m.group(1)) )
225  else:
226  print 'not making dir', self.options_.negate
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
def batchmanager.BatchManager.DefineOptions (   self)

Definition at line 28 of file batchmanager.py.

Referenced by batchmanager.BatchManager.__init__().

28 
29  def DefineOptions(self):
30  # define options and arguments ====================================
31  # how to add more doc to the help?
32  self.parser_ = OptionParser()
33  self.parser_.add_option("-o", "--output-dir", dest="outputDir",
34  help="Name of the local output directory for your jobs. This directory will be created automatically.",
35  default=None)
36  self.parser_.add_option("-r", "--remote-copy", dest="remoteCopy",
37  help="remote output directory for your jobs. Example: /store/cmst3/user/cbern/CMG/HT/Run2011A-PromptReco-v1/AOD/PAT_CMG/RA2. This directory *must* be provided as a logical file name (LFN). When this option is used, all root files produced by a job are copied to the remote directory, and the job index is appended to the root file name. The Logger directory will be sent back to the submision directory. For remote copy to PSI specify path like: '/pnfs/psi.ch/...'. Note: enviromental variable X509_USER_PROXY must point to home area before renewing proxy",
38  default=None)
39  self.parser_.add_option("-f", "--force", action="store_true",
40  dest="force", default=False,
41  help="Don't ask any questions, just over-write")
42  # this opt can be removed
43  self.parser_.add_option("-n", "--negate", action="store_true",
44  dest="negate", default=False,
45  help="create jobs, but does not submit the jobs.")
46  self.parser_.add_option("-b", "--batch", dest="batch",
47  help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.",
48  default="bsub -q 8nh < ./batchScript.sh")
def batchmanager.BatchManager.ManageOutputDir (   self)

Definition at line 120 of file batchmanager.py.

120 
121  def ManageOutputDir( self ):
122 
123  #if the output dir is not specified, generate a name
124  #else
125  #test if the directory exists
126  #if yes, returns
127 
128  outputDir = self.options_.outputDir
129 
130  if outputDir==None:
131  today = datetime.today()
132  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M%S")
133  print 'output directory not specified, using %s' % outputDir
134 
135  self.outputDir_ = os.path.abspath(outputDir)
136 
137  if( os.path.isdir(self.outputDir_) == True ):
138  input = ''
139  if not self.options_.force:
140  while input != 'y' and input != 'n':
141  input = raw_input( 'The directory ' + self.outputDir_ + ' exists. Are you sure you want to continue? its contents will be overwritten [y/n] ' )
142  if input == 'n':
143  sys.exit(1)
144  else:
145  os.system( 'rm -rf ' + self.outputDir_)
146 
147  self.mkdir( self.outputDir_ )
148 
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
def batchmanager.BatchManager.mkdir (   self,
  dirname 
)

Definition at line 228 of file batchmanager.py.

References if().

Referenced by batchmanager.BatchManager.PrepareJob().

228 
229  def mkdir( self, dirname ):
230  # there is probably a command for this in python
231  mkdir = 'mkdir -p %s' % dirname
232  ret = os.system( mkdir )
233  if( ret != 0 ):
234  print 'please remove or rename directory: ', dirname
235  sys.exit(4)
236 
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
def batchmanager.BatchManager.ParseOptions (   self)

Definition at line 49 of file batchmanager.py.

References reco::parser::ExpressionQuaterOperator< Op >.args_, reco::parser::MethodInvoker.args_, ExternalLHEProducer.args_, Json::Path.args_, pftools::CalibCompare.options_, and pftools::Exercises3.options_.

49 
50  def ParseOptions(self):
51  (self.options_,self.args_) = self.parser_.parse_args()
52  if self.options_.remoteCopy == None:
53  self.remoteOutputDir_ = ""
54  else:
55  # removing possible trailing slash
56  self.remoteOutputDir_ = self.options_.remoteCopy.rstrip('/')
57  if "psi.ch" in self.remoteOutputDir_: # T3 @ PSI:
58  # overwriting protection to be improved
59  if self.remoteOutputDir_.startswith("/pnfs/psi.ch"):
60  ld_lib_path = os.environ.get('LD_LIBRARY_PATH')
61  if ld_lib_path != "None":
62  os.environ['LD_LIBRARY_PATH'] = "/usr/lib64/:"+ld_lib_path # to solve gfal conflict with CMSSW
63  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
64  outputDir = self.options_.outputDir
65  if outputDir==None:
66  today = datetime.today()
67  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M")
68  self.remoteOutputDir_+="/"+outputDir
69  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
70  if ld_lib_path != "None":
71  os.environ['LD_LIBRARY_PATH'] = ld_lib_path # back to original to avoid conflicts
72  else:
73  print "remote directory must start with /pnfs/psi.ch to send to the tier3 at PSI"
74  print self.remoteOutputDir_, "not valid"
75  sys.exit(1)
76  else: # assume EOS
77  if not castortools.isLFN( self.remoteOutputDir_ ):
78  print 'When providing an output directory, you must give its LFN, starting by /store. You gave:'
79  print self.remoteOutputDir_
80  sys.exit(1)
81  self.remoteOutputDir_ = castortools.lfnToEOS( self.remoteOutputDir_ )
82  dirExist = castortools.isDirectory( self.remoteOutputDir_ )
83  # nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
84  # dirExist = os.system( nsls )
85  if dirExist is False:
86  print 'creating ', self.remoteOutputDir_
87  if castortools.isEOSFile( self.remoteOutputDir_ ):
88  # the output directory is currently a file..
89  # need to remove it.
90  castortools.rm( self.remoteOutputDir_ )
91  castortools.createEOSDir( self.remoteOutputDir_ )
92  else:
93  # directory exists.
94  if self.options_.negate is False and self.options_.force is False:
95  #COLIN need to reimplement protectedRemove in eostools
96  raise ValueError( ' '.join(['directory ', self.remoteOutputDir_, ' already exists.']))
97  # if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
98  # the user does not want to delete the root files
99 
100  self.remoteOutputFile_ = ""
101  self.ManageOutputDir()
102  return (self.options_, self.args_)
103 
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def batchmanager.BatchManager.PrepareJob (   self,
  value,
  dirname = None 
) 
Prepare a job for a given value.

calls PrepareJobUser, which should be overloaded by the user.

Definition at line 149 of file batchmanager.py.

References join(), TFileService.mkdir(), TFileDirectory.mkdir(), batchmanager.BatchManager.mkdir(), L1GtVhdlWriter.outputDir_, batchmanager.BatchManager.outputDir_, L1GtVhdlWriterCore.outputDir_, LaserSorter.outputDir_, batchmanager.BatchManager.PrepareJobUser(), and cmsBatch.MyBatchManager.PrepareJobUser().

149 
150  def PrepareJob( self, value, dirname=None):
151  '''Prepare a job for a given value.
152 
153  calls PrepareJobUser, which should be overloaded by the user.
154  '''
155  print 'PrepareJob : %s' % value
156  dname = dirname
157  if dname is None:
158  dname = 'Job_{value}'.format( value=value )
159  jobDir = '/'.join( [self.outputDir_, dname])
160  print '\t',jobDir
161  self.mkdir( jobDir )
162  self.listOfJobs_.append( jobDir )
163  self.PrepareJobUser( jobDir, value )
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def batchmanager.BatchManager.PrepareJobs (   self,
  listOfValues,
  listOfDirNames = None 
)

Definition at line 104 of file batchmanager.py.

104 
105  def PrepareJobs(self, listOfValues, listOfDirNames=None):
106  print 'PREPARING JOBS ======== '
107  self.listOfJobs_ = []
108 
109  if listOfDirNames is None:
110  for value in listOfValues:
111  self.PrepareJob( value )
112  else:
113  for value, name in zip( listOfValues, listOfDirNames):
114  self.PrepareJob( value, name )
115  print "list of jobs:"
116  pp = pprint.PrettyPrinter(indent=4)
117  pp.pprint( self.listOfJobs_)
118 
archive.zip
tuple zip
Definition: archive.py:476
def batchmanager.BatchManager.PrepareJobUser (   self,
  value 
) 
Hook allowing user to define how one of his jobs should be prepared.

Definition at line 164 of file batchmanager.py.

Referenced by batchmanager.BatchManager.PrepareJob().

164 
165  def PrepareJobUser(self, value ):
166  '''Hook allowing user to define how one of his jobs should be prepared.'''
167  print '\to be customized'
168 
def batchmanager.BatchManager.RunningMode (   self,
  batch 
) 
Return "LXPUS", "PSI", "NAF", "LOCAL", or None,

"LXPLUS" : batch command is bsub, and logged on lxplus
"PSI"    : batch command is qsub, and logged to t3uiXX
"NAF"    : batch command is qsub, and logged on naf
"IC"     : batch command is qsub, and logged on hep.ph.ic.ac.uk
"LOCAL"  : batch command is nohup.

In all other cases, a CmsBatchException is raised

Definition at line 237 of file batchmanager.py.

Referenced by cmsBatch.MyBatchManager.PrepareJobUser(), and heppy_batch.MyBatchManager.PrepareJobUser().

237 
238  def RunningMode(self, batch):
239 
240  '''Return "LXPUS", "PSI", "NAF", "LOCAL", or None,
241 
242  "LXPLUS" : batch command is bsub, and logged on lxplus
243  "PSI" : batch command is qsub, and logged to t3uiXX
244  "NAF" : batch command is qsub, and logged on naf
245  "IC" : batch command is qsub, and logged on hep.ph.ic.ac.uk
246  "LOCAL" : batch command is nohup.
247 
248  In all other cases, a CmsBatchException is raised
249  '''
250 
251  hostName = os.environ['HOSTNAME']
252 
253  onLxplus = hostName.startswith('lxplus')
254  onPSI = hostName.startswith('t3ui')
255  onNAF = hostName.startswith('naf')
256 
257  batchCmd = batch.split()[0]
258 
259  if batchCmd == 'bsub':
260  if not onLxplus:
261  err = 'Cannot run %s on %s' % (batchCmd, hostName)
262  raise ValueError( err )
263  else:
264  print 'running on LSF : %s from %s' % (batchCmd, hostName)
265  return 'LXPLUS'
266 
267  elif batchCmd == "qsub":
268  if onPSI:
269  print 'running on SGE : %s from %s' % (batchCmd, hostName)
270  return 'PSI'
271  elif onNAF:
272  print 'running on NAF : %s from %s' % (batchCmd, hostName)
273  return 'NAF'
274  elif onIC:
275  print 'running on IC : %s from %s' % (batchCmd, hostName)
276  return 'IC'
277  else:
278  err = 'Cannot run %s on %s' % (batchCmd, hostName)
279  raise ValueError( err )
280 
281  elif batchCmd == 'nohup' or batchCmd == './batchScript.sh':
282  print 'running locally : %s on %s' % (batchCmd, hostName)
283  return 'LOCAL'
284  else:
285  err = 'unknown batch command: X%sX' % batchCmd
286  raise ValueError( err )
def batchmanager.BatchManager.SubmitJob (   self,
  jobDir 
) 
Hook for job submission.

Definition at line 188 of file batchmanager.py.

Referenced by batchmanager.BatchManager.SubmitJobs().

188 
189  def SubmitJob( self, jobDir ):
190  '''Hook for job submission.'''
191  print 'submitting (to be customized): ', jobDir
192  os.system( self.options_.batch )
193 
def batchmanager.BatchManager.SubmitJobArray (   self,
  numbOfJobs = 1 
) 
Hook for array job submission.

Definition at line 194 of file batchmanager.py.

194 
195  def SubmitJobArray( self, numbOfJobs = 1 ):
196  '''Hook for array job submission.'''
197  print 'Submitting array with %s jobs' % numbOfJobs
def batchmanager.BatchManager.SubmitJobs (   self,
  waitingTimeInSec = 0 
) 
Submit all jobs. Possibly wait between each job

Definition at line 169 of file batchmanager.py.

References if(), batchmanager.BatchManager.listOfJobs_, and batchmanager.BatchManager.SubmitJob().

169 
170  def SubmitJobs( self, waitingTimeInSec=0 ):
171  '''Submit all jobs. Possibly wait between each job'''
172 
173  if(self.options_.negate):
174  print '*NOT* SUBMITTING JOBS - exit '
175  return
176  print 'SUBMITTING JOBS ======== '
177  for jobDir in self.listOfJobs_:
178  root = os.getcwd()
179  # run it
180  print 'processing ', jobDir
181  os.chdir( jobDir )
182  self.SubmitJob( jobDir )
183  # and come back
184  os.chdir(root)
185  print 'waiting %s seconds...' % waitingTimeInSec
186  time.sleep( waitingTimeInSec )
187  print 'done.'
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58

Member Data Documentation

batchmanager.BatchManager.listOfJobs_

Definition at line 106 of file batchmanager.py.

Referenced by batchmanager.BatchManager.SubmitJobs().

batchmanager.BatchManager.outputDir_

Definition at line 134 of file batchmanager.py.

Referenced by batchmanager.BatchManager.PrepareJob(), and valtools.webpage.readCaptions().

batchmanager.BatchManager.parser_

Definition at line 31 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputDir_

Definition at line 52 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputFile_

Definition at line 99 of file batchmanager.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5