CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
batchmanager.BatchManager Class Reference

Public Member Functions

def __init__
 
def CheckBatchScript
 
def DefineOptions
 
def ManageOutputDir
 
def mkdir
 
def ParseOptions
 
def PrepareJob
 
def PrepareJobs
 
def PrepareJobUser
 
def RunningMode
 
def SubmitJob
 
def SubmitJobArray
 
def SubmitJobs
 

Public Attributes

 listOfJobs_
 
 outputDir_
 
 parser_
 
 remoteOutputDir_
 
 remoteOutputFile_
 

Detailed Description

This class manages batch jobs
Used in batch scripts
Colin Bernet 2008

Definition at line 14 of file batchmanager.py.

Constructor & Destructor Documentation

def batchmanager.BatchManager.__init__ (   self)

Definition at line 24 of file batchmanager.py.

References batchmanager.BatchManager.DefineOptions().

24 
25  def __init__(self):
26  self.DefineOptions()
27 

Member Function Documentation

def batchmanager.BatchManager.CheckBatchScript (   self,
  batchScript 
)

Definition at line 204 of file batchmanager.py.

References citk.if().

204 
205  def CheckBatchScript( self, batchScript ):
206 
207  if batchScript == '':
208  return
209 
210  if( os.path.isfile(batchScript)== False ):
211  print 'file ',batchScript,' does not exist'
212  sys.exit(3)
213 
214  try:
215  ifile = open(batchScript)
216  except:
217  print 'cannot open input %s' % batchScript
218  sys.exit(3)
219  else:
220  for line in ifile:
221  p = re.compile("\s*cp.*\$jobdir\s+(\S+)$");
222  m=p.match(line)
223  if m:
224  if os.path.isdir( os.path.expandvars(m.group(1)) ):
225  print 'output directory ', m.group(1), 'already exists!'
226  print 'exiting'
227  sys.exit(2)
228  else:
229  if self.options_.negate==False:
230  os.mkdir( os.path.expandvars(m.group(1)) )
231  else:
232  print 'not making dir', self.options_.negate
citk::if
if(c.getParameter< edm::InputTag >("puppiValueMap").label().size()!=0)
Definition: CITKPFIsolationSumProducerForPUPPI.cc:72
def batchmanager.BatchManager.DefineOptions (   self)

Definition at line 28 of file batchmanager.py.

Referenced by batchmanager.BatchManager.__init__().

28 
29  def DefineOptions(self):
30  # define options and arguments ====================================
31  # how to add more doc to the help?
32  self.parser_ = OptionParser()
33  self.parser_.add_option("-o", "--output-dir", dest="outputDir",
34  help="Name of the local output directory for your jobs. This directory will be created automatically.",
35  default=None)
36  self.parser_.add_option("-r", "--remote-copy", dest="remoteCopy",
37  help="remote output directory for your jobs. Example: /store/cmst3/user/cbern/CMG/HT/Run2011A-PromptReco-v1/AOD/PAT_CMG/RA2. This directory *must* be provided as a logical file name (LFN). When this option is used, all root files produced by a job are copied to the remote directory, and the job index is appended to the root file name. The Logger directory will be sent back to the submision directory. For remote copy to PSI specify path like: '/pnfs/psi.ch/...'. Note: enviromental variable X509_USER_PROXY must point to home area before renewing proxy",
38  default=None)
39  self.parser_.add_option("-f", "--force", action="store_true",
40  dest="force", default=False,
41  help="Don't ask any questions, just over-write")
42  # this opt can be removed
43  self.parser_.add_option("-n", "--negate", action="store_true",
44  dest="negate", default=False,
45  help="create jobs, but does not submit the jobs.")
46  self.parser_.add_option("-b", "--batch", dest="batch",
47  help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.",
48  default="bsub -q 8nh < ./batchScript.sh")
49  self.parser_.add_option( "--option",
50  dest="extraOptions",
51  type="string",
52  action="append",
53  default=[],
54  help="Save one extra option (either a flag, or a key=value pair) that can be then accessed from the job config file")
def batchmanager.BatchManager.ManageOutputDir (   self)

Definition at line 126 of file batchmanager.py.

126 
127  def ManageOutputDir( self ):
128 
129  #if the output dir is not specified, generate a name
130  #else
131  #test if the directory exists
132  #if yes, returns
133 
134  outputDir = self.options_.outputDir
135 
136  if outputDir==None:
137  today = datetime.today()
138  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M%S")
139  print 'output directory not specified, using %s' % outputDir
140 
141  self.outputDir_ = os.path.abspath(outputDir)
142 
143  if( os.path.isdir(self.outputDir_) == True ):
144  input = ''
145  if not self.options_.force:
146  while input != 'y' and input != 'n':
147  input = raw_input( 'The directory ' + self.outputDir_ + ' exists. Are you sure you want to continue? its contents will be overwritten [y/n] ' )
148  if input == 'n':
149  sys.exit(1)
150  else:
151  os.system( 'rm -rf ' + self.outputDir_)
152 
153  self.mkdir( self.outputDir_ )
154 
citk::if
if(c.getParameter< edm::InputTag >("puppiValueMap").label().size()!=0)
Definition: CITKPFIsolationSumProducerForPUPPI.cc:72
def batchmanager.BatchManager.mkdir (   self,
  dirname 
)

Definition at line 234 of file batchmanager.py.

References citk.if().

Referenced by batchmanager.BatchManager.PrepareJob().

234 
235  def mkdir( self, dirname ):
236  # there is probably a command for this in python
237  mkdir = 'mkdir -p %s' % dirname
238  ret = os.system( mkdir )
239  if( ret != 0 ):
240  print 'please remove or rename directory: ', dirname
241  sys.exit(4)
242 
citk::if
if(c.getParameter< edm::InputTag >("puppiValueMap").label().size()!=0)
Definition: CITKPFIsolationSumProducerForPUPPI.cc:72
def batchmanager.BatchManager.ParseOptions (   self)

Definition at line 55 of file batchmanager.py.

References reco::parser::ExpressionQuaterOperator< Op >.args_, reco::parser::MethodInvoker.args_, ExternalLHEProducer.args_, Json::Path.args_, pftools::CalibCompare.options_, and pftools::Exercises3.options_.

55 
56  def ParseOptions(self):
57  (self.options_,self.args_) = self.parser_.parse_args()
58  if self.options_.remoteCopy == None:
59  self.remoteOutputDir_ = ""
60  else:
61  # removing possible trailing slash
62  self.remoteOutputDir_ = self.options_.remoteCopy.rstrip('/')
63  if "psi.ch" in self.remoteOutputDir_: # T3 @ PSI:
64  # overwriting protection to be improved
65  if self.remoteOutputDir_.startswith("/pnfs/psi.ch"):
66  ld_lib_path = os.environ.get('LD_LIBRARY_PATH')
67  if ld_lib_path != "None":
68  os.environ['LD_LIBRARY_PATH'] = "/usr/lib64/:"+ld_lib_path # to solve gfal conflict with CMSSW
69  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
70  outputDir = self.options_.outputDir
71  if outputDir==None:
72  today = datetime.today()
73  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M")
74  self.remoteOutputDir_+="/"+outputDir
75  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
76  if ld_lib_path != "None":
77  os.environ['LD_LIBRARY_PATH'] = ld_lib_path # back to original to avoid conflicts
78  else:
79  print "remote directory must start with /pnfs/psi.ch to send to the tier3 at PSI"
80  print self.remoteOutputDir_, "not valid"
81  sys.exit(1)
82  else: # assume EOS
83  if not castortools.isLFN( self.remoteOutputDir_ ):
84  print 'When providing an output directory, you must give its LFN, starting by /store. You gave:'
85  print self.remoteOutputDir_
86  sys.exit(1)
87  self.remoteOutputDir_ = castortools.lfnToEOS( self.remoteOutputDir_ )
88  dirExist = castortools.isDirectory( self.remoteOutputDir_ )
89  # nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
90  # dirExist = os.system( nsls )
91  if dirExist is False:
92  print 'creating ', self.remoteOutputDir_
93  if castortools.isEOSFile( self.remoteOutputDir_ ):
94  # the output directory is currently a file..
95  # need to remove it.
96  castortools.rm( self.remoteOutputDir_ )
97  castortools.createEOSDir( self.remoteOutputDir_ )
98  else:
99  # directory exists.
100  if self.options_.negate is False and self.options_.force is False:
101  #COLIN need to reimplement protectedRemove in eostools
102  raise ValueError( ' '.join(['directory ', self.remoteOutputDir_, ' already exists.']))
103  # if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
104  # the user does not want to delete the root files
105 
106  self.remoteOutputFile_ = ""
107  self.ManageOutputDir()
108  return (self.options_, self.args_)
109 
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def batchmanager.BatchManager.PrepareJob (   self,
  value,
  dirname = None 
) 
Prepare a job for a given value.

calls PrepareJobUser, which should be overloaded by the user.

Definition at line 155 of file batchmanager.py.

References join(), TFileService.mkdir(), TFileDirectory.mkdir(), batchmanager.BatchManager.mkdir(), L1GtVhdlWriter.outputDir_, batchmanager.BatchManager.outputDir_, L1GtVhdlWriterCore.outputDir_, LaserSorter.outputDir_, cmsBatch.MyBatchManager.PrepareJobUser(), and batchmanager.BatchManager.PrepareJobUser().

155 
156  def PrepareJob( self, value, dirname=None):
157  '''Prepare a job for a given value.
158 
159  calls PrepareJobUser, which should be overloaded by the user.
160  '''
161  print 'PrepareJob : %s' % value
162  dname = dirname
163  if dname is None:
164  dname = 'Job_{value}'.format( value=value )
165  jobDir = '/'.join( [self.outputDir_, dname])
166  print '\t',jobDir
167  self.mkdir( jobDir )
168  self.listOfJobs_.append( jobDir )
169  self.PrepareJobUser( jobDir, value )
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def batchmanager.BatchManager.PrepareJobs (   self,
  listOfValues,
  listOfDirNames = None 
)

Definition at line 110 of file batchmanager.py.

110 
111  def PrepareJobs(self, listOfValues, listOfDirNames=None):
112  print 'PREPARING JOBS ======== '
113  self.listOfJobs_ = []
114 
115  if listOfDirNames is None:
116  for value in listOfValues:
117  self.PrepareJob( value )
118  else:
119  for value, name in zip( listOfValues, listOfDirNames):
120  self.PrepareJob( value, name )
121  print "list of jobs:"
122  pp = pprint.PrettyPrinter(indent=4)
123  pp.pprint( self.listOfJobs_)
124 
def batchmanager.BatchManager.PrepareJobUser (   self,
  value 
) 
Hook allowing user to define how one of his jobs should be prepared.

Definition at line 170 of file batchmanager.py.

Referenced by batchmanager.BatchManager.PrepareJob().

170 
171  def PrepareJobUser(self, value ):
172  '''Hook allowing user to define how one of his jobs should be prepared.'''
173  print '\to be customized'
174 
def batchmanager.BatchManager.RunningMode (   self,
  batch 
) 
Return "LXPUS", "PSI", "NAF", "LOCAL", or None,

"LXPLUS" : batch command is bsub, and logged on lxplus
"PSI"    : batch command is qsub, and logged to t3uiXX
"NAF"    : batch command is qsub, and logged on naf
"IC"     : batch command is qsub, and logged on hep.ph.ic.ac.uk
"LOCAL"  : batch command is nohup.

In all other cases, a CmsBatchException is raised

Definition at line 243 of file batchmanager.py.

Referenced by cmsBatch.MyBatchManager.PrepareJobUser(), and heppy_batch.MyBatchManager.PrepareJobUser().

243 
244  def RunningMode(self, batch):
245 
246  '''Return "LXPUS", "PSI", "NAF", "LOCAL", or None,
247 
248  "LXPLUS" : batch command is bsub, and logged on lxplus
249  "PSI" : batch command is qsub, and logged to t3uiXX
250  "NAF" : batch command is qsub, and logged on naf
251  "IC" : batch command is qsub, and logged on hep.ph.ic.ac.uk
252  "LOCAL" : batch command is nohup.
253 
254  In all other cases, a CmsBatchException is raised
255  '''
256 
257  hostName = os.environ['HOSTNAME']
258 
259  onLxplus = hostName.startswith('lxplus')
260  onPSI = hostName.startswith('t3ui')
261  onNAF = hostName.startswith('naf')
262 
263  batchCmd = batch.split()[0]
264 
265  if batchCmd == 'bsub':
266  if not onLxplus:
267  err = 'Cannot run %s on %s' % (batchCmd, hostName)
268  raise ValueError( err )
269  else:
270  print 'running on LSF : %s from %s' % (batchCmd, hostName)
271  return 'LXPLUS'
272 
273  elif batchCmd == "qsub":
274  if onPSI:
275  print 'running on SGE : %s from %s' % (batchCmd, hostName)
276  return 'PSI'
277  elif onNAF:
278  print 'running on NAF : %s from %s' % (batchCmd, hostName)
279  return 'NAF'
280  elif onIC:
281  print 'running on IC : %s from %s' % (batchCmd, hostName)
282  return 'IC'
283  else:
284  err = 'Cannot run %s on %s' % (batchCmd, hostName)
285  raise ValueError( err )
286 
287  elif batchCmd == 'nohup' or batchCmd == './batchScript.sh':
288  print 'running locally : %s on %s' % (batchCmd, hostName)
289  return 'LOCAL'
290  else:
291  err = 'unknown batch command: X%sX' % batchCmd
292  raise ValueError( err )
def batchmanager.BatchManager.SubmitJob (   self,
  jobDir 
) 
Hook for job submission.

Definition at line 194 of file batchmanager.py.

Referenced by batchmanager.BatchManager.SubmitJobs().

194 
195  def SubmitJob( self, jobDir ):
196  '''Hook for job submission.'''
197  print 'submitting (to be customized): ', jobDir
198  os.system( self.options_.batch )
199 
def batchmanager.BatchManager.SubmitJobArray (   self,
  numbOfJobs = 1 
) 
Hook for array job submission.

Definition at line 200 of file batchmanager.py.

200 
201  def SubmitJobArray( self, numbOfJobs = 1 ):
202  '''Hook for array job submission.'''
203  print 'Submitting array with %s jobs' % numbOfJobs
def batchmanager.BatchManager.SubmitJobs (   self,
  waitingTimeInSec = 0 
) 
Submit all jobs. Possibly wait between each job

Definition at line 175 of file batchmanager.py.

References citk.if(), batchmanager.BatchManager.listOfJobs_, and batchmanager.BatchManager.SubmitJob().

175 
176  def SubmitJobs( self, waitingTimeInSec=0 ):
177  '''Submit all jobs. Possibly wait between each job'''
178 
179  if(self.options_.negate):
180  print '*NOT* SUBMITTING JOBS - exit '
181  return
182  print 'SUBMITTING JOBS ======== '
183  for jobDir in self.listOfJobs_:
184  root = os.getcwd()
185  # run it
186  print 'processing ', jobDir
187  os.chdir( jobDir )
188  self.SubmitJob( jobDir )
189  # and come back
190  os.chdir(root)
191  print 'waiting %s seconds...' % waitingTimeInSec
192  time.sleep( waitingTimeInSec )
193  print 'done.'
citk::if
if(c.getParameter< edm::InputTag >("puppiValueMap").label().size()!=0)
Definition: CITKPFIsolationSumProducerForPUPPI.cc:72

Member Data Documentation

batchmanager.BatchManager.listOfJobs_

Definition at line 112 of file batchmanager.py.

Referenced by batchmanager.BatchManager.SubmitJobs().

batchmanager.BatchManager.outputDir_

Definition at line 140 of file batchmanager.py.

Referenced by batchmanager.BatchManager.PrepareJob(), and valtools.webpage.readCaptions().

batchmanager.BatchManager.parser_

Definition at line 31 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputDir_

Definition at line 58 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputFile_

Definition at line 105 of file batchmanager.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5