CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes
batchmanager.BatchManager Class Reference

Public Member Functions

def __init__
 
def CheckBatchScript
 
def DefineOptions
 
def ManageOutputDir
 
def mkdir
 
def ParseOptions
 
def PrepareJob
 
def PrepareJobs
 
def PrepareJobUser
 
def RunningMode
 
def SubmitJob
 
def SubmitJobArray
 
def SubmitJobs
 

Public Attributes

 listOfJobs_
 
 outputDir_
 
 parser_
 
 remoteOutputDir_
 
 remoteOutputFile_
 

Detailed Description

This class manages batch jobs
Used in batch scripts
Colin Bernet 2008

Definition at line 16 of file batchmanager.py.

Constructor & Destructor Documentation

def batchmanager.BatchManager.__init__ (   self)

Definition at line 26 of file batchmanager.py.

References batchmanager.BatchManager.DefineOptions().

26 
27  def __init__(self):
28  self.DefineOptions()
29 

Member Function Documentation

def batchmanager.BatchManager.CheckBatchScript (   self,
  batchScript 
)

Definition at line 206 of file batchmanager.py.

References if(), and print().

206 
207  def CheckBatchScript( self, batchScript ):
208 
209  if batchScript == '':
210  return
211 
212  if( os.path.isfile(batchScript)== False ):
213  print('file ',batchScript,' does not exist')
214  sys.exit(3)
215 
216  try:
217  ifile = open(batchScript)
218  except:
219  print('cannot open input %s' % batchScript)
220  sys.exit(3)
221  else:
222  for line in ifile:
223  p = re.compile("\s*cp.*\$jobdir\s+(\S+)$");
224  m=p.match(line)
225  if m:
226  if os.path.isdir( os.path.expandvars(m.group(1)) ):
227  print('output directory ', m.group(1), 'already exists!')
228  print('exiting')
229  sys.exit(2)
230  else:
231  if self.options_.negate==False:
232  os.mkdir( os.path.expandvars(m.group(1)) )
233  else:
234  print('not making dir', self.options_.negate)
if
if(conf_.getParameter< bool >("UseStripCablingDB"))
Definition: SiStripLAProfileBooker.cc:72
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.DefineOptions (   self)

Definition at line 30 of file batchmanager.py.

Referenced by batchmanager.BatchManager.__init__().

30 
31  def DefineOptions(self):
32  # define options and arguments ====================================
33  # how to add more doc to the help?
34  self.parser_ = OptionParser()
35  self.parser_.add_option("-o", "--output-dir", dest="outputDir",
36  help="Name of the local output directory for your jobs. This directory will be created automatically.",
37  default=None)
38  self.parser_.add_option("-r", "--remote-copy", dest="remoteCopy",
39  help="remote output directory for your jobs. Example: /store/cmst3/user/cbern/CMG/HT/Run2011A-PromptReco-v1/AOD/PAT_CMG/RA2. This directory *must* be provided as a logical file name (LFN). When this option is used, all root files produced by a job are copied to the remote directory, and the job index is appended to the root file name. The Logger directory will be sent back to the submision directory. For remote copy to PSI specify path like: '/pnfs/psi.ch/...'. Note: enviromental variable X509_USER_PROXY must point to home area before renewing proxy",
40  default=None)
41  self.parser_.add_option("-f", "--force", action="store_true",
42  dest="force", default=False,
43  help="Don't ask any questions, just over-write")
44  # this opt can be removed
45  self.parser_.add_option("-n", "--negate", action="store_true",
46  dest="negate", default=False,
47  help="create jobs, but does not submit the jobs.")
48  self.parser_.add_option("-b", "--batch", dest="batch",
49  help="batch command. default is: 'bsub -q 8nh < batchScript.sh'. You can also use 'nohup < ./batchScript.sh &' to run locally.",
50  default="bsub -q 8nh < ./batchScript.sh")
51  self.parser_.add_option( "--option",
52  dest="extraOptions",
53  type="string",
54  action="append",
55  default=[],
56  help="Save one extra option (either a flag, or a key=value pair) that can be then accessed from the job config file")
def batchmanager.BatchManager.ManageOutputDir (   self)

Definition at line 128 of file batchmanager.py.

References print().

128 
129  def ManageOutputDir( self ):
130 
131  #if the output dir is not specified, generate a name
132  #else
133  #test if the directory exists
134  #if yes, returns
135 
136  outputDir = self.options_.outputDir
137 
138  if outputDir==None:
139  today = datetime.today()
140  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M%S")
141  print('output directory not specified, using %s' % outputDir)
142 
143  self.outputDir_ = os.path.abspath(outputDir)
144 
145  if( os.path.isdir(self.outputDir_) == True ):
146  input = ''
147  if not self.options_.force:
148  while input != 'y' and input != 'n':
149  input = raw_input( 'The directory ' + self.outputDir_ + ' exists. Are you sure you want to continue? its contents will be overwritten [y/n] ' )
150  if input == 'n':
151  sys.exit(1)
152  else:
153  os.system( 'rm -rf ' + self.outputDir_)
154 
155  self.mkdir( self.outputDir_ )
156 
if
if(conf_.getParameter< bool >("UseStripCablingDB"))
Definition: SiStripLAProfileBooker.cc:72
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.mkdir (   self,
  dirname 
)

Definition at line 236 of file batchmanager.py.

References if(), and print().

Referenced by batchmanager.BatchManager.PrepareJob().

236 
237  def mkdir( self, dirname ):
238  # there is probably a command for this in python
239  mkdir = 'mkdir -p %s' % dirname
240  ret = os.system( mkdir )
241  if( ret != 0 ):
242  print('please remove or rename directory: ', dirname)
243  sys.exit(4)
244 
if
if(conf_.getParameter< bool >("UseStripCablingDB"))
Definition: SiStripLAProfileBooker.cc:72
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.ParseOptions (   self)

Definition at line 57 of file batchmanager.py.

References reco::parser::MethodInvoker.args_, reco::parser::ExpressionQuaterOperator< Op >.args_, LazyConstructed< WrappedClass, Args >.args_, LazyResult< Func, Args >.args_, ExternalLHEProducer.args_, cmdline::CmdLine.args_, Json::Path.args_, and TritonClient.options_.

57 
58  def ParseOptions(self):
59  (self.options_,self.args_) = self.parser_.parse_args()
60  if self.options_.remoteCopy == None:
61  self.remoteOutputDir_ = ""
62  else:
63  # removing possible trailing slash
64  self.remoteOutputDir_ = self.options_.remoteCopy.rstrip('/')
65  if "psi.ch" in self.remoteOutputDir_: # T3 @ PSI:
66  # overwriting protection to be improved
67  if self.remoteOutputDir_.startswith("/pnfs/psi.ch"):
68  ld_lib_path = os.environ.get('LD_LIBRARY_PATH')
69  if ld_lib_path != "None":
70  os.environ['LD_LIBRARY_PATH'] = "/usr/lib64/:"+ld_lib_path # to solve gfal conflict with CMSSW
71  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
72  outputDir = self.options_.outputDir
73  if outputDir==None:
74  today = datetime.today()
75  outputDir = 'OutCmsBatch_%s' % today.strftime("%d%h%y_%H%M")
76  self.remoteOutputDir_+="/"+outputDir
77  os.system("gfal-mkdir srm://t3se01.psi.ch/"+self.remoteOutputDir_)
78  if ld_lib_path != "None":
79  os.environ['LD_LIBRARY_PATH'] = ld_lib_path # back to original to avoid conflicts
80  else:
81  print("remote directory must start with /pnfs/psi.ch to send to the tier3 at PSI")
82  print(self.remoteOutputDir_, "not valid")
83  sys.exit(1)
84  else: # assume EOS
85  if not castortools.isLFN( self.remoteOutputDir_ ):
86  print('When providing an output directory, you must give its LFN, starting by /store. You gave:')
87  print(self.remoteOutputDir_)
88  sys.exit(1)
89  self.remoteOutputDir_ = castortools.lfnToEOS( self.remoteOutputDir_ )
90  dirExist = castortools.isDirectory( self.remoteOutputDir_ )
91  # nsls = 'nsls %s > /dev/null' % self.remoteOutputDir_
92  # dirExist = os.system( nsls )
93  if dirExist is False:
94  print('creating ', self.remoteOutputDir_)
95  if castortools.isEOSFile( self.remoteOutputDir_ ):
96  # the output directory is currently a file..
97  # need to remove it.
98  castortools.rm( self.remoteOutputDir_ )
99  castortools.createEOSDir( self.remoteOutputDir_ )
100  else:
101  # directory exists.
102  if self.options_.negate is False and self.options_.force is False:
103  #COLIN need to reimplement protectedRemove in eostools
104  raise ValueError( ' '.join(['directory ', self.remoteOutputDir_, ' already exists.']))
105  # if not castortools.protectedRemove( self.remoteOutputDir_, '.*root'):
106  # the user does not want to delete the root files
107 
108  self.remoteOutputFile_ = ""
109  self.ManageOutputDir()
110  return (self.options_, self.args_)
111 
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def batchmanager.BatchManager.PrepareJob (   self,
  value,
  dirname = None 
) 
Prepare a job for a given value.

calls PrepareJobUser, which should be overloaded by the user.

Definition at line 157 of file batchmanager.py.

References join(), TFileDirectory.mkdir(), TFileService.mkdir(), batchHippy.MyBatchManager.mkdir(), batchmanager.BatchManager.mkdir(), dqmoffline::l1t::L1TDiffHarvesting::L1TDiffPlotHandler.outputDir_, MillePedeFileExtractor.outputDir_, L1GtVhdlWriter.outputDir_, dqmoffline::l1t::L1TEfficiencyPlotHandler.outputDir_, batchmanager.BatchManager.outputDir_, L1GtVhdlWriterCore.outputDir_, LaserSorter.outputDir_, cmsBatch.MyBatchManager.PrepareJobUser(), batchmanager.BatchManager.PrepareJobUser(), and print().

157 
158  def PrepareJob( self, value, dirname=None):
159  '''Prepare a job for a given value.
160 
161  calls PrepareJobUser, which should be overloaded by the user.
162  '''
163  print('PrepareJob : %s' % value)
164  dname = dirname
165  if dname is None:
166  dname = 'Job_{value}'.format( value=value )
167  jobDir = '/'.join( [self.outputDir_, dname])
168  print('\t',jobDir)
169  self.mkdir( jobDir )
170  self.listOfJobs_.append( jobDir )
171  self.PrepareJobUser( jobDir, value )
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def batchmanager.BatchManager.PrepareJobs (   self,
  listOfValues,
  listOfDirNames = None 
)

Definition at line 112 of file batchmanager.py.

References print().

112 
113  def PrepareJobs(self, listOfValues, listOfDirNames=None):
114  print('PREPARING JOBS ======== ')
115  self.listOfJobs_ = []
116 
117  if listOfDirNames is None:
118  for value in listOfValues:
119  self.PrepareJob( value )
120  else:
121  for value, name in zip( listOfValues, listOfDirNames):
122  self.PrepareJob( value, name )
123  print("list of jobs:")
124  pp = pprint.PrettyPrinter(indent=4)
125  pp.pprint( self.listOfJobs_)
126 
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:41
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.PrepareJobUser (   self,
  value 
) 
Hook allowing user to define how one of his jobs should be prepared.

Definition at line 172 of file batchmanager.py.

References print().

Referenced by batchmanager.BatchManager.PrepareJob().

172 
173  def PrepareJobUser(self, value ):
174  '''Hook allowing user to define how one of his jobs should be prepared.'''
175  print('\to be customized')
176 
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.RunningMode (   self,
  batch 
) 
Return "LXPUS", "PSI", "NAF", "LOCAL", or None,

"LXPLUS" : batch command is bsub, and logged on lxplus
"PSI"    : batch command is qsub, and logged to t3uiXX
"NAF"    : batch command is qsub, and logged on naf
"IC"     : batch command is qsub, and logged on hep.ph.ic.ac.uk
"LOCAL"  : batch command is nohup.

In all other cases, a CmsBatchException is raised

Definition at line 245 of file batchmanager.py.

References print().

Referenced by cmsBatch.MyBatchManager.PrepareJobUser(), and heppy_batch.MyBatchManager.PrepareJobUser().

245 
246  def RunningMode(self, batch):
247 
248  '''Return "LXPUS", "PSI", "NAF", "LOCAL", or None,
249 
250  "LXPLUS" : batch command is bsub, and logged on lxplus
251  "PSI" : batch command is qsub, and logged to t3uiXX
252  "NAF" : batch command is qsub, and logged on naf
253  "IC" : batch command is qsub, and logged on hep.ph.ic.ac.uk
254  "LOCAL" : batch command is nohup.
255 
256  In all other cases, a CmsBatchException is raised
257  '''
258 
259  hostName = os.environ['HOSTNAME']
260 
261  onLxplus = hostName.startswith('lxplus')
262  onPSI = hostName.startswith('t3ui')
263  onNAF = hostName.startswith('naf')
264 
265  batchCmd = batch.split()[0]
266 
267  if batchCmd == 'bsub':
268  if not onLxplus:
269  err = 'Cannot run %s on %s' % (batchCmd, hostName)
270  raise ValueError( err )
271  else:
272  print('running on LSF : %s from %s' % (batchCmd, hostName))
273  return 'LXPLUS'
274 
275  elif batchCmd == "qsub":
276  if onPSI:
277  print('running on SGE : %s from %s' % (batchCmd, hostName))
278  return 'PSI'
279  elif onNAF:
280  print('running on NAF : %s from %s' % (batchCmd, hostName))
281  return 'NAF'
282  elif onIC:
283  print('running on IC : %s from %s' % (batchCmd, hostName))
284  return 'IC'
285  else:
286  err = 'Cannot run %s on %s' % (batchCmd, hostName)
287  raise ValueError( err )
288 
289  elif batchCmd == 'nohup' or batchCmd == './batchScript.sh':
290  print('running locally : %s on %s' % (batchCmd, hostName))
291  return 'LOCAL'
292  else:
293  err = 'unknown batch command: X%sX' % batchCmd
294  raise ValueError( err )
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.SubmitJob (   self,
  jobDir 
) 
Hook for job submission.

Definition at line 196 of file batchmanager.py.

References print().

Referenced by batchmanager.BatchManager.SubmitJobs().

196 
197  def SubmitJob( self, jobDir ):
198  '''Hook for job submission.'''
199  print('submitting (to be customized): ', jobDir)
200  os.system( self.options_.batch )
201 
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.SubmitJobArray (   self,
  numbOfJobs = 1 
) 
Hook for array job submission.

Definition at line 202 of file batchmanager.py.

References print().

202 
203  def SubmitJobArray( self, numbOfJobs = 1 ):
204  '''Hook for array job submission.'''
205  print('Submitting array with %s jobs' % numbOfJobs)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def batchmanager.BatchManager.SubmitJobs (   self,
  waitingTimeInSec = 0 
) 
Submit all jobs. Possibly wait between each job

Definition at line 177 of file batchmanager.py.

References if(), batchmanager.BatchManager.listOfJobs_, print(), and batchmanager.BatchManager.SubmitJob().

177 
178  def SubmitJobs( self, waitingTimeInSec=0 ):
179  '''Submit all jobs. Possibly wait between each job'''
180 
181  if(self.options_.negate):
182  print('*NOT* SUBMITTING JOBS - exit ')
183  return
184  print('SUBMITTING JOBS ======== ')
185  for jobDir in self.listOfJobs_:
186  root = os.getcwd()
187  # run it
188  print('processing ', jobDir)
189  os.chdir( jobDir )
190  self.SubmitJob( jobDir )
191  # and come back
192  os.chdir(root)
193  print('waiting %s seconds...' % waitingTimeInSec)
194  time.sleep( waitingTimeInSec )
195  print('done.')
if
if(conf_.getParameter< bool >("UseStripCablingDB"))
Definition: SiStripLAProfileBooker.cc:72
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

Member Data Documentation

batchmanager.BatchManager.listOfJobs_

Definition at line 114 of file batchmanager.py.

Referenced by batchmanager.BatchManager.SubmitJobs().

batchmanager.BatchManager.outputDir_

Definition at line 142 of file batchmanager.py.

Referenced by batchmanager.BatchManager.PrepareJob().

batchmanager.BatchManager.parser_

Definition at line 33 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputDir_

Definition at line 60 of file batchmanager.py.

batchmanager.BatchManager.remoteOutputFile_

Definition at line 107 of file batchmanager.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5