CMS 3D CMS Logo

validateAlignments.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #test execute: export CMSSW_BASE=/tmp/CMSSW && ./validateAlignments.py -c defaultCRAFTValidation.ini,test.ini -n -N test
3 from __future__ import print_function
4 import os
5 import sys
6 import optparse
7 import datetime
8 import shutil
9 import fnmatch
10 
11 import six
12 import Alignment.OfflineValidation.TkAlAllInOneTool.configTemplates \
13  as configTemplates
14 import Alignment.OfflineValidation.TkAlAllInOneTool.crabWrapper as crabWrapper
15 from Alignment.OfflineValidation.TkAlAllInOneTool.TkAlExceptions \
16  import AllInOneError
17 from Alignment.OfflineValidation.TkAlAllInOneTool.helperFunctions \
18  import replaceByMap, getCommandOutput2, addIndex
19 from Alignment.OfflineValidation.TkAlAllInOneTool.betterConfigParser \
20  import BetterConfigParser
21 from Alignment.OfflineValidation.TkAlAllInOneTool.alignment import Alignment
22 
24  import GenericValidation, ParallelValidation, ValidationWithComparison, ValidationWithPlots
25 from Alignment.OfflineValidation.TkAlAllInOneTool.geometryComparison \
26  import GeometryComparison
28  import OfflineValidation, OfflineValidationDQM
29 from Alignment.OfflineValidation.TkAlAllInOneTool.monteCarloValidation \
30  import MonteCarloValidation
32  import TrackSplittingValidation
34  import ZMuMuValidation
36  import PrimaryVertexValidation
37 from Alignment.OfflineValidation.TkAlAllInOneTool.preexistingValidation \
38  import *
39 from Alignment.OfflineValidation.TkAlAllInOneTool.plottingOptions \
40  import PlottingOptions
41 import Alignment.OfflineValidation.TkAlAllInOneTool.globalDictionaries \
42  as globalDictionaries
43 
44 
45 ####################--- Classes ---############################
47 
48  def __init__(self, _name, _path, _dependency):
49  self.name=_name
50  self.path=_path
51  self.dependencies=_dependency
52  def addDependency(self,dependency):
53  if isinstance(dependency,list):
54  self.dependencies.extend(dependency)
55  else:
56  self.dependencies.append(dependency)
57  def runJob(self, config):
58  repMap = {
59  "commands": config.getGeneral()["jobmode"].split(",")[1],
60  "jobName": self.name,
61  "logDir": config.getGeneral()["logdir"],
62  "script": self.path,
63  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
64  "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in self.dependencies]) + '"'
65  }
66  return getCommandOutput2("%(bsub)s %(commands)s "
67  "-J %(jobName)s "
68  "-o %(logDir)s/%(jobName)s.stdout "
69  "-e %(logDir)s/%(jobName)s.stderr "
70  "-w %(conditions)s "
71  "%(script)s"%repMap)
72 
74 
75  # these count the jobs of different varieties that are being run
76  crabCount = 0
77  interactCount = 0
78  batchCount = 0
79  batchJobIds = []
80  jobCount = 0
81 
82  def __init__( self, validation, config, options ):
83  self.JobId=[]
84  if validation[1] == "":
85  # intermediate syntax
86  valString = validation[0].split( "->" )[0]
87  alignments = validation[0].split( "->" )[1]
88  # force user to use the normal syntax
89  if "->" in validation[0]:
90  msg = ("Instead of using the intermediate syntax\n'"
91  +valString.strip()+"-> "+alignments.strip()
92  +":'\nyou have to use the now fully supported syntax \n'"
93  +valString.strip()+": "
94  +alignments.strip()+"'.")
95  raise AllInOneError(msg)
96  else:
97  valString = validation[0]
98  alignments = validation[1]
99  valString = valString.split()
100  self.__valType = valString[0]
101  self.__valName = valString[1]
102  self.__commandLineOptions = options
103  self.__config = config
104  self.__preexisting = ("preexisting" in self.__valType)
105  if self.__valType[0] == "*":
106  self.__valType = self.__valType[1:]
107  self.__preexisting = True
108 
109  # workaround for intermediate parallel version
110  if self.__valType == "offlineParallel":
111  print ("offlineParallel and offline are now the same. To run an offline parallel validation,\n"
112  "just set parallelJobs to something > 1. There is no reason to call it offlineParallel anymore.")
113  self.__valType = "offline"
114  section = self.__valType + ":" + self.__valName
115  if not self.__config.has_section( section ):
116  raise AllInOneError("Validation '%s' of type '%s' is requested in"
117  " '[validation]' section, but is not defined."
118  "\nYou have to add a '[%s]' section."
119  %( self.__valName, self.__valType, section ))
121  alignments, self.__config,
122  options )
123 
124  def __getValidation( self, valType, name, alignments, config, options ):
125  if valType == "compare":
126  alignmentsList = alignments.split( "," )
127  firstAlignList = alignmentsList[0].split()
128  firstAlignName = firstAlignList[0].strip()
129  if firstAlignName == "IDEAL":
130  raise AllInOneError("'IDEAL' has to be the second (reference)"
131  " alignment in 'compare <val_name>: "
132  "<alignment> <reference>'.")
133  if len( firstAlignList ) > 1:
134  firstRun = firstAlignList[1]
135  else:
136  raise AllInOneError("Have to provide a run number for geometry comparison")
137  firstAlign = Alignment( firstAlignName, self.__config, firstRun )
138  firstAlignName = firstAlign.name
139  secondAlignList = alignmentsList[1].split()
140  secondAlignName = secondAlignList[0].strip()
141  if secondAlignName == "IDEAL":
142  secondAlign = secondAlignName
143  else:
144  if len( secondAlignList ) > 1:
145  secondRun = secondAlignList[1]
146  else:
147  raise AllInOneError("Have to provide a run number for geometry comparison")
148  secondAlign = Alignment( secondAlignName, self.__config,
149  secondRun )
150  secondAlignName = secondAlign.name
151 
152  validation = GeometryComparison( name, firstAlign, secondAlign,
153  self.__config,
154  self.__commandLineOptions.getImages)
155  elif valType == "offline":
156  validation = OfflineValidation( name,
157  Alignment( alignments.strip(), self.__config ), self.__config )
158  elif valType == "preexistingoffline":
159  validation = PreexistingOfflineValidation(name, self.__config)
160  elif valType == "offlineDQM":
161  validation = OfflineValidationDQM( name,
162  Alignment( alignments.strip(), self.__config ), self.__config )
163  elif valType == "mcValidate":
164  validation = MonteCarloValidation( name,
165  Alignment( alignments.strip(), self.__config ), self.__config )
166  elif valType == "preexistingmcValidate":
167  validation = PreexistingMonteCarloValidation(name, self.__config)
168  elif valType == "split":
169  validation = TrackSplittingValidation( name,
170  Alignment( alignments.strip(), self.__config ), self.__config )
171  elif valType == "preexistingsplit":
172  validation = PreexistingTrackSplittingValidation(name, self.__config)
173  elif valType == "zmumu":
174  validation = ZMuMuValidation( name,
175  Alignment( alignments.strip(), self.__config ), self.__config )
176  elif valType == "primaryvertex":
177  validation = PrimaryVertexValidation( name,
178  Alignment( alignments.strip(), self.__config ), self.__config )
179  elif valType == "preexistingprimaryvertex":
180  validation = PreexistingPrimaryVertexValidation(name, self.__config)
181  else:
182  raise AllInOneError("Unknown validation mode '%s'"%valType)
183 
184  return validation
185 
186  def __createJob( self, jobMode, outpath ):
187  """This private method creates the needed files for the validation job.
188  """
189  self.validation.createConfiguration( outpath )
190  if self.__preexisting:
191  return
192  self.__scripts = sum([addIndex(script, self.validation.NJobs) for script in self.validation.createScript( outpath )], [])
193  if jobMode.split( ',' )[0] == "crab":
194  self.validation.createCrabCfg( outpath )
195  return None
196 
197  def createJob(self):
198  """This is the method called to create the job files."""
199  self.__createJob( self.validation.jobmode,
200  os.path.abspath( self.__commandLineOptions.Name) )
201 
202  def runJob( self ):
203  if self.__preexisting:
204  if self.validation.jobid:
205  self.batchJobIds.append(self.validation.jobid)
206  log = "> " + self.validation.name + " is already validated."
207  print(log)
208  return log
209  else:
210  if self.validation.jobid:
211  print("jobid {} will be ignored, since the validation {} is not preexisting".format(self.validation.jobid, self.validation.name))
212 
213  general = self.__config.getGeneral()
214  log = ""
215  for script in self.__scripts:
216  name = os.path.splitext( os.path.basename( script) )[0]
217  ValidationJob.jobCount += 1
218  if self.__commandLineOptions.dryRun:
219  print("%s would run: %s"%( name, os.path.basename( script) ))
220  continue
221  log = "> Validating "+name
222  print("> Validating "+name)
223  if self.validation.jobmode == "interactive":
224  log += getCommandOutput2( script )
225  ValidationJob.interactCount += 1
226  elif self.validation.jobmode.split(",")[0] == "lxBatch":
227  repMap = {
228  "commands": self.validation.jobmode.split(",")[1],
229  "logDir": general["logdir"],
230  "jobName": name,
231  "script": script,
232  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
233  }
234  for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
235  oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
236  if os.path.exists(oldlog):
237  os.remove(oldlog)
238  bsubOut=getCommandOutput2("%(bsub)s %(commands)s "
239  "-J %(jobName)s "
240  "-o %(logDir)s/%(jobName)s.stdout "
241  "-e %(logDir)s/%(jobName)s.stderr "
242  "%(script)s"%repMap)
243  #Attention: here it is assumed that bsub returns a string
244  #containing a job id like <123456789>
245  jobid=bsubOut.split("<")[1].split(">")[0]
246  self.JobId.append(jobid)
247  ValidationJob.batchJobIds.append(jobid)
248  log+=bsubOut
249  ValidationJob.batchCount += 1
250  elif self.validation.jobmode.split( "," )[0] == "crab":
251  os.chdir( general["logdir"] )
252  crabName = "crab." + os.path.basename( script )[:-3]
253  theCrab = crabWrapper.CrabWrapper()
254  options = { "-create": "",
255  "-cfg": crabName + ".cfg",
256  "-submit": "" }
257  try:
258  theCrab.run( options )
259  except AllInOneError as e:
260  print("crab:", str(e).split("\n")[0])
261  exit(1)
262  ValidationJob.crabCount += 1
263 
264  else:
265  raise AllInOneError("Unknown 'jobmode'!\n"
266  "Please change this parameter either in "
267  "the [general] or in the ["
268  + self.__valType + ":" + self.__valName
269  + "] section to one of the following "
270  "values:\n"
271  "\tinteractive\n\tlxBatch, -q <queue>\n"
272  "\tcrab, -q <queue>")
273 
274  return log
275 
276  def getValidation( self ):
277  return self.validation
278 
279  @property
280  def needsproxy(self):
281  return self.validation.needsproxy and not self.__preexisting and not self.__commandLineOptions.dryRun
282 
283 
284 ####################--- Functions ---############################
285 def createMergeScript( path, validations, options ):
286  if(len(validations) == 0):
287  raise AllInOneError("Cowardly refusing to merge nothing!")
288 
289  config = validations[0].config
290  repMap = config.getGeneral()
291  repMap.update({
292  "DownloadData":"",
293  "CompareAlignments":"",
294  "RunValidationPlots":"",
295  "CMSSW_BASE": os.environ["CMSSW_BASE"],
296  "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
297  "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
298  })
299 
300  comparisonLists = {} # directory of lists containing the validations that are comparable
301  for validation in validations:
302  for referenceName in validation.filesToCompare:
303  validationtype = type(validation)
304  if issubclass(validationtype, PreexistingValidation):
305  #find the actual validationtype
306  for parentclass in validationtype.mro():
307  if not issubclass(parentclass, PreexistingValidation):
308  validationtype = parentclass
309  break
310  key = (validationtype, referenceName)
311  if key in comparisonLists:
312  comparisonLists[key].append(validation)
313  else:
314  comparisonLists[key] = [validation]
315 
316  # introduced to merge individual validation outputs separately
317  # -> avoids problems with merge script
318  repMap["doMerge"] = "mergeRetCode=0\n"
319  repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
320  " echo -e \\n\"Merging succeeded, removing original files.\"\n")
321  repMap["beforeMerge"] = ""
322  repMap["mergeParallelFilePrefixes"] = ""
323  repMap["createResultsDirectory"]=""
324 
325 
326  anythingToMerge = []
327 
328 
329  #prepare dictionary containing handle objects for parallel merge batch jobs
330  if options.mergeOfflineParallel:
331  parallelMergeObjects={}
332  for (validationType, referencename), validations in six.iteritems(comparisonLists):
333  for validation in validations:
334  #parallel merging
335  if (isinstance(validation, PreexistingValidation)
336  or validation.NJobs == 1
337  or not isinstance(validation, ParallelValidation)):
338  continue
339  if options.mergeOfflineParallel and validationType.valType=='offline' and validation.jobmode.split(",")[0]=="lxBatch":
340  repMapTemp=repMap.copy()
341  if validationType not in anythingToMerge:
342  anythingToMerge += [validationType]
343  #create init script
344  fileName="TkAlMergeInit"
345  filePath = os.path.join(path, fileName+".sh")
346  theFile = open( filePath, "w" )
347  repMapTemp["createResultsDirectory"]="#!/bin/bash"
348  repMapTemp["createResultsDirectory"]+=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMapTemp)
349  theFile.write( replaceByMap( configTemplates.createResultsDirectoryTemplate, repMapTemp ) )
350  theFile.close()
351  os.chmod(filePath,0o755)
352  #create handle
353  parallelMergeObjects["init"]=ParallelMergeJob(fileName, filePath,[])
354  #clear 'create result directory' code
355  repMapTemp["createResultsDirectory"]=""
356 
357  #edit repMapTmp as necessary:
358  #fill contents of mergeParallelResults
359  repMapTemp["beforeMerge"] += validationType.doInitMerge()
360  repMapTemp["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs with alignment %s"\n\n' % (validationType.valType,validation.alignmentToValidate.name)
361  repMapTemp["doMerge"] += validation.doMerge()
362  for f in validation.getRepMap()["outputFiles"]:
363  longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
364  validation.getRepMap()["eosdir"], f)
365  repMapTemp["rmUnmerged"] += " rm "+longName+"\n"
366 
367  repMapTemp["rmUnmerged"] += ("else\n"
368  " echo -e \\n\"WARNING: Merging failed, unmerged"
369  " files won't be deleted.\\n"
370  "(Ignore this warning if merging was done earlier)\"\n"
371  "fi\n")
372 
373  #fill mergeParallelResults area of mergeTemplate
374  repMapTemp["DownloadData"] = replaceByMap( configTemplates.mergeParallelResults, repMapTemp )
375  #fill runValidationPlots area of mergeTemplate
376  repMapTemp["RunValidationPlots"] = validationType.doRunPlots(validations)
377 
378  #create script file
379  fileName="TkAlMergeOfflineValidation"+validation.name+validation.alignmentToValidate.name
380  filePath = os.path.join(path, fileName+".sh")
381  theFile = open( filePath, "w" )
382  theFile.write( replaceByMap( configTemplates.mergeParallelOfflineTemplate, repMapTemp ) )
383  theFile.close()
384  os.chmod(filePath,0o755)
385  #create handle object
386  if "parallel" in parallelMergeObjects:
387  parallelMergeObjects["parallel"].append(ParallelMergeJob(fileName, filePath,[]))
388  else:
389  parallelMergeObjects["parallel"]=[ParallelMergeJob(fileName, filePath,[])]
390  continue
391 
392 
393  else:
394  if validationType not in anythingToMerge:
395  anythingToMerge += [validationType]
396  repMap["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType.valType
397  repMap["beforeMerge"] += validationType.doInitMerge()
398  repMap["doMerge"] += validation.doMerge()
399  for f in validation.getRepMap()["outputFiles"]:
400  longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
401  validation.getRepMap()["eosdir"], f)
402  repMap["rmUnmerged"] += " rm "+longName+"\n"
403 
404 
405 
406  repMap["rmUnmerged"] += ("else\n"
407  " echo -e \\n\"WARNING: Merging failed, unmerged"
408  " files won't be deleted.\\n"
409  "(Ignore this warning if merging was done earlier)\"\n"
410  "fi\n")
411 
412 
413 
414  if anythingToMerge:
415  repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
416  else:
417  repMap["DownloadData"] = ""
418 
419  repMap["RunValidationPlots"] = ""
420  for (validationType, referencename), validations in six.iteritems(comparisonLists):
421  if issubclass(validationType, ValidationWithPlots):
422  repMap["RunValidationPlots"] += validationType.doRunPlots(validations)
423 
424  repMap["CompareAlignments"] = "#run comparisons"
425  for (validationType, referencename), validations in six.iteritems(comparisonLists):
426  if issubclass(validationType, ValidationWithComparison):
427  repMap["CompareAlignments"] += validationType.doComparison(validations)
428 
429  #if user wants to merge parallely and if there are valid parallel scripts, create handle for plotting job and set merge script name accordingly
430  if options.mergeOfflineParallel and parallelMergeObjects!={}:
431  parallelMergeObjects["continue"]=ParallelMergeJob("TkAlMergeFinal",os.path.join(path, "TkAlMergeFinal.sh"),[])
432  filePath = os.path.join(path, "TkAlMergeFinal.sh")
433  #if not merging parallel, add code to create results directory and set merge script name accordingly
434  else:
435  repMap["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap)
436  filePath = os.path.join(path, "TkAlMerge.sh")
437 
438 
439  #filePath = os.path.join(path, "TkAlMerge.sh")
440  theFile = open( filePath, "w" )
441  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
442  theFile.close()
443  os.chmod(filePath,0o755)
444 
445  if options.mergeOfflineParallel:
446  return {'TkAlMerge.sh':filePath, 'parallelMergeObjects':parallelMergeObjects}
447  else:
448  return filePath
449 
450 def loadTemplates( config ):
451  if config.has_section("alternateTemplates"):
452  for templateName in config.options("alternateTemplates"):
453  if templateName == "AutoAlternates":
454  continue
455  newTemplateName = config.get("alternateTemplates", templateName )
456  #print "replacing default %s template by %s"%( templateName, newTemplateName)
457  configTemplates.alternateTemplate(templateName, newTemplateName)
458 
459 
460 ####################--- Main ---############################
461 def main(argv = None):
462  if argv == None:
463  argv = sys.argv[1:]
464  optParser = optparse.OptionParser()
465  optParser.description = """All-in-one Alignment Validation.
466 This will run various validation procedures either on batch queues or interactively.
467 If no name is given (-N parameter) a name containing time and date is created automatically.
468 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
469 """
470  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
471  help="create all scripts and cfg File but do not start jobs (default=False)")
472  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
473  help="get all Images created during the process (default= True)")
474  defaultConfig = "TkAlConfig.ini"
475  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
476  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
477  optParser.add_option("-N", "--Name", dest="Name",
478  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
479  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
480  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
481  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
482  help="get the status of the crab jobs", metavar="STATUS")
483  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
484  default = False,
485  help="run the tool to get full traceback of errors",
486  metavar="DEBUG")
487  optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
488  help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
489  " Works only for batch jobs")
490  optParser.add_option("--mergeOfflineParallel", dest="mergeOfflineParallel", action="store_true", default = False,
491  help="Enable parallel merging of offline data. Best used with -m option. Only works with lxBatch-jobmode", metavar="MERGE_PARALLEL")
492 
493 
494  (options, args) = optParser.parse_args(argv)
495 
496  if not options.restrictTo == None:
497  options.restrictTo = options.restrictTo.split(",")
498 
499  options.config = [ os.path.abspath( iniFile ) for iniFile in \
500  options.config.split( "," ) ]
501  config = BetterConfigParser()
502  outputIniFileSet = set( config.read( options.config ) )
503  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
504 
505  # Check for missing ini file
506  if options.config == [ os.path.abspath( defaultConfig ) ]:
507  if ( not options.crabStatus ) and \
508  ( not os.path.exists( defaultConfig ) ):
509  raise AllInOneError( "Default 'ini' file '%s' not found!\n"
510  "You can specify another name with the "
511  "command line option '-c'/'--config'."
512  %( defaultConfig ))
513  else:
514  for iniFile in failedIniFiles:
515  if not os.path.exists( iniFile ):
516  raise AllInOneError( "'%s' does not exist. Please check for "
517  "typos in the filename passed to the "
518  "'-c'/'--config' option!"
519  %( iniFile ))
520  else:
521  raise AllInOneError(( "'%s' does exist, but parsing of the "
522  "content failed!" ) % iniFile)
523 
524  # get the job name
525  if options.Name == None:
526  if not options.crabStatus:
527  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
528  else:
529  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
530  "alignmentValidation_*" )
531  if len( existingValDirs ) > 0:
532  options.Name = existingValDirs[-1]
533  else:
534  print("Cannot guess last working directory!")
535  print ( "Please use the parameter '-N' or '--Name' to specify "
536  "the task for which you want a status report." )
537  return 1
538 
539  # set output path
540  outPath = os.path.abspath( options.Name )
541 
542  # Check status of submitted jobs and return
543  if options.crabStatus:
544  os.chdir( outPath )
545  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
546  if len( crabLogDirs ) == 0:
547  print("Found no crab tasks for job name '%s'"%( options.Name ))
548  return 1
549  theCrab = crabWrapper.CrabWrapper()
550  for crabLogDir in crabLogDirs:
551  print()
552  print("*" + "=" * 78 + "*")
553  print ( "| Status report and output retrieval for:"
554  + " " * (77 - len( "Status report and output retrieval for:" ) )
555  + "|" )
556  taskName = crabLogDir.replace( "crab.", "" )
557  print("| " + taskName + " " * (77 - len( taskName ) ) + "|")
558  print("*" + "=" * 78 + "*")
559  print()
560  crabOptions = { "-getoutput":"",
561  "-c": crabLogDir }
562  try:
563  theCrab.run( crabOptions )
564  except AllInOneError as e:
565  print("crab: No output retrieved for this task.")
566  crabOptions = { "-status": "",
567  "-c": crabLogDir }
568  theCrab.run( crabOptions )
569  return
570 
571  general = config.getGeneral()
572  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
573  config.set("internals","scriptsdir",outPath)
574  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
575  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
576  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
577 
578  if not os.path.exists( outPath ):
579  os.makedirs( outPath )
580  elif not os.path.isdir( outPath ):
581  raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
582 
583  # replace default templates by the ones specified in the "alternateTemplates" section
584  loadTemplates( config )
585 
586  #save backup configuration file
587  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
588  config.write( backupConfigFile )
589 
590  #copy proxy, if there is one
591  try:
592  proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
593  except RuntimeError:
594  proxyexists = False
595 
596  if proxyexists:
597  shutil.copyfile(getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy"))
598 
599  validations = []
600  for validation in config.items("validation"):
601  alignmentList = [validation[1]]
602  validationsToAdd = [(validation[0],alignment) \
603  for alignment in alignmentList]
604  validations.extend(validationsToAdd)
605  jobs = [ ValidationJob( validation, config, options) \
606  for validation in validations ]
607  for job in jobs:
608  if job.needsproxy and not proxyexists:
609  raise AllInOneError("At least one job needs a grid proxy, please init one.")
610  map( lambda job: job.createJob(), jobs )
611  validations = [ job.getValidation() for job in jobs ]
612 
613  if options.mergeOfflineParallel:
614  parallelMergeObjects=createMergeScript(outPath, validations, options)['parallelMergeObjects']
615  else:
616  createMergeScript(outPath, validations, options)
617 
618 
619  print()
620  map( lambda job: job.runJob(), jobs )
621 
622  if options.autoMerge and ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
623  print("> Automatically merging jobs when they have ended")
624  # if everything is done as batch job, also submit TkAlMerge.sh to be run
625  # after the jobs have finished
626 
627  #if parallel merge scripts: manage dependencies
628  if options.mergeOfflineParallel and parallelMergeObjects!={}:
629  initID=parallelMergeObjects["init"].runJob(config).split("<")[1].split(">")[0]
630  parallelIDs=[]
631  for parallelMergeScript in parallelMergeObjects["parallel"]:
632  parallelMergeScript.addDependency(initID)
633  for job in jobs:
634  if isinstance(job.validation, OfflineValidation) and "TkAlMerge"+job.validation.alignmentToValidate.name==parallelMergeScript.name:
635  parallelMergeScript.addDependency(job.JobId)
636  parallelIDs.append(parallelMergeScript.runJob(config).split("<")[1].split(">")[0])
637  parallelMergeObjects["continue"].addDependency(parallelIDs)
638  parallelMergeObjects["continue"].addDependency(ValidationJob.batchJobIds)
639  parallelMergeObjects["continue"].runJob(config)
640 
641 
642 
643 
644  else:
645  repMap = {
646  "commands": config.getGeneral()["jobmode"].split(",")[1],
647  "jobName": "TkAlMerge",
648  "logDir": config.getGeneral()["logdir"],
649  "script": "TkAlMerge.sh",
650  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
651  "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
652  }
653  for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
654  oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
655  if os.path.exists(oldlog):
656  os.remove(oldlog)
657 
658  #issue job
659  getCommandOutput2("%(bsub)s %(commands)s "
660  "-o %(logDir)s/%(jobName)s.stdout "
661  "-e %(logDir)s/%(jobName)s.stderr "
662  "-w %(conditions)s "
663  "%(logDir)s/%(script)s"%repMap)
664 
665 if __name__ == "__main__":
666  # main(["-n","-N","test","-c","defaultCRAFTValidation.ini,latestObjects.ini","--getImages"])
667  if "-d" in sys.argv[1:] or "--debug" in sys.argv[1:]:
668  main()
669  else:
670  try:
671  main()
672  except AllInOneError as e:
673  print("\nAll-In-One Tool:", str(e))
674  exit(1)
def main(argv=None)
— Main —############################
def __init__(self, _name, _path, _dependency)
def createMergeScript(path, validations, options)
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:65
def getCommandOutput2(command)
— Classes —############################
def addIndex(filename, njobs, index=None)
def __getValidation(self, valType, name, alignments, config, options)
def replaceByMap(target, the_map)
— Helpers —############################
def __createJob(self, jobMode, outpath)
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def alternateTemplate(templateName, alternateTemplateName)
Alternate Templates ###
def __init__(self, validation, config, options)
Definition: main.py:1
#define str(s)
double split
Definition: MVATrainer.cc:139