CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Static Public Member Functions | Public Attributes | Private Member Functions
validateAlignments.ValidationJobMultiIOV Class Reference
Inheritance diagram for validateAlignments.ValidationJobMultiIOV:
validateAlignments.ValidationBase

Public Member Functions

def __init__
 
def __iter__
 
def __next__
 
def createJob
 
def getValidation
 
def needsproxy
 
def runJob
 
- Public Member Functions inherited from validateAlignments.ValidationBase
def __init__
 
def createJob
 
def getValidation
 
def needsproxy
 
def runJob
 

Static Public Member Functions

def runCondorJobs
 

Public Attributes

 config
 
 end
 
 optionMultiIOV
 
 options
 
 outPath
 
 start
 
 validation
 
 validations
 
 valSection
 
 valType
 
- Public Attributes inherited from validateAlignments.ValidationBase
 alignments
 
 commandLineOptions
 
 config
 
 preexisting
 
 valName
 
 valSection
 
 valType
 

Private Member Functions

def __performMultiIOV
 

Detailed Description

Definition at line 294 of file validateAlignments.py.

Constructor & Destructor Documentation

def validateAlignments.ValidationJobMultiIOV.__init__ (   self,
  validation,
  config,
  options,
  outPath,
  args,
  kwargs 
)

Definition at line 296 of file validateAlignments.py.

296 
297  def __init__( self, validation, config, options, outPath, *args, **kwargs):
298  self.start = 0
299  self.end = args
300  super(ValidationJobMultiIOV, self).__init__( validation, config, options )
301  self.optionMultiIOV = self.config.getboolean( self.valSection, "multiIOV" )
302  if self.optionMultiIOV == True:
303  self.validation = validation
304  self.config = config
305  self.options = options
306  self.outPath = outPath
307  self.validations = self.__performMultiIOV(self.validation, self.alignments, self.config,
308  self.options, self.outPath)
309 

Member Function Documentation

def validateAlignments.ValidationJobMultiIOV.__iter__ (   self)

Definition at line 473 of file validateAlignments.py.

473 
474  def __iter__(self):
475  yield self
def validateAlignments.ValidationJobMultiIOV.__next__ (   self)

Definition at line 476 of file validateAlignments.py.

References validateAlignments.ValidationJob.end, validateAlignments.ValidationJobMultiIOV.end, svgfig.LineAxis.end, validateAlignments.ValidationJob.start, validateAlignments.ValidationJobMultiIOV.start, and svgfig.LineAxis.start.

476 
477  def __next__(self):
478  if self.start >= len(self.end):
479  raise StopIteration
480  else:
481  self.start += 1
482  return self.end[self.start-1]
483 
def validateAlignments.ValidationJobMultiIOV.__performMultiIOV (   self,
  validation,
  alignments,
  config,
  options,
  outPath 
)
private

Definition at line 310 of file validateAlignments.py.

310 
311  def __performMultiIOV(self, validation, alignments, config, options, outPath):
312  validations = []
313  if self.valType == "compare":
314  alignmentsList = alignments.split( "," )
315  firstAlignList = alignmentsList[0].split()
316  firstAlignName = firstAlignList[0].strip()
317  secondAlignList = alignmentsList[1].split()
318  secondAlignName = secondAlignList[0].strip()
319  compareAlignments = "%s"%firstAlignName + "_vs_%s"%secondAlignName
320  sectionMultiIOV = "multiIOV:compare"
321  if not self.config.has_section(sectionMultiIOV):
322  raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
323  iovList = self.config.get( sectionMultiIOV, "iovs" )
324  iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
325  for iov in iovList:
326  tmpConfig = BetterConfigParser()
327  tmpConfig.read( options.config )
328  general = tmpConfig.getGeneral()
329  tmpConfig.add_section("IOV")
330  tmpConfig.set("IOV", "iov", iov)
331  tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
332  tmpConfig.set("internals","scriptsdir",os.path.join(outPath, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
333  tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
334  tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
335  tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
336  tmpOptions = copy.deepcopy(options)
337  tmpOptions.Name = os.path.join(options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov)
338  tmpOptions.config = tmpConfig
339  newOutPath = os.path.abspath( tmpOptions.Name )
340  if not os.path.exists( newOutPath ):
341  os.makedirs( newOutPath )
342  elif not os.path.isdir( newOutPath ):
343  raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
344  job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
345  validations.append(job)
346 
347  return validations
348 
349  if "preexisting" in self.valType:
350  preexistingValType = self.valType
351  preexistingValSection = self.valSection
352  preexistingEosdir = self.config.get( self.valSection, "eosdirName" )
353  originalValType = preexistingValType.replace('preexisting', '')
354  originalValName = self.config.get( self.valSection, "originalValName" )
355  self.valSection = originalValType + ":" + originalValName
356  originalAlignment = self.valName
357 
358  datasetList = self.config.get( self.valSection, "dataset" )
359  datasetList = re.sub(r"\s+", "", datasetList, flags=re.UNICODE).split( "," )
360  for dataset in datasetList:
361  sectionMultiIOV = "multiIOV:%s"%dataset
362  if not self.config.has_section(sectionMultiIOV):
363  raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
364  else:
365  datasetBaseName = self.config.get( sectionMultiIOV, "dataset" )
366  iovList = self.config.get( sectionMultiIOV, "iovs" )
367  iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
368  for iov in iovList:
369  datasetName = datasetBaseName+"_since%s"%iov
370  tmpConfig = BetterConfigParser()
371  tmpConfig.read( options.config )
372  general = tmpConfig.getGeneral()
373  if "preexisting" in self.valType:
374  valType = originalValType
375  valName = originalValName
376  else:
377  valType = self.valType
378  valName = self.valName
379  tmpConfig.add_section("IOV")
380  tmpConfig.set("IOV", "iov", iov)
381  tmpConfig.set( self.valSection, "dataset", datasetName )
382  tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
383  tmpConfig.set("internals","scriptsdir",os.path.join(outPath, valType + "_" + valName + "_%s"%iov) )
384  tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, valType + "_" + valName + "_%s"%iov) )
385  tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
386  tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
387  if "preexisting" in self.valType:
388  if self.valType == "preexistingoffline":
389  validationClassName = "AlignmentValidation"
390  #elif self.valType == "preexistingmcValidate":
391  # validationClassName = "MonteCarloValidation"
392  #elif self.valType == "preexistingsplit":
393  # validationClassName = "TrackSplittingValidation"
394  #elif self.valType == "preexistingprimaryvertex":
395  # validationClassName = "PrimaryVertexValidation"
396  else:
397  raise AllInOneError("Unknown validation mode for preexisting option:'%s'"%self.valType)
398  preexistingEosdirPath = os.path.join("AlignmentValidation", preexistingEosdir, valType + "_" + valName + "_%s"%iov)
399  file = "/eos/cms/store/group/alca_trackeralign/AlignmentValidation/" + "%s"%preexistingEosdirPath + "/%s"%validationClassName + "_%s"%originalValName + "_%s"%originalAlignment + ".root"
400  tmpConfig.set(preexistingValSection, "file", file)
401  tmpOptions = copy.deepcopy(options)
402  tmpOptions.Name = os.path.join(options.Name, valType + "_" + valName + "_%s"%iov)
403  tmpOptions.config = tmpConfig
404  newOutPath = os.path.abspath( tmpOptions.Name )
405  if not os.path.exists( newOutPath ):
406  os.makedirs( newOutPath )
407  elif not os.path.isdir( newOutPath ):
408  raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
409  job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
410  validations.append(job)
411 
412  return validations
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:45
def validateAlignments.ValidationJobMultiIOV.createJob (   self)

Definition at line 413 of file validateAlignments.py.

References validateAlignments.ValidationJobMultiIOV.validations.

413 
414  def createJob( self ):
415  lmap( lambda validation: validation.createJob(), self.validations )
def validateAlignments.ValidationJobMultiIOV.getValidation (   self)

Definition at line 467 of file validateAlignments.py.

References validateAlignments.ValidationJobMultiIOV.validations.

467 
468  def getValidation( self ):
469  return [validation.getValidation() for validation in self.validations]
def validateAlignments.ValidationJobMultiIOV.needsproxy (   self)

Definition at line 470 of file validateAlignments.py.

References join(), validateAlignments.ValidationBase.preexisting, and validateAlignments.ValidationJobMultiIOV.validations.

470 
471  def needsproxy( self ):
472  return [validation.needsproxy() for validation in self.validations].join("and") and not self.preexisting and not self.commandLineOptions.dryRun
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def validateAlignments.ValidationJobMultiIOV.runCondorJobs (   outdir)
static

Definition at line 420 of file validateAlignments.py.

References bitset_utilities.append(), join(), and submitPVValidationJobs.split().

420 
421  def runCondorJobs(outdir):
422  dagmanLog = "{}/daglogs".format(outdir)
423  os.system("mkdir -p {}".format(dagmanLog))
424 
425 
426  with open("{}/validation.condor".format(outdir), "w") as condor:
427  condor.write("universe = vanilla" + "\n")
428  condor.write("executable = $(scriptName).sh" + "\n")
429  condor.write("log = $(scriptName).log" + "\n")
430  condor.write("error = $(scriptName).stderr" + "\n")
431  condor.write("output = $(scriptName).stdout" + "\n")
432  condor.write('requirements = (OpSysAndVer =?= "CentOS7")' + '\n')
433  condor.write('+JobFlavour = "tomorrow"' + "\n")
434  condor.write('+RequestMemory = {}'.format(1540) + "\n")
435  condor.write('+FileTransferDownloadBytes = {}'.format(1540) + "\n")
436  condor.write('+AccountingGroup = "group_u_CMS.CAF.ALCA"' + '\n')
437  condor.write("queue")
438 
439  with open("{}/validation.dagman".format(outdir), "w") as dagman:
440  parents = {}
441  for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
442 
443  parents[(valType, valName, iov)] = []
444  for jobInfo in alignments:
445  if not "preexisting" in jobInfo[0]:
446  dagman.write("JOB {}_{} {}/validation.condor".format(jobInfo[0], iov, outdir) + "\n")
447  dagman.write('VARS {}_{} '.format(jobInfo[0], iov) + 'scriptName="{}"'.format('.'.join(jobInfo[1].split('.')[:-1])) + "\n")
448  parents[(valType, valName, iov)].append('{}_{}'.format(jobInfo[0], iov))
449  dagman.write("\n")
450 
451  path = os.path.join(jobInfo[2], "TkAlMerge.sh")
452  if os.path.exists( path ):
453  dagman.write("JOB Merge_{}_{}_{} {}/validation.condor".format(valType, valName, iov, outdir) + "\n")
454  dagman.write("VARS Merge_{}_{}_{} ".format(valType, valName, iov) + 'scriptName="{}"'.format(os.path.join(jobInfo[2], "TkAlMerge")) + "\n")
455  dagman.write("\n")
456  else:
457  raise AllInOneError("Merge script '[%s]' not found!"%path)
458 
459  for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
460  if len(parents[(valType, valName, iov)]) != 0:
461  dagman.write('PARENT {} '.format(" ".join([parent for parent in parents[(valType, valName, iov)]])) + 'CHILD Merge_{}_{}_{}'.format(valType, valName, iov) + "\n")
462 
463  submitCommands = ["condor_submit_dag -no_submit -outfile_dir {} {}/validation.dagman".format(dagmanLog, outdir), "condor_submit {}/validation.dagman.condor.sub".format(outdir)]
464 
465  for command in submitCommands:
466  subprocess.call(command.split(" "))
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def validateAlignments.ValidationJobMultiIOV.runJob (   self)

Definition at line 416 of file validateAlignments.py.

References validateAlignments.ValidationJobMultiIOV.validations.

416 
417  def runJob( self ):
418  return [validation.runJob() for validation in self.validations]

Member Data Documentation

validateAlignments.ValidationJobMultiIOV.config

Definition at line 303 of file validateAlignments.py.

Referenced by zMuMuValidation.ZMuMuValidation.trackcollection().

validateAlignments.ValidationJobMultiIOV.end

Definition at line 298 of file validateAlignments.py.

Referenced by OrderedSet.OrderedSet.__iter__(), validateAlignments.ValidationJobMultiIOV.__next__(), OrderedSet.OrderedSet.__reversed__(), OrderedSet.OrderedSet.add(), Types.LuminosityBlockRange.cppID(), Types.EventRange.cppID(), o2olib.O2ORunMgr.endJob(), and OrderedSet.OrderedSet.pop().

validateAlignments.ValidationJobMultiIOV.optionMultiIOV

Definition at line 300 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.options

Definition at line 304 of file validateAlignments.py.

Referenced by betterConfigParser.BetterConfigParser.__updateDict(), submitPVValidationJobs.BetterConfigParser.__updateDict(), DTWorkflow.DTWorkflow.add_local_custom_db(), confdb.HLTProcess.buildOptions(), DTWorkflow.DTWorkflow.check_missing_options(), betterConfigParser.BetterConfigParser.checkInput(), CrabHelper.CrabHelper.crab_config_filename(), CrabHelper.CrabHelper.crab_taskname(), DTWorkflow.DTWorkflow.dump_options(), CrabHelper.CrabHelper.fill_options_from_crab_config(), confdb.HLTProcess.fixPrescales(), DTWorkflow.DTWorkflow.load_options(), DTWorkflow.DTWorkflow.prepare_common_submit(), edmIntegrityCheck.IntegrityCheck.query(), DTCalibrationWorker.DTCalibrationWorker.run(), cmsswPreprocessor.CmsswPreprocessor.run(), DTWorkflow.DTWorkflow.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), production_tasks.CleanJobFiles.run(), and DTWorkflow.DTWorkflow.user().

validateAlignments.ValidationJobMultiIOV.outPath

Definition at line 305 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.start

Definition at line 297 of file validateAlignments.py.

Referenced by progressbar.ProgressBar.__next__(), validateAlignments.ValidationJobMultiIOV.__next__(), Types.LuminosityBlockRange.cppID(), Types.EventRange.cppID(), and o2olib.O2ORunMgr.endJob().

validateAlignments.ValidationJobMultiIOV.validation

Definition at line 302 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.validations

Definition at line 306 of file validateAlignments.py.

Referenced by validateAlignments.ValidationJobMultiIOV.createJob(), validateAlignments.ValidationJobMultiIOV.getValidation(), validateAlignments.ValidationJobMultiIOV.needsproxy(), and validateAlignments.ValidationJobMultiIOV.runJob().

validateAlignments.ValidationJobMultiIOV.valSection

Definition at line 354 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.valType

Definition at line 312 of file validateAlignments.py.

Referenced by preexistingValidation.PreexistingValidation.getRepMap(), and genericValidation.GenericValidation.getRepMap().

Generated for CMSSW Reference Manual by doxygen 1.8.5