CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
validateAlignments.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #test execute: export CMSSW_BASE=/tmp/CMSSW && ./validateAlignments.py -c defaultCRAFTValidation.ini,test.ini -n -N test
3 import os
4 import sys
5 import optparse
6 import datetime
7 import shutil
8 import fnmatch
9 
10 import Alignment.OfflineValidation.TkAlAllInOneTool.configTemplates \
11  as configTemplates
12 import Alignment.OfflineValidation.TkAlAllInOneTool.crabWrapper as crabWrapper
13 from Alignment.OfflineValidation.TkAlAllInOneTool.TkAlExceptions \
14  import AllInOneError
15 from Alignment.OfflineValidation.TkAlAllInOneTool.helperFunctions \
16  import replaceByMap, getCommandOutput2
17 from Alignment.OfflineValidation.TkAlAllInOneTool.betterConfigParser \
18  import BetterConfigParser
19 from Alignment.OfflineValidation.TkAlAllInOneTool.alignment import Alignment
20 
21 from Alignment.OfflineValidation.TkAlAllInOneTool.genericValidation \
22  import GenericValidation
23 from Alignment.OfflineValidation.TkAlAllInOneTool.geometryComparison \
24  import GeometryComparison
25 from Alignment.OfflineValidation.TkAlAllInOneTool.offlineValidation \
26  import OfflineValidation, OfflineValidationDQM, OfflineValidationParallel
27 from Alignment.OfflineValidation.TkAlAllInOneTool.monteCarloValidation \
28  import MonteCarloValidation
29 from Alignment.OfflineValidation.TkAlAllInOneTool.trackSplittingValidation \
30  import TrackSplittingValidation
31 from Alignment.OfflineValidation.TkAlAllInOneTool.zMuMuValidation \
32  import ZMuMuValidation
33 import Alignment.OfflineValidation.TkAlAllInOneTool.globalDictionaries \
34  as globalDictionaries
35 
36 
37 ####################--- Classes ---############################
39  def __init__( self, validation, config, options ):
40  if validation[1] == "":
41  # intermediate syntax
42  valString = validation[0].split( "->" )[0]
43  alignments = validation[0].split( "->" )[1]
44  # force user to use the normal syntax
45  if "->" in validation[0]:
46  msg = ("Instead of using the intermediate syntax\n'"
47  +valString.strip()+"-> "+alignments.strip()
48  +":'\nyou have to use the now fully supported syntax \n'"
49  +valString.strip()+": "
50  +alignments.strip()+"'.")
51  raise AllInOneError(msg)
52  else:
53  valString = validation[0]
54  alignments = validation[1]
55  valString = valString.split()
56  self.__valType = valString[0]
57  self.__valName = valString[1]
58  self.__commandLineOptions = options
59  self.__config = config
60  # workaround for intermediate parallel version
61  if self.__valType == "offlineParallel":
62  section = "offline" + ":" + self.__valName
63  else:
64  section = self.__valType + ":" + self.__valName
65  if not self.__config.has_section( section ):
66  raise AllInOneError, ("Validation '%s' of type '%s' is requested in"
67  " '[validation]' section, but is not defined."
68  "\nYou have to add a '[%s]' section."
69  %( self.__valName, self.__valType, section ))
70  self.validation = self.__getValidation( self.__valType, self.__valName,
71  alignments, self.__config,
72  options )
73 
74  def __getValidation( self, valType, name, alignments, config, options ):
75  if valType == "compare":
76  alignmentsList = alignments.split( "," )
77  firstAlignList = alignmentsList[0].split()
78  firstAlignName = firstAlignList[0].strip()
79  if firstAlignName == "IDEAL":
80  raise AllInOneError, ("'IDEAL' has to be the second (reference)"
81  " alignment in 'compare <val_name>: "
82  "<alignment> <reference>'.")
83  if len( firstAlignList ) > 1:
84  firstRun = firstAlignList[1]
85  else:
86  firstRun = "1"
87  firstAlign = Alignment( firstAlignName, self.__config, firstRun )
88  secondAlignList = alignmentsList[1].split()
89  secondAlignName = secondAlignList[0].strip()
90  if len( secondAlignList ) > 1:
91  secondRun = secondAlignList[1]
92  else:
93  secondRun = "1"
94  if secondAlignName == "IDEAL":
95  secondAlign = secondAlignName
96  else:
97  secondAlign = Alignment( secondAlignName, self.__config,
98  secondRun )
99  # check if alignment was already compared previously
100  try:
101  randomWorkdirPart = \
102  globalDictionaries.alignRandDict[firstAlignName]
103  except KeyError:
104  randomWorkdirPart = None
105 
106  validation = GeometryComparison( name, firstAlign, secondAlign,
107  self.__config,
108  self.__commandLineOptions.getImages,
109  randomWorkdirPart )
110  globalDictionaries.alignRandDict[firstAlignName] = \
111  validation.randomWorkdirPart
112  if not secondAlignName == "IDEAL":
113  globalDictionaries.alignRandDict[secondAlignName] = \
114  validation.randomWorkdirPart
115  elif valType == "offline":
116  validation = OfflineValidation( name,
117  Alignment( alignments.strip(), self.__config ), self.__config )
118  elif valType == "offlineDQM":
119  validation = OfflineValidationDQM( name,
120  Alignment( alignments.strip(), self.__config ), self.__config )
121  elif valType == "offlineParallel":
122  validation = OfflineValidationParallel( name,
123  Alignment( alignments.strip(), self.__config ), self.__config )
124  elif valType == "mcValidate":
125  validation = MonteCarloValidation( name,
126  Alignment( alignments.strip(), self.__config ), self.__config )
127  elif valType == "split":
128  validation = TrackSplittingValidation( name,
129  Alignment( alignments.strip(), self.__config ), self.__config )
130  elif valType == "zmumu":
131  validation = ZMuMuValidation( name,
132  Alignment( alignments.strip(), self.__config ), self.__config )
133  else:
134  raise AllInOneError, "Unknown validation mode '%s'"%valType
135  return validation
136 
137  def __createJob( self, jobMode, outpath ):
138  """This private method creates the needed files for the validation job.
139  """
140  self.validation.createConfiguration( outpath )
141  self.__scripts = self.validation.createScript( outpath )
142  if jobMode.split( ',' )[0] == "crab":
143  self.validation.createCrabCfg( outpath )
144  return None
145 
146  def createJob(self):
147  """This is the method called to create the job files."""
148  self.__createJob( self.validation.jobmode,
149  os.path.abspath( self.__commandLineOptions.Name) )
150 
151  def runJob( self ):
152  general = self.__config.getGeneral()
153  log = ""
154  for script in self.__scripts:
155  name = os.path.splitext( os.path.basename( script) )[0]
156  if self.__commandLineOptions.dryRun:
157  print "%s would run: %s"%( name, os.path.basename( script) )
158  continue
159  log = "> Validating "+name
160  print "> Validating "+name
161  if self.validation.jobmode == "interactive":
162  log += getCommandOutput2( script )
163  elif self.validation.jobmode.split(",")[0] == "lxBatch":
164  repMap = {
165  "commands": self.validation.jobmode.split(",")[1],
166  "logDir": general["logdir"],
167  "jobName": name,
168  "script": script,
169  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
170  }
171  log+=getCommandOutput2("%(bsub)s %(commands)s -J %(jobName)s "
172  "-o %(logDir)s/%(jobName)s.stdout -e "
173  "%(logDir)s/%(jobName)s.stderr "
174  "%(script)s"%repMap)
175  elif self.validation.jobmode.split( "," )[0] == "crab":
176  os.chdir( general["logdir"] )
177  crabName = "crab." + os.path.basename( script )[:-3]
178  theCrab = crabWrapper.CrabWrapper()
179  options = { "-create": "",
180  "-cfg": crabName + ".cfg",
181  "-submit": "" }
182  try:
183  theCrab.run( options )
184  except AllInOneError, e:
185  print "crab:", str(e).split("\n")[0]
186  exit(1)
187  else:
188  raise AllInOneError, ("Unknown 'jobmode'!\n"
189  "Please change this parameter either in "
190  "the [general] or in the ["
191  + self.__valType + ":" + self.__valName
192  + "] section to one of the following "
193  "values:\n"
194  "\tinteractive\n\tlxBatch, -q <queue>\n"
195  "\tcrab, -q <queue>")
196  return log
197 
198  def getValidation( self ):
199  return self.validation
200 
201 
202 ####################--- Functions ---############################
203 def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
204  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
205  repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
206 
207  theFile = open( outFilePath, "w" )
208  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
209  theFile.close()
210 
211 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
212  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
213  repMap[ "resultPlotFile" ] = resultPlotFile
214  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
215 
216  for validation in offlineValidationList:
217  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
218 
219  theFile = open( outFilePath, "w" )
220  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
221  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
222  theFile.close()
223 
224 def createMergeScript( path, validations ):
225  if(len(validations) == 0):
226  msg = "Cowardly refusing to merge nothing!"
227  raise AllInOneError(msg)
228 
229  repMap = validations[0].getRepMap() #FIXME - not nice this way
230  repMap.update({
231  "DownloadData":"",
232  "CompareAlignments":"",
233  "RunExtendedOfflineValidation":""
234  })
235 
236  comparisonLists = {} # directory of lists containing the validations that are comparable
237  resultPlotFile = "" # string of a file name for createExtendedValidationScript
238  for validation in validations:
239  for referenceName in validation.filesToCompare:
240  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
241  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
242  if validationName in comparisonLists:
243  comparisonLists[ validationName ].append( validation )
244  else:
245  comparisonLists[ validationName ] = [ validation ]
246  if validationName == "OfflineValidation":
247  resultPlotFile = validationName
248 
249  if "OfflineValidation" in comparisonLists:
250  repMap["extendeValScriptPath"] = \
251  os.path.join(path, "TkAlExtendedOfflineValidation.C")
252  createExtendedValidationScript(comparisonLists["OfflineValidation"],
253  repMap["extendeValScriptPath"],
254  resultPlotFile)
255  repMap["RunExtendedOfflineValidation"] = \
256  replaceByMap(configTemplates.extendedValidationExecution, repMap)
257 
258  repMap["CompareAlignments"] = "#run comparisons"
259  for validationId in comparisonLists:
260  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
261 
262  repMap.update({"validationId": validationId,
263  "compareStrings": " , ".join(compareStrings) })
264 
265  repMap["CompareAlignments"] += \
266  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
267 
268  filePath = os.path.join(path, "TkAlMerge.sh")
269  theFile = open( filePath, "w" )
270  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
271  theFile.close()
272  os.chmod(filePath,0755)
273 
274  return filePath
275 
276 def createParallelMergeScript( path, validations ):
277  if( len(validations) == 0 ):
278  raise AllInOneError, "cowardly refusing to merge nothing!"
279 
280  repMap = validations[0].getRepMap() #FIXME - not nice this way
281  repMap.update({
282  "DownloadData":"",
283  "CompareAlignments":"",
284  "RunExtendedOfflineValidation":""
285  })
286 
287  comparisonLists = {} # directory of lists containing the validations that are comparable
288  resultPlotFile = "" # string of a file name for createExtendedValidationScript
289  for validation in validations:
290  for referenceName in validation.filesToCompare:
291  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
292  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
293  if validationName in comparisonLists:
294  comparisonLists[ validationName ].append( validation )
295  else:
296  comparisonLists[ validationName ] = [ validation ]
297  if validationName == "OfflineValidationParallel":
298  resultPlotFile = validationName
299 
300  if "OfflineValidationParallel" in comparisonLists:
301  repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
302  createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"], resultPlotFile )
303  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
304  createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"],
305  repMap["mergeOfflineParJobsScriptPath"] )
306 
307  # introduced to merge individual validation outputs separately
308  # -> avoids problems with merge script
309  repMap["haddLoop"] = "mergeRetCode=0\n"
310  repMap["rmUnmerged"] = "if [[ mergeRetCode -eq 0 ]]; then\n"
311  for validation in comparisonLists["OfflineValidationParallel"]:
312  repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"])
313  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
314  repMap["haddLoop"] += ("if [[ mergeRetCode -eq 0 ]]; "
315  "then mergeRetCode=${tmpMergeRetCode}; "
316  "fi\n")
317  repMap["haddLoop"] += ("cmsStage -f "
318  +validation.getRepMap()["outputFile"]
319  +" "
320  +validation.getRepMap()["resultFile"]
321  +"\n")
322  for f in validation.outputFiles:
323  longName = os.path.join("/store/caf/user/$USER/",
324  validation.getRepMap()["eosdir"], f)
325  repMap["rmUnmerged"] += " cmsRm "+longName+"\n"
326  repMap["rmUnmerged"] += ("else\n"
327  " echo \"WARNING: Merging failed, unmerged"
328  " files won't be deleted.\"\n"
329  "fi\n")
330 
331  repMap["RunExtendedOfflineValidation"] = \
332  replaceByMap(configTemplates.extendedValidationExecution, repMap)
333 
334  # DownloadData is the section which merges output files from parallel jobs
335  # it uses the file TkAlOfflineJobsMerge.C
336  repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
337  repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
338 
339  repMap["CompareAlignments"] = "#run comparisons"
340  for validationId in comparisonLists:
341  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
342 
343  repMap.update({"validationId": validationId,
344  "compareStrings": " , ".join(compareStrings) })
345 
346  repMap["CompareAlignments"] += \
347  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
348 
349  filePath = os.path.join(path, "TkAlMerge.sh")
350  theFile = open( filePath, "w" )
351  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
352  theFile.close()
353  os.chmod(filePath,0755)
354 
355  return filePath
356 
357 def loadTemplates( config ):
358  if config.has_section("alternateTemplates"):
359  for templateName in config.options("alternateTemplates"):
360  newTemplateName = config.get("alternateTemplates", templateName )
361  #print "replacing default %s template by %s"%( templateName, newTemplateName)
362  configTemplates.alternateTemplate(templateName, newTemplateName)
363 
364 
365 ####################--- Main ---############################
366 def main(argv = None):
367  if argv == None:
368  argv = sys.argv[1:]
369  optParser = optparse.OptionParser()
370  optParser.description = """ all-in-one alignment Validation
371  This will run various validation procedures either on batch queues or interactviely.
372 
373  If no name is given (-N parameter) a name containing time and date is created automatically
374 
375  To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
376  """
377  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
378  help="create all scripts and cfg File but do not start jobs (default=False)")
379  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
380  help="get all Images created during the process (default= False)")
381  defaultConfig = "TkAlConfig.ini"
382  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
383  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
384  optParser.add_option("-N", "--Name", dest="Name",
385  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
386  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
387  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
388  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
389  help="get the status of the crab jobs", metavar="STATUS")
390  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
391  default = False,
392  help="Run the tool to get full traceback of errors.",
393  metavar="DEBUG")
394 
395  (options, args) = optParser.parse_args(argv)
396 
397  if not options.restrictTo == None:
398  options.restrictTo = options.restrictTo.split(",")
399 
400  options.config = [ os.path.abspath( iniFile ) for iniFile in \
401  options.config.split( "," ) ]
402  config = BetterConfigParser()
403  outputIniFileSet = set( config.read( options.config ) )
404  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
405 
406  # Check for missing ini file
407  if options.config == [ os.path.abspath( defaultConfig ) ]:
408  if ( not options.crabStatus ) and \
409  ( not os.path.exists( defaultConfig ) ):
410  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
411  "You can specify another name with the "
412  "command line option '-c'/'--config'."
413  %( defaultConfig ))
414  else:
415  for iniFile in failedIniFiles:
416  if not os.path.exists( iniFile ):
417  raise AllInOneError, ( "'%s' does not exist. Please check for "
418  "typos in the filename passed to the "
419  "'-c'/'--config' option!"
420  %( iniFile ) )
421  else:
422  raise AllInOneError, ( "'%s' does exist, but parsing of the "
423  "content failed!" )
424 
425  # get the job name
426  if options.Name == None:
427  if not options.crabStatus:
428  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
429  else:
430  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
431  "alignmentValidation_*" )
432  if len( existingValDirs ) > 0:
433  options.Name = existingValDirs[-1]
434  else:
435  print "Cannot guess last working directory!"
436  print ( "Please use the parameter '-N' or '--Name' to specify "
437  "the task for which you want a status report." )
438  return 1
439 
440  # set output path
441  outPath = os.path.abspath( options.Name )
442 
443  # Check status of submitted jobs and return
444  if options.crabStatus:
445  os.chdir( outPath )
446  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
447  if len( crabLogDirs ) == 0:
448  print "Found no crab tasks for job name '%s'"%( options.Name )
449  return 1
450  theCrab = crabWrapper.CrabWrapper()
451  for crabLogDir in crabLogDirs:
452  print
453  print "*" + "=" * 78 + "*"
454  print ( "| Status report and output retrieval for:"
455  + " " * (77 - len( "Status report and output retrieval for:" ) )
456  + "|" )
457  taskName = crabLogDir.replace( "crab.", "" )
458  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
459  print "*" + "=" * 78 + "*"
460  print
461  crabOptions = { "-getoutput":"",
462  "-c": crabLogDir }
463  try:
464  theCrab.run( crabOptions )
465  except AllInOneError, e:
466  print "crab: No output retrieved for this task."
467  crabOptions = { "-status": "",
468  "-c": crabLogDir }
469  theCrab.run( crabOptions )
470  return
471 
472  general = config.getGeneral()
473  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
474  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
475  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
476  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
477 
478  # clean up of log directory to avoid cluttering with files with different
479  # random numbers for geometry comparison
480  if os.path.isdir( outPath ):
481  shutil.rmtree( outPath )
482 
483  if not os.path.exists( outPath ):
484  os.makedirs( outPath )
485  elif not os.path.isdir( outPath ):
486  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
487 
488  # replace default templates by the ones specified in the "alternateTemplates" section
489  loadTemplates( config )
490 
491  #save backup configuration file
492  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
493  config.write( backupConfigFile )
494 
495  validations = []
496  for validation in config.items("validation"):
497  alignmentList = validation[1].split(config.getSep())
498  validationsToAdd = [(validation[0],alignment) \
499  for alignment in alignmentList]
500  validations.extend(validationsToAdd)
501  jobs = [ ValidationJob( validation, config, options) \
502  for validation in validations ]
503  map( lambda job: job.createJob(), jobs )
504  validations = [ job.getValidation() for job in jobs ]
505 
506  if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]:
507  createMergeScript(outPath, validations)
508  else:
509  createParallelMergeScript( outPath, validations )
510 
511  print
512  map( lambda job: job.runJob(), jobs )
513 
514 
515 if __name__ == "__main__":
516  # main(["-n","-N","test","-c","defaultCRAFTValidation.ini,latestObjects.ini","--getImages"])
517  if "-d" in sys.argv[1:] or "--debug" in sys.argv[1:]:
518  main()
519  else:
520  try:
521  main()
522  except AllInOneError, e:
523  print "\nAll-In-One Tool:", str(e)
524  exit(1)
— Classes —############################
def main
— Main —############################
def alternateTemplate
### Alternate Templates ###
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def replaceByMap
— Helpers —############################
if(dp >Float(M_PI)) dp-
Definition: main.py:1
double split
Definition: MVATrainer.cc:139