validateAlignments Namespace Reference

Classes
class	Alignment
class	BetterConfigParser
	--- Classes ---############################ More...
class	GenericValidation
class	GeometryComparison
class	MonteCarloValidation
class	OfflineValidation
class	OfflineValidationDQM
class	OfflineValidationParallel
class	TrackSplittingValidation
class	ValidationJob
class	ZMuMuValidation
Functions
def	castorDirExists
def	createCrabScript
def	createExtendedValidationScript
def	createMergeScript
def	createOfflineJobsMergeScript
def	createParallelMergeScript
def	getCommandOutput2
def	loadTemplates
def	main
	--- Main ---############################
def	replaceByMap
	--- Helpers ---############################ replaces .oO[id]Oo.
Variables
dictionary	alignRandDict = {}

---

Function Documentation

def validateAlignments::castorDirExists

(

path

)

Definition at line 44 of file validateAlignments.py.

                         :
    if path[-1] == "/":
        path = path[:-1]
    containingPath = os.path.join( *path.split("/")[:-1] )
    dirInQuestion = path.split("/")[-1]
    try:
        rawLines =getCommandOutput2("rfdir /"+containingPath).splitlines()
    except RuntimeError:
        return False
    for line in rawLines:
        if line.split()[0][0] == "d":
            if line.split()[8] == dirInQuestion:
                return True
    return False

def validateAlignments::createCrabScript	(		path,
			validations
	)

Definition at line 1108 of file validateAlignments.py.

                                         :
    repMap = {"crabCommand": "",
              "crabBaseDir": path,
              "useCshell": ""
              }
    for validation in validations:
        if validation.jobmode.split( ',' )[0] == 'crab':
            valRepMap = {"crabWorkingDir": validation.crabWorkingDir,
                         "crabCfgName": validation.crabWorkingDir + ".cfg"
                      }
            repMap["crabCommand"] += replaceByMap( configTemplates.crabCommandTemplate,
                                                   valRepMap )
    if len( repMap["crabCommand"] ) == 0:
        return None
    crabShScriptPath = os.path.join( path, 'TkAlRunCrab.sh' )
    crabShScript = open( crabShScriptPath, 'w' )
    crabShScript.write( replaceByMap( configTemplates.crabShellScriptTemplate,
                                      repMap ) )
    crabShScript.close()
    repMap["useCshell"] = "c"
    crabCshScriptPath = os.path.join( path, 'TkAlRunCrab.csh' )
    crabCshScript = open( crabCshScriptPath, 'w' )
    crabCshScript.write( replaceByMap( configTemplates.crabShellScriptTemplate,
                                       repMap ) )
    crabCshScript.close()
    return crabShScriptPath, crabCshScriptPath

def validateAlignments::createExtendedValidationScript	(		offlineValidationList,
			outFilePath
	)

Definition at line 1147 of file validateAlignments.py.

01148                                                                       :
01149     repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
01150     repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
01151 
01152     for validation in offlineValidationList:
01153         repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
01154     
01155     theFile = open( outFilePath, "w" )
01156     theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
01157     theFile.close()
    

def validateAlignments::createMergeScript	(		path,
			validations
	)

Definition at line 1158 of file validateAlignments.py.

01159                                           :
01160     if( len(validations) == 0 ):
01161         raise StandardError, "cowardly refusing to merge nothing!"
01162 
01163     repMap = validations[0].getRepMap() #FIXME - not nice this way
01164     repMap.update({
01165             "DownloadData":"",
01166             "CompareAllignments":"",
01167             "RunExtendedOfflineValidation":""
01168             })
01169 
01170     comparisonLists = {} # directory of lists containing the validations that are comparable
01171     for validation in validations:
01172         for referenceName in validation.filesToCompare:
01173             validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
01174             validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
01175             if validationName in comparisonLists:
01176                 comparisonLists[ validationName ].append( validation )
01177             else:
01178                 comparisonLists[ validationName ] = [ validation ]
01179 
01180     if "OfflineValidation" in comparisonLists:
01181         repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
01182         createExtendedValidationScript( comparisonLists["OfflineValidation"], repMap["extendeValScriptPath"] )
01183         repMap["RunExtendedOfflineValidation"] = replaceByMap(configTemplates.extendedValidationExecution, repMap)
01184 
01185     repMap["CompareAllignments"] = "#run comparisons"
01186     for validationId in comparisonLists:
01187         compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
01188             
01189         repMap.update({"validationId": validationId,
01190                        "compareStrings": " , ".join(compareStrings) })
01191         
01192         repMap["CompareAllignments"] += replaceByMap( configTemplates.compareAlignmentsExecution, repMap )
01193       
01194     filePath = os.path.join(path, "TkAlMerge.sh")
01195     theFile = open( filePath, "w" )
01196     theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
01197     theFile.close()
01198     os.chmod(filePath,0755)
01199     
01200     return filePath
    

def validateAlignments::createOfflineJobsMergeScript	(		offlineValidationList,
			outFilePath
	)

Definition at line 1135 of file validateAlignments.py.

                                                                    :
    repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
    repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back

    for validation in offlineValidationList:
        repMap[ "mergeOfflinParJobsInstantiation" ] = validation.appendToMergeParJobs( repMap[ "mergeOfflinParJobsInstantiation" ] )
#                    validationsSoFar = 'PlotAlignmentValidation p("%(resultFile)s", "%(name)s", %(color)s, %(style)s);\n'%repMap
    
    theFile = open( outFilePath, "w" )
    theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
    theFile.close()

def validateAlignments::createParallelMergeScript	(		path,
			validations
	)

Definition at line 1201 of file validateAlignments.py.

01202                                                   :
01203     if( len(validations) == 0 ):
01204         raise StandardError, "cowardly refusing to merge nothing!"
01205 
01206     repMap = validations[0].getRepMap() #FIXME - not nice this way
01207     repMap.update({
01208             "DownloadData":"",
01209             "CompareAllignments":"",
01210             "RunExtendedOfflineValidation":""
01211             })
01212 
01213     comparisonLists = {} # directory of lists containing the validations that are comparable
01214     for validation in validations:
01215         for referenceName in validation.filesToCompare:    
01216             validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
01217             validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
01218             if validationName in comparisonLists:
01219                 comparisonLists[ validationName ].append( validation )
01220             else:
01221                 comparisonLists[ validationName ] = [ validation ]
01222 
01223     if "OfflineValidationParallel" in comparisonLists:
01224         repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
01225         createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] )
01226         repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
01227         createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"], repMap["mergeOfflineParJobsScriptPath"] )
01228         repMap["RunExtendedOfflineValidation"] = replaceByMap(configTemplates.extendedValidationExecution, repMap)
01229         # DownloadData is the section which merges output files from parallel jobs
01230         # it uses the file TkAlOfflineJobsMerge.C
01231         repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
01232 
01233     repMap["CompareAllignments"] = "#run comparisons"
01234     for validationId in comparisonLists:
01235         compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
01236             
01237         repMap.update({"validationId": validationId,
01238                        "compareStrings": " , ".join(compareStrings) })
01239         
01240         repMap["CompareAllignments"] += replaceByMap( configTemplates.compareAlignmentsExecution, repMap )
01241       
01242     filePath = os.path.join(path, "TkAlMerge.sh")
01243     theFile = open( filePath, "w" )
01244     theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
01245     theFile.close()
01246     os.chmod(filePath,0755)
01247     
01248     return filePath
    

def validateAlignments::getCommandOutput2

(

command

)

Definition at line 35 of file validateAlignments.py.

00036                               :
00037     child = os.popen(command)
00038     data = child.read()
00039     err = child.close()
00040     if err:
00041         raise RuntimeError, '%s failed w/ exit code %d' % (command, err)
00042     return data
00043 
#check if a directory exsits on castor

def validateAlignments::loadTemplates

(

config

)

Definition at line 1249 of file validateAlignments.py.

01250                            :
01251     if config.has_section("alternateTemplates"):
01252         for templateName in config.options("alternateTemplates"):
01253             newTemplateName = config.get("alternateTemplates", templateName )
01254             #print "replacing default %s template by %s"%( templateName, newTemplateName)
01255             configTemplates.alternateTemplate(templateName, newTemplateName)
    

def validateAlignments::main

(

argv = None

)

--- Main ---############################

Definition at line 1257 of file validateAlignments.py.

                     :
    if argv == None:
       argv = sys.argv[1:]
    optParser = optparse.OptionParser()
    optParser.description = """ all-in-one alignment Validation 
    This will run various validation procedures either on batch queues or interactviely. 
    
    If no name is given (-N parameter) a name containing time and date is created automatically
    
    To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
    """
    optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
                         help="create all scripts and cfg File but do not start jobs (default=False)")
    optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
                          help="get all Images created during the process (default= False)")
    optParser.add_option("-c", "--config", dest="config",
                         help="configuration to use (default compareConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
    optParser.add_option("-N", "--Name", dest="Name",
                         help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
    optParser.add_option("-r", "--restrictTo", dest="restrictTo",
                         help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")

    (options, args) = optParser.parse_args(argv)

    if not options.restrictTo == None:
        options.restrictTo = options.restrictTo.split(",")
    if options.config == None:
        options.config = "compareConfig.ini"
    else:
        options.config = options.config.split(",")
        result = []
        for iniFile in options.config:
            result.append( os.path.abspath(iniFile) )
        options.config = result
    config = BetterConfigParser()
    config.read( options.config )

    if options.Name == None:
        options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))

    outPath = os.path.abspath( options.Name )
    general = config.getGeneral()
    config.set("general","workdir",os.path.join(general["workdir"],options.Name) )
    config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
    config.set("general","logdir",os.path.join(general["logdir"],options.Name) )

    # clean up of log directory to avoid cluttering with files with different
    # random numbers for geometry comparison
    if os.path.isdir( config.getGeneral()["logdir"] ):
        shutil.rmtree( config.getGeneral()["logdir"] )
    
    if not os.path.exists( outPath ):
        os.makedirs( outPath )
    elif not os.path.isdir( outPath ):
        raise StandardError,"the file %s is in the way rename the Job or move it away"%outPath

    #replace default templates by the once specified in the "alternateTemplates" section
    loadTemplates( config )

    #save backup configuration file
    backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w"  )
    config.write( backupConfigFile )

    jobs = [ ValidationJob( validation, config, options) \
                 for validation in config.items( "validation" ) ]
    map( lambda job: job.createJob(), jobs )
    validations = [ job.getValidation() for job in jobs ]

    if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]:
        createMergeScript( outPath, validations )
    else:
        createParallelMergeScript( outPath, validations )
    crabScript = createCrabScript( outPath, validations )
    
    map( lambda job: job.runJob(), jobs )
    if crabScript != None:
        print
        print "="*80
        print "To start parallel validation please type:"
        print "source "+outPath+"/TkAlRunCrab.sh"
        print "\t<or>"
        print "source "+outPath+"/TkAlRunCrab.csh"
        print "="*80
        print
    

def validateAlignments::replaceByMap	(		target,
			map
	)

--- Helpers ---############################ replaces .oO[id]Oo.

by map[id] in target

Definition at line 15 of file validateAlignments.py.

00016                              :
00017     result = target
00018     for id in map:
00019         #print "  "+id+": "+map[id]
00020         lifeSaver = 10e3
00021         iteration = 0
00022         while ".oO[" in result and "]Oo." in result:
00023             for id in map:
00024                 result = result.replace(".oO["+id+"]Oo.",map[id])
00025                 iteration += 1
00026             if iteration > lifeSaver:
00027                 problematicLines = ""
00028                 print map.keys()
00029                 for line in result.splitlines():
00030                     if  ".oO[" in result and "]Oo." in line:
00031                         problematicLines += "%s\n"%line
00032                 raise StandardError, "Oh Dear, there seems to be an endless loop in replaceByMap!!\n%s\nrepMap"%problematicLines
00033     return result
00034 
#excute [command] and return output

---

Variable Documentation

dictionary validateAlignments::alignRandDict = {}

Definition at line 962 of file validateAlignments.py.