validateAlignments Namespace Reference

Classes
class	Alignment
class	BetterConfigParser
	--- Classes ---############################ More...
class	GenericValidation
class	GeometryComparision
class	MonteCarloValidation
class	OfflineValidation
class	OfflineValidationDQM
class	TrackSplittingValidation
Functions
def	castorDirExists
def	createExtendedValidationScript
def	createMergeScript
def	getCommandOutput2
def	loadTemplates
def	main
	--- Main ---############################
def	readAlignments
def	readCompare
def	readGeneral
def	replaceByMap
	--- Helpers ---############################ replaces .oO[id]Oo.
def	runJob

---

Function Documentation

def validateAlignments::castorDirExists

(

path

)

Definition at line 43 of file validateAlignments.py.

                         :
    if path[-1] == "/":
        path = path[:-1]
    containingPath = os.path.join( *path.split("/")[:-1] )
    dirInQuestion = path.split("/")[-1]
    try:
        rawLines =getCommandOutput2("rfdir /"+containingPath).splitlines()
    except RuntimeError:
        return False
    for line in rawLines:
        if line.split()[0][0] == "d":
            if line.split()[8] == dirInQuestion:
                return True
    return False

def validateAlignments::createExtendedValidationScript	(		offlineValidationList,
			outFilePath
	)

Definition at line 582 of file validateAlignments.py.

00583                                                                       :
00584     repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
00585     repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
00586 
00587     for validation in offlineValidationList:
00588         repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
00589     
00590     theFile = open( outFilePath, "w" )
00591     theFile.write( replaceByMap( configTemplates.extendedVaidationTemplate ,repMap ) )
00592     theFile.close()
    

def validateAlignments::createMergeScript	(		path,
			validations
	)

Definition at line 593 of file validateAlignments.py.

00594                                           :
00595     if( len(validations) == 0 ):
00596         raise StandardError, "cowardly refusing to merge nothing!"
00597 
00598     repMap = validations[0].getRepMap() #FIXME - not nice this way
00599     repMap.update({
00600             "DownloadData":"",
00601             "CompareAllignments":"",
00602             "RunExtendedOfflineValidation":""
00603             })
00604 
00605     comparisonLists = {} # directory of lists containing the validations that are comparable
00606     for validation in validations:
00607         for referenceName in validation.filesToCompare:    
00608             validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
00609             validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
00610             if validationName in comparisonLists:
00611                 comparisonLists[ validationName ].append( validation )
00612             else:
00613                 comparisonLists[ validationName ] = [ validation ]
00614 
00615     if "OfflineValidation" in comparisonLists:
00616         repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
00617         createExtendedValidationScript( comparisonLists["OfflineValidation"], repMap["extendeValScriptPath"] )
00618         repMap["RunExtendedOfflineValidation"] = replaceByMap(configTemplates.extendedVaidationExecution, repMap)
00619 
00620     repMap["CompareAllignments"] = "#run comparisons"
00621     for validationId in comparisonLists:
00622         compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
00623             
00624         repMap.update({"validationId": validationId,
00625                        "compareStrings": " , ".join(compareStrings) })
00626         
00627         repMap["CompareAllignments"] += replaceByMap( configTemplates.compareAlignmentsExecution, repMap )
00628       
00629     filePath = os.path.join(path, "TkAlMerge.sh")
00630     theFile = open( filePath, "w" )
00631     theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
00632     theFile.close()
00633     os.chmod(filePath,0755)
00634     
00635     return filePath
    

def validateAlignments::getCommandOutput2

(

command

)

Definition at line 34 of file validateAlignments.py.

00035                               :
00036     child = os.popen(command)
00037     data = child.read()
00038     err = child.close()
00039     if err:
00040         raise RuntimeError, '%s failed w/ exit code %d' % (command, err)
00041     return data
00042 
#check if a directory exsits on castor

def validateAlignments::loadTemplates

(

config

)

Definition at line 636 of file validateAlignments.py.

00637                            :
00638     if config.has_section("alternateTemplates"):
00639         for templateName in config.options("alternateTemplates"):
00640             newTemplateName = config.get("alternateTemplates", templateName )
00641             #print "replacing default %s template by %s"%( templateName, newTemplateName)
00642             configTemplates.alternateTemplate(templateName, newTemplateName)
    

def validateAlignments::main

(

argv = None

)

--- Main ---############################

Definition at line 644 of file validateAlignments.py.

                     :
    if argv == None:
       argv = sys.argv[1:]
    optParser = optparse.OptionParser()
    optParser.description = """ all-in-one alignment Validation 
    This will run various validation procedures either on batch queues or interactviely. 
    
    If no name is given (-N parameter) a name containing time and date is created automatically
    
    To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
    """
    optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
                         help="create all scripts and cfg File but do not start jobs (default=False)")
    optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
                          help="get all Images created during the process (default= False)")
    optParser.add_option("-c", "--config", dest="config",
                         help="configuration to use (default compareConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
    optParser.add_option("-N", "--Name", dest="Name",
                         help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
    optParser.add_option("-r", "--restrictTo", dest="restrictTo",
                         help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")

    (options, args) = optParser.parse_args(argv)

    if not options.restrictTo == None:
        options.restrictTo = options.restrictTo.split(",")
    if options.config == None:
        options.config = "compareConfig.ini"
    else:
        options.config = options.config.split(",")
        result = []
        for iniFile in options.config:
            result.append( os.path.abspath(iniFile) )
        options.config = result
    config = BetterConfigParser()
    config.read( options.config )

    if options.Name == None:
        options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))

    outPath = os.path.abspath( options.Name )
    general = readGeneral( config )
    config.set("general","workdir",os.path.join(general["workdir"],options.Name) )
    config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
    config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
    if not os.path.exists( outPath ):
        os.makedirs( outPath )
    elif not os.path.isdir( outPath ):
        raise StandardError,"the file %s is in the way rename the Job or move it away"%outPath

    #replace default templates by the once specified in the "alternateTemplates" section
    loadTemplates( config )

    log = ""
    alignments = readAlignments( config )
    validations = []
    for alignment in alignments:
        alignment.restrictTo( options.restrictTo )
        validations.extend( alignment.createValidations( config, options, alignments ) )

    scripts = []
    for validation in validations:
        validation.createConfiguration( outPath )
        scripts.extend( validation.createScript( outPath ) )
    
    createMergeScript( outPath, validations )

    #save backup configuration file
    backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w"  )
    config.write( backupConfigFile )

    for script in scripts:
        name = os.path.splitext( os.path.basename( script ) )[0]
        if options.dryRun:
            print "%s would run: %s"%( name, script)
        else:
            runJob( name, script, config)

def validateAlignments::readAlignments

(

config

)

Definition at line 524 of file validateAlignments.py.

                            :
    result = []
    for section in config.sections():
        if "alignment:" in section:
            result.append( Alignment( section.split("alignment:")[1], config ) )
    return result

def validateAlignments::readCompare

(

config

)

Definition at line 531 of file validateAlignments.py.

                         :
    result = {}
    for section in config.sections():
        if "compare:" in section:
            levels =  config.get(section, "levels")
            dbOutput = config.get(section, "dbOutput")
            result[section.split(":")[1]] =(levels,dbOutput)
    return result

def validateAlignments::readGeneral

(

config

)

Definition at line 540 of file validateAlignments.py.

                         :
    result = {
        "jobmode":"interactive",
        "superPointingDataset":"",# see remark about rethroughung missing entries in general. this is needed to be backward compatible
        "workdir":os.getcwd(),
        "datadir":os.getcwd(),
        "logdir":os.getcwd(),
        "offlineModuleLevelHistsTransient":"False",
        "OfflineTreeBaseDir":"TrackHitFilter",
        "DMRMethod":"medianX",
        "DMRMinimum":"30"
        }
    try:
       for option in config.options("general"):
          result[option] = config.get("general",option)
           
       if "localGeneral" in config.sections():
          for option in result:
             if option in [item[0] for item in config.items("localGeneral")]:
                result[ option ] = config.get("localGeneral", option)
    except ConfigParser.NoSectionError, section:
       raise StandardError, "missing section '%s' in configuration files. This section is mandatory."%section
    return result

def validateAlignments::replaceByMap	(		target,
			map
	)

--- Helpers ---############################ replaces .oO[id]Oo.

by map[id] in target

Definition at line 14 of file validateAlignments.py.

00015                              :
00016     result = target
00017     for id in map:
00018         #print "  "+id+": "+map[id]
00019         lifeSaver = 10e3
00020         iteration = 0
00021         while ".oO[" in result and "]Oo." in result:
00022             for id in map:
00023                 result = result.replace(".oO["+id+"]Oo.",map[id])
00024                 iteration += 1
00025             if iteration > lifeSaver:
00026                 problematicLines = ""
00027                 print map.keys()
00028                 for line in result.splitlines():
00029                     if  ".oO[" in result and "]Oo." in line:
00030                         problematicLines += "%s\n"%line
00031                 raise StandardError, "Oh Dear, there seems to be an endless loop in replaceByMap!!\n%s\nrepMap"%problematicLines
00032     return result
00033 
#excute [command] and return output

def validateAlignments::runJob	(		jobName,
			script,
			config
	)

Definition at line 564 of file validateAlignments.py.

                                   :
    general = readGeneral( config )
    log = ">             Validating "+jobName
    print ">             Validating "+jobName
    if general["jobmode"] == "interactive":
        log += getCommandOutput2( script )
    if general["jobmode"].split(",")[0] == "lxBatch":
        repMap = { 
            "commands": general["jobmode"].split(",")[1],
            "logDir": general["logdir"],
            "jobName": jobName,
            "script": script,
            "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
            }
        
        log+=getCommandOutput2("%(bsub)s %(commands)s -J %(jobName)s -o %(logDir)s/%(jobName)s.stdout -e %(logDir)s/%(jobName)s.stderr %(script)s"%repMap)
    return log