cmsPerfSuiteHarvest Namespace Reference

Functions
def	_eventContent_DEBUG (edm_report)
def	assign_event_content_for_product (product)
def	doQuery (query, database)
def	exportIgProfReport (path, igProfReport, igProfType, runinfo)
def	exportMemcheckReport (path, MemcheckReport, runinfo)
def	exportSequences ()
def	exportTimeSizeJob (path, timeSizeReport, runinfo)
def	get_modules_sequences_relationships ()
def	get_params (argv)
def	getIgSummary (path)
def	getMemcheckError (path)
def	getSummaryInfo (database)
def	process_igprof_dir (path, runinfo)
def	process_memcheck_dir (path, runinfo)
def	process_timesize_dir (path, runinfo)
def	searchIgProfFiles (runinfo)
def	searchMemcheckFiles (runinfo)
def	searchTimeSizeFiles (runinfo)
def	usage (argv)

Variables
	_TEST_RUN
	candles
	eventContentRules
	EventContents_OK
	file_name
	now
	output_dir
	p
	pileups
	release
	run_info
	Sequences_OK
	steps
	test_igprof_report_log
	test_memcheck_report_log
	test_timing_report_log
	xmldoc

Function Documentation

def cmsPerfSuiteHarvest._eventContent_DEBUG (   edm_report )

private

Definition at line 82 of file cmsPerfSuiteHarvest.py.

References mps_setup.append, join(), and parseEventContent.List_ECs_forProduct().

Referenced by process_timesize_dir().

def _eventContent_DEBUG(edm_report):
    # for testing / information
    EC_count = {}
    if not _TEST_RUN:
        # count the products in event-content's
        for prod in edm_report:
            ecs = parseEventContent.List_ECs_forProduct(prod)
            for ec in ecs:
                if ec not in EC_count:
                    EC_count[ec] = []   
                EC_count[ec].append(prod)
        #print out the statistics
        for (ec, prods) in EC_count.items():
            print "==== %s EVENT CONTENT: have %d items, the listing is: ===" % (ec, len(prods))
            # list of products
            print "\n *".join(["%(cpp_type)s_%(module_name)s_%(module_label)s" % prod for prod in prods])

def cmsPerfSuiteHarvest.assign_event_content_for_product

(

product

)

returns modified product by adding the event content relationship 

Definition at line 100 of file cmsPerfSuiteHarvest.py.

References join(), and parseEventContent.List_ECs_forProduct().

def assign_event_content_for_product(product):
    """ returns modified product by adding the event content relationship """

    if not _TEST_RUN:
        product["event_content"] = ",".join(parseEventContent.List_ECs_forProduct(product))
    return product

def cmsPerfSuiteHarvest.doQuery	(		query,
			database
	)

Definition at line 459 of file cmsPerfSuiteHarvest.py.

Referenced by getSummaryInfo().

def doQuery(query, database):
    if os.path.exists("/usr/bin/sqlite3"):
        sqlite="/usr/bin/sqlite3"
    else:
        sqlite="/afs/cern.ch/user/e/eulisse/www/bin/sqlite"
    return getstatusoutput("echo '%s' | %s -separator @@@ %s" % (query, sqlite, database))

#TimeSize

def cmsPerfSuiteHarvest.exportIgProfReport	(		path,
			igProfReport,
			igProfType,
			runinfo
	)

Definition at line 113 of file cmsPerfSuiteHarvest.py.

References mps_setup.append, python.rootplot.root2matplotlib.replace(), harvestTrackValidationPlots.str, and pileupCalc.upper.

Referenced by process_igprof_dir().

def exportIgProfReport(path, igProfReport, igProfType, runinfo):
    jobID = igProfReport["jobID"]
    #print jobID
    candleLong = os.path.split(path)[1].replace("_IgProf_Perf", "").replace("_IgProf_Mem", "").replace("_PU", "")
    found = False
    #print igProfType
    if igProfType in runinfo['TestResults']:
        for result in runinfo['TestResults'][igProfType]:
            if candleLong == result["candle"] and jobID["pileup_type"] == result['pileup_type'] and jobID["conditions"] == result['conditions'] and jobID["event_content"] == result['event_content']:
                jobID["candle"] = jobID["candle"].upper()
                if "jobs" not in result:
                    result['jobs'] = []
                result['jobs'].append(igProfReport)
                found = True
                break
        
    if not found:
        print "============ (almost) ERROR: NOT FOUND THE ENTRY in cmsPerfSuite.log, exporting as separate entry ======== "
        print "JOB ID: %s " % str(jobID)
        print " ====================== "
        runinfo['unrecognized_jobs'].append(igProfReport)
        #export_xml(xml_doc = xmldoc, **igProfReport)   
        

def cmsPerfSuiteHarvest.exportMemcheckReport	(		path,
			MemcheckReport,
			runinfo
	)

Definition at line 164 of file cmsPerfSuiteHarvest.py.

References mps_setup.append, python.rootplot.root2matplotlib.replace(), and harvestTrackValidationPlots.str.

Referenced by process_memcheck_dir().

def exportMemcheckReport(path, MemcheckReport, runinfo):
        candleLong = os.path.split(path)[1].replace("_Memcheck", "").replace("_PU", "")
        jobID = MemcheckReport["jobID"]

        #search for a run Test to which could belong our JOB
        found = False
        if 'Memcheck' in runinfo['TestResults']:
            for result in runinfo['TestResults']['Memcheck']:
                #print result
                                #print jobID
                """ If this is the testResult which fits Memcheck job """
                #TODO: we do not check teh step when assigning because of the different names, check if this is really OK. make a decission which step name to use later, long or short one
                #and jobID["step"] in result['steps'].split(parserPerfsuiteMetadata._LINE_SEPARATOR)
                if result['candle'] == candleLong  and jobID["pileup_type"] == result['pileup_type'] and jobID["conditions"] == result['conditions'] and jobID["event_content"] == result['event_content']:
                    #print result
                    if "jobs" not in result:
                        result['jobs'] = []
                    result['jobs'].append(MemcheckReport)
                    found = True
                    break
        
        if not found:
            print "============ (almost) ERROR: NOT FOUND THE ENTRY in cmsPerfSuite.log, exporting as separate entry ======== "
            print "JOB ID: %s " % str(jobID)
            print " ====================== "
            runinfo['unrecognized_jobs'].append(MemcheckReport)

def cmsPerfSuiteHarvest.exportSequences

(

)

Exports the sequences to XML Doc 

Definition at line 514 of file cmsPerfSuiteHarvest.py.

References get_modules_sequences_relationships(), and cmssw_exportdb_xml.xml_export_Sequences().

def exportSequences():
    """ Exports the sequences to XML Doc """
    try:
        env_cmssw_version = os.environ["CMSSW_VERSION"]
    except KeyError:
        print "<<<<<  ====== Error: cannot get CMSSW version [just integrity check for sequences]. \
                     Is the CMSSW environment initialized? (use cmsenv) ==== >>>>"
    env_cmssw_version = None

    print " ==== exporting the sequences. loading files for currently loaded CMSSW version: %s, while the CMSSW we are currently harversting is %s ===" %(env_cmssw_version, release)
    xml_export_Sequences(xml_doc = xmldoc, sequences = get_modules_sequences_relationships(), release=release)

def cmsPerfSuiteHarvest.exportTimeSizeJob	(		path,
			timeSizeReport,
			runinfo
	)

Definition at line 137 of file cmsPerfSuiteHarvest.py.

References mps_setup.append, python.rootplot.root2matplotlib.replace(), and harvestTrackValidationPlots.str.

Referenced by process_timesize_dir().

def exportTimeSizeJob(path, timeSizeReport,  runinfo):
        candleLong = os.path.split(path)[1].replace("_TimeSize", "").replace("_PU", "")
        jobID = timeSizeReport["jobID"]

        #search for a run Test to which could belong our JOB
        found = False
        if 'TimeSize' in runinfo['TestResults']:
            for result in runinfo['TestResults']['TimeSize']:
                #print result
                """ If this is the testResult which fits TimeSize job """
                #TODO: we do not check teh step when assigning because of the different names, check if this is really OK. make a decission which step name to use later, long or short one
                #and jobID["step"] in result['steps'].split(parserPerfsuiteMetadata._LINE_SEPARATOR)
                if result['candle'] == candleLong  and jobID["pileup_type"] == result['pileup_type'] and jobID["conditions"] == result['conditions'] and jobID["event_content"] == result['event_content']:
                    #print result
                    if "jobs" not in result:
                        result['jobs'] = []
                    result['jobs'].append(timeSizeReport)
                    found = True
                    break
        
        if not found:
            print "============ (almost) ERROR: NOT FOUND THE ENTRY in cmsPerfSuite.log, exporting as separate entry ======== "
            print "JOB ID: %s " % str(jobID)
            print " ====================== "
            runinfo['unrecognized_jobs'].append(timeSizeReport)
            #export_xml(xml_doc = xmldoc, **timeSizeReport) 
            

def cmsPerfSuiteHarvest.get_modules_sequences_relationships

(

)

Definition at line 108 of file cmsPerfSuiteHarvest.py.

References ModuleToSequenceAssign.assignModulesToSeqs(), and join().

Referenced by exportSequences().

def get_modules_sequences_relationships():
    (sequenceWithModules, sequenceWithModulesString) =ModuleToSequenceAssign.assignModulesToSeqs()
    return [{"name": seq, "modules": ",".join(modules)} for (seq, modules) in sequenceWithModulesString.items()]

def cmsPerfSuiteHarvest.get_params

(

argv

)

Returns the version of CMSSW to be used which it is taken from:
* command line parameter or 
* environment variable 
in case of error returns None

And also the directory to put the xml files to: if none --> returns ""

try to get the version for command line argument 

Definition at line 45 of file cmsPerfSuiteHarvest.py.

def get_params(argv):
    """ 
    Returns the version of CMSSW to be used which it is taken from:
    * command line parameter or 
    * environment variable 
    in case of error returns None

    And also the directory to put the xml files to: if none --> returns ""
    """
    
    """ try to get the version for command line argument """
    #print argv
    #FIXME: this should be rewritten using getopt properly
    version = None
    #xml_dir = "cmsperfvm:/data/projects/conf/PerfSuiteDB/xml_dropbox" #Set this as default (assume change in write_xml to write to remote machines)
    #NB write_xml is in Validation/Performance/python/cmssw_exportdb_xml.py
    #Setting the default to write to a local directory:
    xml_dir="PerfSuiteDBData"
    try:                              
        opts, args = getopt.getopt(argv[1:], "v:", ["version=", "outdir="])
    except getopt.GetoptError as e:  
        print e
    for opt, arg in opts:
        if opt in ("-v", "--version"):
            version = arg
    if opt == "--outdir":
         xml_dir = arg
    
    """ if not get it from environment string """
    if not version:
        try:
            version = os.environ["CMSSW_VERSION"]
        except KeyError:
            pass
    
    return (version, xml_dir)
        

def cmsPerfSuiteHarvest.getIgSummary

(

path

)

Definition at line 416 of file cmsPerfSuiteHarvest.py.

References getSummaryInfo(), and split.

Referenced by process_igprof_dir().

def getIgSummary(path):
    igresult = []
    globbed = glob.glob(os.path.join(path, "*.sql3"))
    
    for f in globbed:
        #print f
        profileInfo = getSummaryInfo(f)
        if not profileInfo:
            continue
        cumCounts, cumCalls = profileInfo
        dump, architecture, release, rest = f.rsplit("/", 3)
        candle, sequence, pileup, conditions, process, counterType, events = rest.split("___")
        events = events.replace(".sql3", "")
        igresult.append({"counter_type": counterType, "event": events, "cumcounts": cumCounts, "cumcalls": cumCalls})

    #fail-safe(nasty) fix for the diff (even if it gets fixed in the sqls, won't screw this up again...)
    for ig in igresult:
        if 'diff' in ig['event']:
            eventLast,eventOne = ig['event'].split('_diff_')
            for part in igresult:
                if part['counter_type'] == ig['counter_type'] and part['event'] == eventOne:
                    cumcountsOne = part['cumcounts']
                    cumcallsOne = part['cumcalls']
                if part['counter_type'] == ig['counter_type'] and part['event'] == eventLast:
                    cumcountsLast = part['cumcounts']
                    cumcallsLast = part['cumcalls']
            ig['cumcounts'] = cumcountsLast - cumcountsOne
            ig['cumcalls'] = cumcallsLast - cumcallsOne

    return igresult
    

def cmsPerfSuiteHarvest.getMemcheckError

(

path

)

Definition at line 335 of file cmsPerfSuiteHarvest.py.

References createfilelist.int.

Referenced by process_memcheck_dir().

def getMemcheckError(path):
    globbed = glob.glob(os.path.join(path, "*memcheck_vlgd.xml"))

    errnum = 0

    for f in globbed:
        #print f
        cmd = "grep '<error>' "+f+ " | wc -l "
        p = os.popen(cmd, 'r')
        errnum += int(p.readlines()[0])
                        
    return errnum
    

def cmsPerfSuiteHarvest.getSummaryInfo

(

database

)

Definition at line 447 of file cmsPerfSuiteHarvest.py.

References doQuery(), objects.autophobj.float, and createfilelist.int.

Referenced by getIgSummary().

def getSummaryInfo(database):
    summary_query="""SELECT counter, total_count, total_freq, tick_period
                     FROM summary;"""
    error, output = doQuery(summary_query, database)
    if error or not output or output.count("\n") > 1:
        return None
    counter, total_count, total_freq, tick_period = output.split("@@@")
    if counter == "PERF_TICKS":
        return float(tick_period) * float(total_count), int(total_freq)
    else:
        return int(total_count), int(total_freq)
    

def cmsPerfSuiteHarvest.process_igprof_dir	(		path,
			runinfo
	)

Definition at line 349 of file cmsPerfSuiteHarvest.py.

References cmsPerfStripChart.dict, Exception, exportIgProfReport(), getIgSummary(), FileNamesHelper.getJobID_fromIgProfLogName(), list(), FileNamesHelper.read_SimulationCandles(), python.rootplot.root2matplotlib.replace(), harvestTrackValidationPlots.str, and ComparisonHelper.zip().

Referenced by searchIgProfFiles().

def process_igprof_dir(path, runinfo):
    global release,event_content,conditions
    """ if the release is not provided explicitly we take it from the Simulation candles file """
    if (not release):
        release_fromlogfile = read_SimulationCandles(path)
        release  = release_fromlogfile 
        print "release from simulation candles: %s" % release
    
    if (not release):
        # TODO: raise exception!
        raise Exception("the release was not found!")

    """ process the IgProf sql3 files """

        # get the file list 
    files = os.listdir(path)
    igprof_files = [os.path.join(path, f) for f in files
                 if test_igprof_report_log.search(f) 
                    and os.path.isfile(os.path.join(path, f)) ]

        if len(igprof_files) == 0: # No files...
            print "No igprof files found!"
        else:
            for file in igprof_files:
                jobID = getJobID_fromIgProfLogName(file)

                (candle, step, pileup_type, conditions, event_content) = jobID

                print "jobID: %s" % str(jobID)
                jobID = dict(list(zip(("candle", "step", "pileup_type", "conditions", "event_content"), jobID)))
                
                print "Dictionary based jobID %s: " % str(jobID)
                
                igProfType = path.split("/")[-1].replace("TTbar_", "").replace("MinBias_", "").replace("PU_", "")
                
            #if any of jobID fields except (isPILEUP) is empty we discard the job as all those are the jobID keys and we must have them
                discard = len([key for key, value in jobID.items() if key != "pileup_type" and not value])
                if discard:
                    print " ====================== The job HAS BEEN DISCARDED =============== "
                    print " NOT ALL DATA WAS AVAILABLE "
                    print " JOB ID = %s " % str(jobID)
                    print " ======================= end ===================================== "
                    continue
        
                # add to the list to generate the readable filename :)
                steps[step] = 1
                candles[candle.upper()] = 1
                if pileup_type=="":
                    pileups["NoPileUp"]=1
                else:
                    pileups[pileup_type] = 1
            
                igs = getIgSummary(path)
                #print igs

                igProfReport = {
                    "jobID": jobID,
                    "release": release, 
                    "igprof_result": igs,
                    "metadata": {"testname": igProfType},
                    }

                # print igProfReport
                # export to xml: actualy exporting gets suspended and put into runinfo
                exportIgProfReport(path, igProfReport, igProfType, runinfo)      

#get IgProf summary information from the sql3 files

def cmsPerfSuiteHarvest.process_memcheck_dir	(		path,
			runinfo
	)

Definition at line 273 of file cmsPerfSuiteHarvest.py.

References cmsPerfStripChart.dict, Exception, exportMemcheckReport(), FileNamesHelper.getJobID_fromMemcheckLogName(), getMemcheckError(), list(), FileNamesHelper.read_SimulationCandles(), harvestTrackValidationPlots.str, and ComparisonHelper.zip().

Referenced by searchMemcheckFiles().

def process_memcheck_dir(path, runinfo):
    global release,event_content,conditions
    """ if the release is not provided explicitly we take it from the Simulation candles file """
    if (not release):
        release_fromlogfile = read_SimulationCandles(path)
        release  = release_fromlogfile 
        print "release from simulation candles: %s" % release
    
    if (not release):
        # TODO: raise exception!
        raise Exception("the release was not found!")

    """ process the vlgd files """

        # get the file list 
    files = os.listdir(path)
    memcheck_files = [os.path.join(path, f) for f in files
                 if test_memcheck_report_log.search(f) 
                    and os.path.isfile(os.path.join(path, f)) ]

        if len(memcheck_files) == 0: # Fast protection for old runs, where the _vlgd files is not created...
            print "No _vlgd files found!"
        else:
            for file in memcheck_files:
                jobID = getJobID_fromMemcheckLogName(os.path.join(path, file))

                (candle, step, pileup_type, conditions, event_content) = jobID
                
                print "jobID: %s" % str(jobID)
                jobID = dict(list(zip(("candle", "step", "pileup_type", "conditions", "event_content"), jobID)))

                print "Dictionary based jobID %s: " % str(jobID)
            
                #if any of jobID fields except (isPILEUP) is empty we discard the job as all those are the jobID keys and we must have them
                discard = len([key for key, value in jobID.items() if key != "pileup_type" and not value])
                if discard:
                    print " ====================== The job HAS BEEN DISCARDED =============== "
                    print " NOT ALL DATA WAS AVAILABLE "
                    print " JOB ID = %s " % str(jobID)
                    print " ======================= end ===================================== "
                    continue
            
                # add to the list to generate the readable filename :)
                steps[step] = 1
                candles[candle.upper()] = 1
                if pileup_type=="":
                    pileups["NoPileUp"]=1
                else:
                    pileups[pileup_type] = 1
                
                memerror = getMemcheckError(path)

                MemcheckReport = {
                    "jobID": jobID,
                    "release": release,
                    "memcheck_errors": {"error_num": memerror},
                    "metadata": {"testname": "Memcheck"},
                    }

                # export to xml: actualy exporting gets suspended and put into runinfo
                exportMemcheckReport(path, MemcheckReport, runinfo)

def cmsPerfSuiteHarvest.process_timesize_dir	(		path,
			runinfo
	)

Definition at line 191 of file cmsPerfSuiteHarvest.py.

References _eventContent_DEBUG(), cmsPerfStripChart.dict, Exception, exportTimeSizeJob(), parserEdmSize.getEdmReport(), FileNamesHelper.getJobID_fromTimeReportLogName(), FileNamesHelper.getRootFileSize(), list(), parserTimingReport.loadTimeLog(), genParticles_cff.map, parserTimingReport.processModuleTimeLogData(), FileNamesHelper.read_ConfigurationFromSimulationCandles(), FileNamesHelper.read_SimulationCandles(), harvestTrackValidationPlots.str, and ComparisonHelper.zip().

Referenced by searchTimeSizeFiles().

def process_timesize_dir(path, runinfo):
    global release,event_content,conditions
    """ if the release is not provided explicitly we take it from the Simulation candles file """
    if (not release):
        release_fromlogfile = read_SimulationCandles(path)
        release  = release_fromlogfile 
        print "release from simulation candles: %s" % release
    
    if (not release):
        # TODO: raise exception!
        raise Exception("the release was not found!")


    """ process the TimingReport log files """

        # get the file list 
    files = os.listdir(path)
    timing_report_files = [os.path.join(path, f) for f in files
                 if test_timing_report_log.search(f) 
                    and os.path.isfile(os.path.join(path, f)) ]

    # print timing_report_files
    for timelog_f in timing_report_files:
        print "\nProcessing file: %s" % timelog_f
        print "------- "
        
        jobID = getJobID_fromTimeReportLogName(os.path.join(path, timelog_f))
        print "jobID: %s" % str(jobID)
        (candle, step, pileup_type, conditions, event_content) = jobID
        jobID = dict(list(zip(("candle", "step", "pileup_type", "conditions", "event_content"), jobID)))
        print "Dictionary based jobID %s: " % str(jobID)
        
        #if any of jobID fields except (isPILEUP) is empty we discard the job as all those are the jobID keys and we must have them
        discard = len([key for key, value in jobID.items() if key != "pileup_type" and not value])
        if discard:
            print " ====================== The job HAS BEEN DISCARDED =============== "
            print " NOT ALL DATA WAS AVAILABLE "
            print " JOB ID = %s " % str(jobID)
            print " ======================= end ===================================== "
            continue

        # TODO: automaticaly detect type of report file!!!
        (mod_timelog, evt_timelog, rss_data, vsize_data) =loadTimeLog(timelog_f)
    
        mod_timelog= processModuleTimeLogData(mod_timelog, groupBy = "module_name")
        print "Number of modules grouped by (module_label+module_name): %s" % len(mod_timelog)

        # add to the list to generate the readable filename :)
        steps[step] = 1
        candles[candle] = 1
                if pileup_type=="":
                    pileups["NoPileUp"]=1
                else:
                    pileups[pileup_type] = 1
    
        # root file size (number)
                root_file_size = getRootFileSize(path = path, candle = candle, step = step.replace(':', '='))
                # number of events
                num_events = read_ConfigurationFromSimulationCandles(path = path, step = step, is_pileup = pileup_type)["num_events"]

        #EdmSize
        edm_report = parserEdmSize.getEdmReport(path = path, candle = candle, step = step)
        if edm_report != False:
            try:
                # add event content data
                edm_report  = map(assign_event_content_for_product, edm_report)
                # for testing / imformation
                _eventContent_DEBUG(edm_report)
            except Exception as e:
                print e

        timeSizeReport = {
                "jobID":jobID,
                "release": release,
                                "timelog_result": (mod_timelog, evt_timelog, rss_data, vsize_data), 
                "metadata": {"testname": "TimeSize", "root_file_size": root_file_size, "num_events": num_events}, 
                "edmSize_result": edm_report 
        }
        
        # export to xml: actualy exporting gets suspended and put into runinfo
        exportTimeSizeJob(path, timeSizeReport,  runinfo)

def cmsPerfSuiteHarvest.searchIgProfFiles

(

runinfo

)

so far we will use the current dir to search in 

Definition at line 499 of file cmsPerfSuiteHarvest.py.

References process_igprof_dir().

def searchIgProfFiles(runinfo):
    """ so far we will use the current dir to search in """
    path = os.getcwd()
    #print path
    print 'full path =', os.path.abspath(path)

    files = os.listdir(path)
    
    test_IgProfDirs = re.compile("_IgProf(.*)$", re.IGNORECASE)          
    igprof_dirs = [os.path.join(path, f) for f in files if test_IgProfDirs.search(f) and os.path.isdir(os.path.join(path, f))]
    
    for igprof_dir in igprof_dirs:
        print igprof_dir
        process_igprof_dir(igprof_dir, runinfo)

def cmsPerfSuiteHarvest.searchMemcheckFiles

(

runinfo

)

so far we will use the current dir to search in 

Definition at line 483 of file cmsPerfSuiteHarvest.py.

References process_memcheck_dir().

def searchMemcheckFiles(runinfo):
    """ so far we will use the current dir to search in """
    path = os.getcwd()
    #print path
    print 'full path =', os.path.abspath(path)

    files = os.listdir(path)
    
    test_MemcheckDirs = re.compile("_Memcheck(.*)$", re.IGNORECASE)          
    memcheck_dirs = [os.path.join(path, f) for f in files if test_MemcheckDirs.search(f) and os.path.isdir(os.path.join(path, f))]
    
    for memcheck_dir in memcheck_dirs:
        print memcheck_dir
        process_memcheck_dir(memcheck_dir, runinfo)

#IgProf

def cmsPerfSuiteHarvest.searchTimeSizeFiles

(

runinfo

)

so far we will use the current dir to search in 

Definition at line 467 of file cmsPerfSuiteHarvest.py.

References process_timesize_dir().

def searchTimeSizeFiles(runinfo):
    """ so far we will use the current dir to search in """
    path = os.getcwd()
    #print path
    print 'full path =', os.path.abspath(path)

    files = os.listdir(path)
    
    test_timeSizeDirs = re.compile("_TimeSize$", re.IGNORECASE)          
    timesize_dirs = [os.path.join(path, f) for f in files if test_timeSizeDirs.search(f) and os.path.isdir(os.path.join(path, f))]
    
    for timesize_dir in timesize_dirs:
        # print timesize_dir
        process_timesize_dir(timesize_dir, runinfo)

#Memcheck

def cmsPerfSuiteHarvest.usage

(

argv

)

Definition at line 31 of file cmsPerfSuiteHarvest.py.

def usage(argv):
    script = argv[0]
    return """
    Usage: %(script)s [-v cmssw_version] [--version=cmssw_version]
    
    if the cmssw version is in the system's environment (after running cmsenv):
    $ %(script)s 
    
    otherwise one must specify the cmssw version:
    $ %(script)s --version=CMSSW_3_2_0
    $ %(script)s -v CMSSW_3_2_0    
    
    """ % locals()

Variable Documentation

cmsPerfSuiteHarvest._TEST_RUN

private

Definition at line 17 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.candles

Definition at line 28 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.eventContentRules

Definition at line 583 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.EventContents_OK

Definition at line 549 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.file_name

Definition at line 593 of file cmsPerfSuiteHarvest.py.

Referenced by HcalLutManager.create_lut_loader(), SiStripHistoPlotter.createStaticPlot(), DTTPGLutFile.open(), L1TriggerLutFile.open(), TEcnaRead.ReadAverageHighFrequencyNoise(), TEcnaRead.ReadAverageLowFrequencyNoise(), TEcnaRead.ReadAverageMeanCorrelationsBetweenSamples(), TEcnaRead.ReadAveragePedestals(), TEcnaRead.ReadAverageSigmaOfCorrelationsBetweenSamples(), TEcnaRead.ReadAverageTotalNoise(), TEcnaRead.ReadCorrelationsBetweenSamples(), TEcnaRead.ReadCovariancesBetweenSamples(), TEcnaRead.ReadHighFrequencyCorrelationsBetweenChannels(), TEcnaRead.ReadHighFrequencyCovariancesBetweenChannels(), TEcnaRead.ReadHighFrequencyMeanCorrelationsBetweenStins(), TEcnaRead.ReadHighFrequencyNoise(), TEcnaRead.ReadLowFrequencyCorrelationsBetweenChannels(), TEcnaRead.ReadLowFrequencyCovariancesBetweenChannels(), TEcnaRead.ReadLowFrequencyMeanCorrelationsBetweenStins(), TEcnaRead.ReadLowFrequencyNoise(), TEcnaRead.ReadMeanCorrelationsBetweenSamples(), TEcnaRead.ReadNumberOfEventsForSamples(), TEcnaRead.ReadPedestals(), TEcnaRead.ReadRelevantCorrelationsBetweenSamples(), TEcnaRead.ReadRootFileHeader(), TEcnaRead.ReadSampleAdcValues(), TEcnaRead.ReadSampleAdcValuesSameFile(), TEcnaRead.ReadSampleMeans(), TEcnaRead.ReadSampleSigmas(), TEcnaRead.ReadSigmaOfCorrelationsBetweenSamples(), TEcnaRead.ReadStinNumbers(), TEcnaRead.ReadTotalNoise(), SummaryOutputProducer.writeCSV_module(), SummaryOutputProducer.writeCSV_trigger(), SummaryOutputProducer.writeHistograms(), and TEcnaRun.WriteRootFile().

cmsPerfSuiteHarvest.now

Definition at line 590 of file cmsPerfSuiteHarvest.py.

Referenced by edm::CountAndLimit.add(), lumi::RevisionDML.addEntry(), lumi::RevisionDML.addRevision(), lumi::RevisionDML.addRunToCurrentHFDataTag(), dqmservices::DQMFileIterator.advanceToLumi(), HcalLuttoDB.beginJob(), dqm::RamdiskMonitor.beginLuminosityBlock(), RawEventOutputModuleForBU< Consumer >.beginLuminosityBlock(), edm::service::CondorStatusService.beginPost(), condbon.cdbon_write(), XrdAdaptor::RequestManager.checkSourcesImpl(), dqmservices::DQMFileIterator.collect(), edm::service::ConcurrentModuleTimer.ConcurrentModuleTimer(), dqmservices::DQMMonitoringService.DQMMonitoringService(), DTHVStatusHandler.dumpSnapshot(), XrdAdaptor::XrdReadStatistics.elapsedNS(), dqmservices::DQMMonitoringService.evLumi(), DaqFakeReader.fillRawData(), Fit.getFitQuality(), XrdAdaptor::RequestManager.getInstance(), DTHVStatusHandler.getNewObjects(), XrdAdaptor::RequestManager.handle(), XrdAdaptor::RequestManager::OpenHandler.HandleResponseWithHosts(), HtrXmlPatternWriter.HtrXmlPatternWriter(), cond::persistency.importPayload(), edm::InputSource.InputSource(), cond::persistency::IOVEditor.insert(), edm::service::CondorStatusService.lastUpdate(), DQMNet.logme(), edm::InputSource.lumiLimitReached(), FastTimerService::Measurement.measure(), FastTimerService::Measurement.measure_and_accumulate(), FastTimerService::Measurement.measure_and_store(), Fit.multiplyMatrices(), operator<<(), dqmservices::DQMMonitoringService.outputLumiUpdate(), dqmservices::DQMMonitoringService.outputUpdate(), edm::service::StallMonitor.postBeginJob(), ThroughputService.postEvent(), edm::service::StallMonitor.postEvent(), edm::service::StallMonitor.postEventReadFromSource(), edm::service::StallMonitor.postModuleEvent(), edm::service::StallMonitor.postModuleEventPrefetching(), edm::service::StallMonitor.postSourceEvent(), ThroughputService.preallocate(), edm::service::StallMonitor.preEvent(), edm::service::StallMonitor.preEventReadFromSource(), edm::service::StallMonitor.preModuleEvent(), ThroughputService.preSourceEvent(), edm::service::StallMonitor.preSourceEvent(), HGCalClusterProducer.produce(), Fit.PropagateErrors(), XrdFile.readv(), FedRawDataInputSource.readWorker(), XrdAdaptor::RequestManager.requestFailure(), dqmservices::DQMFileIterator.reset(), lumi::CMSRunSummaryDummy2DB.retrieveData(), DQMNet.run(), ALIUtils.set_time_now(), XrdAdaptor::Source.setLastDowngrade(), edm::TimeoutPoolOutputModule.shouldWeCloseFile(), XrdAdaptor::Source.Source(), edm::service::ConcurrentModuleTimer.start(), edm::service::ConcurrentModuleTimer.stop(), StorageAccount::Stamp.tick(), dqmservices::DQMMonitoringService.tryUpdate(), edm::service::CondorStatusService.update(), dqmservices::DQMFileIterator.update_state(), SiStripPayloadHandler< SiStripPayload >.updateConfigMap(), and edm::service::CondorStatusService.updateImpl().

cmsPerfSuiteHarvest.output_dir

Definition at line 533 of file cmsPerfSuiteHarvest.py.

Referenced by L1TCSCTFClient.dqmEndJob(), and L1TCSCTFClient.initialize().

cmsPerfSuiteHarvest.p

Definition at line 543 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.pileups

Definition at line 29 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.release

Definition at line 26 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.run_info

Definition at line 544 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.Sequences_OK

Definition at line 548 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.steps

Definition at line 27 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.test_igprof_report_log

Definition at line 21 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.test_memcheck_report_log

Definition at line 22 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.test_timing_report_log

Definition at line 20 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.xmldoc

Definition at line 25 of file cmsPerfSuiteHarvest.py.

Referenced by emtf::Tree.loadFromXML(), and emtf::Tree.saveToXML().