dqm_interfaces.DirWalkerFile Class Reference

Inheritance diagram for dqm_interfaces.DirWalkerFile:

Public Member Functions
def	__del__ (self)
def	__init__ (self, name, topdirname, rootfilename1, rootfilename2, run=-1, black_list=[], stat_test="KS", test_threshold=.5, draw_success=True, do_pngs=False, black_list_histos=[])
def	cd (self, directory_name, on_disk=False, regexp=False)
def	getObjs (self, name)
def	ls (self, directory_name="")
def	walk (self)

Public Attributes
	black_list
	black_list_histos
	different_histograms
	directory
	dqmrootfile1
	dqmrootfile2
	filename1
	filename2
	name
	run
	stat_test
	workdir

Private Member Functions
def	__fill_single_dir (self, dir_name, directory, mother_name="", depth=0)

Detailed Description

Definition at line 545 of file dqm_interfaces.py.

Constructor & Destructor Documentation

__init__()

def dqm_interfaces.DirWalkerFile.__init__	(		self,
			name,
			topdirname,
			rootfilename1,
			rootfilename2,
			run = -1,
			black_list = [],
			stat_test = "KS",
			test_threshold = .5,
			draw_success = True,
			do_pngs = False,
			black_list_histos = []
	)

Definition at line 546 of file dqm_interfaces.py.

    def __init__(self, name, topdirname,rootfilename1, rootfilename2, run=-1, black_list=[], stat_test="KS", test_threshold=.5,draw_success=True,do_pngs=False, black_list_histos=[]):
        self.name=name
        self.dqmrootfile1=DQMRootFile(abspath(rootfilename1))
        self.dqmrootfile2=DQMRootFile(abspath(rootfilename2))
        self.run=run
        self.stat_test=Statistical_Tests[stat_test](test_threshold)
        self.workdir=getcwd()
        self.black_list=black_list
        self.directory=Directory(topdirname)
        #print "DIRWALKERFILE %s %s" %(draw_success,do_pngs)
        self.directory.draw_success=draw_success
        self.directory.do_pngs=do_pngs
        self.black_list_histos = black_list_histos
        self.different_histograms = {}
        self.filename1 = basename(rootfilename2)
        self.filename2 = basename(rootfilename1)

__del__()

def dqm_interfaces.DirWalkerFile.__del__

(

self

)

Definition at line 563 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.workdir.

    def __del__(self):
        chdir(self.workdir)

Member Function Documentation

__fill_single_dir()

def dqm_interfaces.DirWalkerFile.__fill_single_dir (   self,   dir_name,   directory,   mother_name = "",   depth = 0  )

private

Definition at line 634 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.walk().

    def __fill_single_dir(self,dir_name,directory,mother_name="",depth=0):
        #print "MOTHER NAME  = +%s+" %mother_name
     #print "About to study %s (in dir %s)" %(dir_name,getcwd())

        # see if in black_list
        this_dir=DirID(dir_name,depth)
        #print "  ## this_dir: %s"%(this_dir)
        if this_dir in self.black_list: 
            #print "Directory %s skipped because black-listed" %dir_name
            return 0        

        depth+=1

        self.cd(dir_name)
        #if dir_name == 'HLTJETMET':
        #    print self.ls()

        #print "Test %s with thre %s" %(self.stat_test.name, self.stat_test.threshold)

        contents=self.ls()
        if depth==1:
            n_top_contents=len(contents)

        #print contents
        cont_counter=1
        comparisons=[]
        for name,obj_type in contents.items():
            if obj_type=="TDirectoryFile":        
                #We have a dir, launch recursion!
                #Some feedback on the progress
                if depth==1:
                    print("Studying directory %s, %s/%s" %(name,cont_counter,n_top_contents))
                    cont_counter+=1          

                #print "Studying directory",name
                # ok recursion on!
                subdir=Directory(name)
                subdir.draw_success=directory.draw_success
                subdir.do_pngs=directory.do_pngs
                self.__fill_single_dir(name,subdir,join(mother_name,dir_name),depth)
                if not subdir.is_empty():
                    if depth==1:
                        print(" ->Appending %s..." %name, end=' ')
                    directory.subdirs.append(subdir)
                    if depth==1:
                        print("Appended.")
            elif name != '':
                # We have probably an histo. Let's make the plot and the png.        
                if obj_type[:2]!="TH" and obj_type[:3]!="TPr" :
                    continue
                h1,h2=self.getObjs(name)
                #print "COMPARISON : +%s+%s+" %(mother_name,dir_name)
                path = join(mother_name,dir_name,name)
                if path in self.black_list_histos:
                    print("  Skipping %s" %(path))
                    directory.comparisons.append(Comparison(name,
                                        join(mother_name,dir_name),
                                        h1,h2,
                                        deepcopy(self.stat_test),
                                        draw_success=directory.draw_success,
                                        do_pngs=directory.do_pngs, skip=True))
                else:
                    directory.comparisons.append(Comparison(name,
                                          join(mother_name,dir_name),
                                          h1,h2,
                                          deepcopy(self.stat_test),
                                          draw_success=directory.draw_success,
                                          do_pngs=directory.do_pngs, skip=False))
                    directory.filename1 = self.filename1
                    directory.filename2 = self.filename2
                    directory.different_histograms['file1'] = self.different_histograms['file1']
                    directory.different_histograms['file2'] = self.different_histograms['file2']

        self.cd("..")

cd()

def dqm_interfaces.DirWalkerFile.cd	(		self,
			directory_name,
			on_disk = False,
			regexp = False
	)

Definition at line 566 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.dqmrootfile1, dqm_interfaces.DirWalkerFile.dqmrootfile2, relativeConstraints.keys, and dqm_interfaces.DirWalkerFile.ls().

Referenced by dqm_interfaces.DirWalkerFile.walk().

    def cd(self,directory_name, on_disk=False, regexp=False,):
        if regexp == True:
            if len(directory_name)!=0:
                if on_disk:
                    if not exists(directory_name):
                        makedirs(directory_name)
                        chdir(directory_name)  
                tmp = self.dqmrootfile2.ls().keys()
                for elem in tmp:
                    if "Run" in elem:
                        next_dir = elem
                self.dqmrootfile2.cd(next_dir)
                tmp = self.dqmrootfile1.ls().keys()
                for elem in tmp:
                    if "Run" in elem:
                        next_dir = elem
                self.dqmrootfile1.cd(next_dir)
        else:
            if len(directory_name)!=0:
                if on_disk:
                    if not exists(directory_name):
                        makedirs(directory_name)
                        chdir(directory_name)
                self.dqmrootfile2.cd(directory_name)
                self.dqmrootfile1.cd(directory_name)

getObjs()

def dqm_interfaces.DirWalkerFile.getObjs	(		self,
			name
	)

Definition at line 629 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.dqmrootfile1, and dqm_interfaces.DirWalkerFile.dqmrootfile2.

    def getObjs(self,name):
        h1=self.dqmrootfile1.getObj(name)
        h2=self.dqmrootfile2.getObj(name)
        return h1,h2

ls()

def dqm_interfaces.DirWalkerFile.ls	(		self,
			directory_name = ""
	)

Return common objects to the 2 files.

Definition at line 592 of file dqm_interfaces.py.

References dirstructure.Directory.different_histograms, dqm_interfaces.DirWalkerFile.different_histograms, dqm_interfaces.DirWalkerFile.dqmrootfile1, and dqm_interfaces.DirWalkerFile.dqmrootfile2.

Referenced by dqm_interfaces.DirWalkerFile.cd(), and dqm_interfaces.DirWalkerFile.walk().

    def ls(self,directory_name=""):
        """Return common objects to the 2 files.
        """
        contents1=self.dqmrootfile1.ls(directory_name)
        contents2=self.dqmrootfile2.ls(directory_name)
        #print "cont1: %s"%(contents1)
        #print "cont2: %s"%(contents2)
        contents={}
        self.different_histograms['file1']= {}
        self.different_histograms['file2']= {}
        keys = [key for key in contents2.keys() if key in contents1] #set of all possible contents from both files
        #print " ## keys: %s" %(keys)
        for key in keys:  #iterate on all unique keys
            if contents1[key]!=contents2[key]:
                diff_file1 = set(contents1.keys()) - set(contents2.keys()) #set of contents that file1 is missing
                diff_file2 = set(contents2.keys()) - set(contents1.keys()) #--'-- that file2 is missing
                for key1 in diff_file1:
                    obj_type = contents1[key1]
                    if obj_type == "TDirectoryFile":
                        self.different_histograms['file1'][key1] = contents1[key1] #if direcory
                        #print "\n Missing inside a dir: ", self.ls(key1)
                        #contents[key] = contents1[key1]
                    if obj_type[:2]!="TH" and obj_type[:3]!="TPr" : #if histogram
                        continue
                    self.different_histograms['file1'][key1] = contents1[key1]
                for key1 in diff_file2:
                    obj_type = contents2[key1]
                    if obj_type == "TDirectoryFile":
                        self.different_histograms['file2'][key1] = contents2[key1] #if direcory
                        #print "\n Missing inside a dir: ", self.ls(key1)
                        #contents[key] = contents2[key1]
                    if obj_type[:2]!="TH" and obj_type[:3]!="TPr" : #if histogram
                        continue
                    self.different_histograms['file2'][key1] = contents2[key1]
            contents[key]=contents1[key]
        return contents

walk()

def dqm_interfaces.DirWalkerFile.walk

(

self

)

Definition at line 709 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.__fill_single_dir(), dqm::implementation::NavigatorBase.cd(), TFileDirectory.cd(), TFileService.cd(), dqm_interfaces.DQMcommunicator.cd(), Folder.cd(), mkfit::ConfigJsonPatcher.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMRootFile.cd(), dqm_interfaces.DirWalkerFile.cd(), dqm::implementation::DQMStore.cd(), Book.directory, edmIntegrityCheck.IntegrityCheck.directory, dqm_interfaces.DirFetcher.directory, dqm_interfaces.DirWalkerDB.directory, dqm_interfaces.DirWalkerFile.directory, ALCARECOTkAlBeamHalo_cff.filter, createfilelist.int, relativeConstraints.keys, WZInterestingEventSelector::event.ls, BeamProfile2DBReader::TheBSfromDB.ls, BeamProfileHLLHC2DBReader::TheBSfromDB.ls, BeamSpotRcdReader::theBSfromDB.ls, BeamSpotOnlineRecordsReader::theBSOfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, dqm_interfaces.DQMcommunicator.ls(), dqm_interfaces.DQMRootFile.ls(), dqm_interfaces.DirWalkerFile.ls(), print(), uploadPayloads.FileObj.run, uploader.FileObj.run, RPCRunIOV::RunIOV_Item.run, splitter.FileObj.run, beamspot::BeamSpotContainer.run, TB06Reco.run, TB06RecoH2.run, RPCDQMObject.run, MultiEventFilter::Event.run, L1Analysis::L1AnalysisEventDataFormat.run, EfficiencyPlotter.run, lumi::Lumi2DB::LumiSource.run, generateEDF.LumiInfo.run, WZInterestingEventSelector::event.run, MuonRecoTest.run, Point.run, MatacqProducer::MatacqEventId.run, BeamProfile2DBReader::TheBSfromDB.run, DTResolutionAnalysisTest.run, BeamProfileHLLHC2DBReader::TheBSfromDB.run, options.ConnectionHLTMenu.run, BeamSpotRcdReader::theBSfromDB.run, DTDataIntegrityTest.run, BeamSpotOnlineRecordsReader::theBSOfromDB.run, DTBlockedROChannelsTest.run, DTChamberEfficiencyTest.run, DTResolutionTest.run, DTEfficiencyTest.run, ME::Header.run, personalPlayback.Playback.run, DQMNet::CoreObject.run, DTLocalTriggerBaseTest.run, MatrixUtil.InputInfo.run, EcalDeadCellDeltaRFilter.run, HitEff.run, cond::RunInfo_t.run, SiPixelErrorEstimation.run, personalPlayback.FrameworkJob.run, l1ct::Event.run, dqm_interfaces.DirWalkerFile.run, JsonOutputProducer::JsonEvent.run, SummaryOutputProducer.run, and SummaryOutputProducer::Event.run.

    def walk(self):
        # Build the top dir in the rootfile first
        rundir=""
        if self.run<0:
            # change dir in the first one...
            #print  self.ls().keys()
            first_run_dir = ""
            try:
                first_run_dir = list(filter(lambda k: "Run " in k, self.ls().keys()))[0]
            except:
                print("\nRundir not there: Is this a generic rootfile?\n")
            rundir=first_run_dir
            try:
                self.run= int(rundir.split(" ")[1])
            except:
                print("Setting run number to 0")
                self.run= 0
        else:
            rundir="Run %s"%self.run

        try:
            self.cd(rundir, False, True) #True -> for checking the Rundir in case of different runs
        except:
            print("\nRundir not there: Is this a generic rootfile?\n")

        # Let's rock!
        self.__fill_single_dir(self.directory.name,self.directory)
        print("Finished")
        n_left_threads=len(tcanvas_print_processes)
        if n_left_threads>0:
            print("Waiting for %s threads to finish..." %n_left_threads)
            for p in tcanvas_print_processes:
                p.join()  

#-------------------------------------------------------------------------------

Member Data Documentation

black_list

dqm_interfaces.DirWalkerFile.black_list

Definition at line 553 of file dqm_interfaces.py.

black_list_histos

dqm_interfaces.DirWalkerFile.black_list_histos

Definition at line 558 of file dqm_interfaces.py.

different_histograms

dqm_interfaces.DirWalkerFile.different_histograms

Definition at line 559 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.ls().

directory

dqm_interfaces.DirWalkerFile.directory

Definition at line 554 of file dqm_interfaces.py.

Referenced by utils.unpickler.__init__(), utils.unpickler.run(), and dqm_interfaces.DirWalkerFile.walk().

dqmrootfile1

dqm_interfaces.DirWalkerFile.dqmrootfile1

Definition at line 548 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.cd(), dqm_interfaces.DirWalkerFile.getObjs(), and dqm_interfaces.DirWalkerFile.ls().

dqmrootfile2

dqm_interfaces.DirWalkerFile.dqmrootfile2

Definition at line 549 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.cd(), dqm_interfaces.DirWalkerFile.getObjs(), and dqm_interfaces.DirWalkerFile.ls().

filename1

dqm_interfaces.DirWalkerFile.filename1

Definition at line 560 of file dqm_interfaces.py.

filename2

dqm_interfaces.DirWalkerFile.filename2

Definition at line 561 of file dqm_interfaces.py.

name

dqm_interfaces.DirWalkerFile.name

Definition at line 547 of file dqm_interfaces.py.

Referenced by config.CFG.__str__(), validation.Sample.digest(), utils.StatisticalTest.get_status(), and VIDSelectorBase.VIDSelectorBase.initialize().

run

dqm_interfaces.DirWalkerFile.run

Definition at line 550 of file dqm_interfaces.py.

Referenced by DTWorkflow.DTWorkflow.all(), Types.EventID.cppID(), Types.LuminosityBlockID.cppID(), o2olib.O2OTool.execute(), and dqm_interfaces.DirWalkerFile.walk().

stat_test

dqm_interfaces.DirWalkerFile.stat_test

Definition at line 551 of file dqm_interfaces.py.

workdir

dqm_interfaces.DirWalkerFile.workdir

Definition at line 552 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.__del__().