CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
List of all members | Public Member Functions | Public Attributes | Private Member Functions
dqm_interfaces.DirWalkerFile Class Reference
Inheritance diagram for dqm_interfaces.DirWalkerFile:

Public Member Functions

def __del__ (self)
 
def __init__ (self, name, topdirname, rootfilename1, rootfilename2, run=-1, black_list=[], stat_test="KS", test_threshold=.5, draw_success=True, do_pngs=False, black_list_histos=[])
 
def cd (self, directory_name, on_disk=False, regexp=False)
 
def getObjs (self, name)
 
def ls (self, directory_name="")
 
def walk (self)
 

Public Attributes

 black_list
 
 black_list_histos
 
 different_histograms
 
 directory
 
 dqmrootfile1
 
 dqmrootfile2
 
 filename1
 
 filename2
 
 name
 
 run
 
 stat_test
 
 workdir
 

Private Member Functions

def __fill_single_dir (self, dir_name, directory, mother_name="", depth=0)
 

Detailed Description

Definition at line 545 of file dqm_interfaces.py.

Constructor & Destructor Documentation

◆ __init__()

def dqm_interfaces.DirWalkerFile.__init__ (   self,
  name,
  topdirname,
  rootfilename1,
  rootfilename2,
  run = -1,
  black_list = [],
  stat_test = "KS",
  test_threshold = .5,
  draw_success = True,
  do_pngs = False,
  black_list_histos = [] 
)

Definition at line 546 of file dqm_interfaces.py.

546  def __init__(self, name, topdirname,rootfilename1, rootfilename2, run=-1, black_list=[], stat_test="KS", test_threshold=.5,draw_success=True,do_pngs=False, black_list_histos=[]):
547  self.name=name
548  self.dqmrootfile1=DQMRootFile(abspath(rootfilename1))
549  self.dqmrootfile2=DQMRootFile(abspath(rootfilename2))
550  self.run=run
551  self.stat_test=Statistical_Tests[stat_test](test_threshold)
552  self.workdir=getcwd()
553  self.black_list=black_list
554  self.directory=Directory(topdirname)
555  #print "DIRWALKERFILE %s %s" %(draw_success,do_pngs)
556  self.directory.draw_success=draw_success
557  self.directory.do_pngs=do_pngs
558  self.black_list_histos = black_list_histos
559  self.different_histograms = {}
560  self.filename1 = basename(rootfilename2)
561  self.filename2 = basename(rootfilename1)
562 
def __init__(self, dataset, job_number, job_id, job_name, isDA, isMC, applyBOWS, applyEXTRACOND, extraconditions, runboundary, lumilist, intlumi, maxevents, gt, allFromGT, alignmentDB, alignmentTAG, apeDB, apeTAG, bowDB, bowTAG, vertextype, tracktype, refittertype, ttrhtype, applyruncontrol, ptcut, CMSSW_dir, the_dir)
abspath
Definition: dataset.py:55

◆ __del__()

def dqm_interfaces.DirWalkerFile.__del__ (   self)

Definition at line 563 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.workdir.

563  def __del__(self):
564  chdir(self.workdir)
565 

Member Function Documentation

◆ __fill_single_dir()

def dqm_interfaces.DirWalkerFile.__fill_single_dir (   self,
  dir_name,
  directory,
  mother_name = "",
  depth = 0 
)
private

Definition at line 634 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.walk().

634  def __fill_single_dir(self,dir_name,directory,mother_name="",depth=0):
635  #print "MOTHER NAME = +%s+" %mother_name
636  #print "About to study %s (in dir %s)" %(dir_name,getcwd())
637 
638  # see if in black_list
639  this_dir=DirID(dir_name,depth)
640  #print " ## this_dir: %s"%(this_dir)
641  if this_dir in self.black_list:
642  #print "Directory %s skipped because black-listed" %dir_name
643  return 0
644 
645  depth+=1
646 
647  self.cd(dir_name)
648  #if dir_name == 'HLTJETMET':
649  # print self.ls()
650 
651  #print "Test %s with thre %s" %(self.stat_test.name, self.stat_test.threshold)
652 
653  contents=self.ls()
654  if depth==1:
655  n_top_contents=len(contents)
656 
657  #print contents
658  cont_counter=1
659  comparisons=[]
660  for name,obj_type in contents.items():
661  if obj_type=="TDirectoryFile":
662  #We have a dir, launch recursion!
663  #Some feedback on the progress
664  if depth==1:
665  print("Studying directory %s, %s/%s" %(name,cont_counter,n_top_contents))
666  cont_counter+=1
667 
668  #print "Studying directory",name
669  # ok recursion on!
670  subdir=Directory(name)
671  subdir.draw_success=directory.draw_success
672  subdir.do_pngs=directory.do_pngs
673  self.__fill_single_dir(name,subdir,join(mother_name,dir_name),depth)
674  if not subdir.is_empty():
675  if depth==1:
676  print(" ->Appending %s..." %name, end=' ')
677  directory.subdirs.append(subdir)
678  if depth==1:
679  print("Appended.")
680  elif name != '':
681  # We have probably an histo. Let's make the plot and the png.
682  if obj_type[:2]!="TH" and obj_type[:3]!="TPr" :
683  continue
684  h1,h2=self.getObjs(name)
685  #print "COMPARISON : +%s+%s+" %(mother_name,dir_name)
686  path = join(mother_name,dir_name,name)
687  if path in self.black_list_histos:
688  print(" Skipping %s" %(path))
689  directory.comparisons.append(Comparison(name,
690  join(mother_name,dir_name),
691  h1,h2,
692  deepcopy(self.stat_test),
693  draw_success=directory.draw_success,
694  do_pngs=directory.do_pngs, skip=True))
695  else:
696  directory.comparisons.append(Comparison(name,
697  join(mother_name,dir_name),
698  h1,h2,
699  deepcopy(self.stat_test),
700  draw_success=directory.draw_success,
701  do_pngs=directory.do_pngs, skip=False))
702  directory.filename1 = self.filename1
703  directory.filename2 = self.filename2
704  directory.different_histograms['file1'] = self.different_histograms['file1']
705  directory.different_histograms['file2'] = self.different_histograms['file2']
706 
707  self.cd("..")
708 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
static std::string join(char **cmd)
Definition: RemoteFile.cc:19

◆ cd()

def dqm_interfaces.DirWalkerFile.cd (   self,
  directory_name,
  on_disk = False,
  regexp = False 
)

Definition at line 566 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.dqmrootfile1, dqm_interfaces.DirWalkerFile.dqmrootfile2, relativeConstraints.keys, and dqm_interfaces.DirWalkerFile.ls().

Referenced by dqm_interfaces.DirWalkerFile.walk().

566  def cd(self,directory_name, on_disk=False, regexp=False,):
567  if regexp == True:
568  if len(directory_name)!=0:
569  if on_disk:
570  if not exists(directory_name):
571  makedirs(directory_name)
572  chdir(directory_name)
573  tmp = self.dqmrootfile2.ls().keys()
574  for elem in tmp:
575  if "Run" in elem:
576  next_dir = elem
577  self.dqmrootfile2.cd(next_dir)
578  tmp = self.dqmrootfile1.ls().keys()
579  for elem in tmp:
580  if "Run" in elem:
581  next_dir = elem
582  self.dqmrootfile1.cd(next_dir)
583  else:
584  if len(directory_name)!=0:
585  if on_disk:
586  if not exists(directory_name):
587  makedirs(directory_name)
588  chdir(directory_name)
589  self.dqmrootfile2.cd(directory_name)
590  self.dqmrootfile1.cd(directory_name)
591 
def ls(path, rec=False)
Definition: eostools.py:349

◆ getObjs()

def dqm_interfaces.DirWalkerFile.getObjs (   self,
  name 
)

Definition at line 629 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.dqmrootfile1, and dqm_interfaces.DirWalkerFile.dqmrootfile2.

629  def getObjs(self,name):
630  h1=self.dqmrootfile1.getObj(name)
631  h2=self.dqmrootfile2.getObj(name)
632  return h1,h2
633 

◆ ls()

def dqm_interfaces.DirWalkerFile.ls (   self,
  directory_name = "" 
)
Return common objects to the 2 files.

Definition at line 592 of file dqm_interfaces.py.

References dirstructure.Directory.different_histograms, dqm_interfaces.DirWalkerFile.different_histograms, dqm_interfaces.DirWalkerFile.dqmrootfile1, and dqm_interfaces.DirWalkerFile.dqmrootfile2.

Referenced by dqm_interfaces.DirWalkerFile.cd(), and dqm_interfaces.DirWalkerFile.walk().

592  def ls(self,directory_name=""):
593  """Return common objects to the 2 files.
594  """
595  contents1=self.dqmrootfile1.ls(directory_name)
596  contents2=self.dqmrootfile2.ls(directory_name)
597  #print "cont1: %s"%(contents1)
598  #print "cont2: %s"%(contents2)
599  contents={}
600  self.different_histograms['file1']= {}
601  self.different_histograms['file2']= {}
602  keys = [key for key in contents2.keys() if key in contents1] #set of all possible contents from both files
603  #print " ## keys: %s" %(keys)
604  for key in keys: #iterate on all unique keys
605  if contents1[key]!=contents2[key]:
606  diff_file1 = set(contents1.keys()) - set(contents2.keys()) #set of contents that file1 is missing
607  diff_file2 = set(contents2.keys()) - set(contents1.keys()) #--'-- that file2 is missing
608  for key1 in diff_file1:
609  obj_type = contents1[key1]
610  if obj_type == "TDirectoryFile":
611  self.different_histograms['file1'][key1] = contents1[key1] #if direcory
612  #print "\n Missing inside a dir: ", self.ls(key1)
613  #contents[key] = contents1[key1]
614  if obj_type[:2]!="TH" and obj_type[:3]!="TPr" : #if histogram
615  continue
616  self.different_histograms['file1'][key1] = contents1[key1]
617  for key1 in diff_file2:
618  obj_type = contents2[key1]
619  if obj_type == "TDirectoryFile":
620  self.different_histograms['file2'][key1] = contents2[key1] #if direcory
621  #print "\n Missing inside a dir: ", self.ls(key1)
622  #contents[key] = contents2[key1]
623  if obj_type[:2]!="TH" and obj_type[:3]!="TPr" : #if histogram
624  continue
625  self.different_histograms['file2'][key1] = contents2[key1]
626  contents[key]=contents1[key]
627  return contents
628 
def ls(path, rec=False)
Definition: eostools.py:349

◆ walk()

def dqm_interfaces.DirWalkerFile.walk (   self)

Definition at line 709 of file dqm_interfaces.py.

References dqm_interfaces.DirWalkerFile.__fill_single_dir(), dqm::implementation::NavigatorBase.cd(), TFileDirectory.cd(), TFileService.cd(), dqm_interfaces.DQMcommunicator.cd(), Folder.cd(), mkfit::ConfigJsonPatcher.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMRootFile.cd(), dqm_interfaces.DirWalkerFile.cd(), dqm::implementation::DQMStore.cd(), Book.directory, edmIntegrityCheck.IntegrityCheck.directory, dqm_interfaces.DirFetcher.directory, dqm_interfaces.DirWalkerDB.directory, dqm_interfaces.DirWalkerFile.directory, ALCARECOTkAlBeamHalo_cff.filter, createfilelist.int, relativeConstraints.keys, WZInterestingEventSelector::event.ls, BeamSpotRcdReader::theBSfromDB.ls, BeamSpotOnlineRecordsReader::theBSOfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, dqm_interfaces.DQMcommunicator.ls(), dqm_interfaces.DQMRootFile.ls(), dqm_interfaces.DirWalkerFile.ls(), print(), uploader.FileObj.run, splitter.FileObj.run, uploadPayloads.FileObj.run, RPCRunIOV::RunIOV_Item.run, beamspot::BeamSpotContainer.run, TB06RecoH2.run, RPCDQMObject.run, TB06Reco.run, MultiEventFilter::Event.run, L1Analysis::L1AnalysisEventDataFormat.run, EfficiencyPlotter.run, lumi::Lumi2DB::LumiSource.run, generateEDF.LumiInfo.run, WZInterestingEventSelector::event.run, MuonRecoTest.run, MatacqProducer::MatacqEventId.run, Point.run, DTResolutionAnalysisTest.run, options.ConnectionHLTMenu.run, BeamSpotRcdReader::theBSfromDB.run, DTDataIntegrityTest.run, BeamSpotOnlineRecordsReader::theBSOfromDB.run, DTBlockedROChannelsTest.run, DTResolutionTest.run, DTChamberEfficiencyTest.run, DTEfficiencyTest.run, ME::Header.run, personalPlayback.Playback.run, DQMNet::CoreObject.run, DTLocalTriggerBaseTest.run, MatrixUtil.InputInfo.run, EcalDeadCellDeltaRFilter.run, HitEff.run, cond::RunInfo_t.run, SiPixelErrorEstimation.run, l1ct::Event.run, personalPlayback.FrameworkJob.run, dqm_interfaces.DirWalkerFile.run, JsonOutputProducer::JsonEvent.run, SummaryOutputProducer.run, and SummaryOutputProducer::Event.run.

709  def walk(self):
710  # Build the top dir in the rootfile first
711  rundir=""
712  if self.run<0:
713  # change dir in the first one...
714  #print self.ls().keys()
715  first_run_dir = ""
716  try:
717  first_run_dir = list(filter(lambda k: "Run " in k, self.ls().keys()))[0]
718  except:
719  print("\nRundir not there: Is this a generic rootfile?\n")
720  rundir=first_run_dir
721  try:
722  self.run= int(rundir.split(" ")[1])
723  except:
724  print("Setting run number to 0")
725  self.run= 0
726  else:
727  rundir="Run %s"%self.run
728 
729  try:
730  self.cd(rundir, False, True) #True -> for checking the Rundir in case of different runs
731  except:
732  print("\nRundir not there: Is this a generic rootfile?\n")
733 
734  # Let's rock!
735  self.__fill_single_dir(self.directory.name,self.directory)
736  print("Finished")
737  n_left_threads=len(tcanvas_print_processes)
738  if n_left_threads>0:
739  print("Waiting for %s threads to finish..." %n_left_threads)
740  for p in tcanvas_print_processes:
741  p.join()
742 
743 #-------------------------------------------------------------------------------
744 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

Member Data Documentation

◆ black_list

dqm_interfaces.DirWalkerFile.black_list

Definition at line 553 of file dqm_interfaces.py.

◆ black_list_histos

dqm_interfaces.DirWalkerFile.black_list_histos

Definition at line 558 of file dqm_interfaces.py.

◆ different_histograms

dqm_interfaces.DirWalkerFile.different_histograms

Definition at line 559 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.ls().

◆ directory

dqm_interfaces.DirWalkerFile.directory

◆ dqmrootfile1

dqm_interfaces.DirWalkerFile.dqmrootfile1

◆ dqmrootfile2

dqm_interfaces.DirWalkerFile.dqmrootfile2

◆ filename1

dqm_interfaces.DirWalkerFile.filename1

Definition at line 560 of file dqm_interfaces.py.

◆ filename2

dqm_interfaces.DirWalkerFile.filename2

Definition at line 561 of file dqm_interfaces.py.

◆ name

dqm_interfaces.DirWalkerFile.name

◆ run

dqm_interfaces.DirWalkerFile.run

◆ stat_test

dqm_interfaces.DirWalkerFile.stat_test

Definition at line 551 of file dqm_interfaces.py.

◆ workdir

dqm_interfaces.DirWalkerFile.workdir

Definition at line 552 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DirWalkerFile.__del__().