CMS 3D CMS Logo

Public Types | Public Member Functions | Protected Member Functions | Private Member Functions | Private Attributes

DQMStoreStats Class Reference

#include <DQMStoreStats.h>

Inheritance diagram for DQMStoreStats:
edm::EDAnalyzer

List of all members.

Public Types

enum  statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 }

Public Member Functions

 DQMStoreStats (const edm::ParameterSet &)
 ~DQMStoreStats ()

Protected Member Functions

void analyze (const edm::Event &e, const edm::EventSetup &c)
void beginJob ()
void beginLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &context)
void beginRun (const edm::Run &r, const edm::EventSetup &c)
void endJob ()
void endLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c)
void endRun (const edm::Run &r, const edm::EventSetup &c)

Private Member Functions

int calcstats (int)
void dumpMemoryProfile (void)
void print ()
std::pair< unsigned int,
unsigned int > 
readMemoryEntry (void) const

Private Attributes

DQMStoredbe_
bool dumpMemHistory_
bool isOpenProcFileSuccessful_
int maxbinsglobal_
std::string maxbinsmeglobal_
std::string maxbinsmesubsys_
int maxbinssubsys_
std::vector< std::pair< time_t,
unsigned int > > 
memoryHistoryVector_
int nbinsglobal_
int nbinssubsys_
int nmeglobal_
int nmesubsys_
edm::ParameterSet parameters_
std::string pathnamematch_
std::stringstream procFileName_
bool runineventloop_
bool runonendjob_
bool runonendlumi_
bool runonendrun_
time_t startingTime_
int statsdepth_
std::string subfolder_
std::string subsystem_
int verbose_

Detailed Description

DQMStoreStats itself

* DQM Test Client

Date:
2010/01/18 14:52:31
Revision:
1.7
Author:
Andreas Meyer CERN
Jan Olzem DESY

Definition at line 78 of file DQMStoreStats.h.


Member Enumeration Documentation

Enumerator:
considerAllME 
considerOnlyLumiProductME 

Definition at line 83 of file DQMStoreStats.h.


Constructor & Destructor Documentation

DQMStoreStats::DQMStoreStats ( const edm::ParameterSet ps)
DQMStoreStats::~DQMStoreStats ( )

Definition at line 50 of file DQMStoreStats.cc.

                             {
}

Member Function Documentation

void DQMStoreStats::analyze ( const edm::Event e,
const edm::EventSetup c 
) [protected, virtual]
void DQMStoreStats::beginJob ( void  ) [protected, virtual]

Reimplemented from edm::EDAnalyzer.

Definition at line 410 of file DQMStoreStats.cc.

References ExpressReco_HICollisions_FallBack::cerr, dbe_, recoMuon::in, isOpenProcFileSuccessful_, cmsCodeRules::cppFunctionSkipper::operator, and procFileName_.

                             {

  dbe_ = Service<DQMStore>().operator->();

  // access the proc/ folder for memory information
  procFileName_ << "/proc/" << getpid() << "/status";

  // open for a test
  std::ifstream procFile( procFileName_.str().c_str(), ios::in );

  if( procFile.good() ) {
    isOpenProcFileSuccessful_ = true;
  }
  else {
    std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
    std::cerr << "  Total memory profile will not be available." << std::endl;
    isOpenProcFileSuccessful_ = false;
  }

  procFile.close();

}
void DQMStoreStats::beginLuminosityBlock ( const edm::LuminosityBlock lumiSeg,
const edm::EventSetup context 
) [protected, virtual]

Reimplemented from edm::EDAnalyzer.

Definition at line 444 of file DQMStoreStats.cc.

                                                                       {
}
void DQMStoreStats::beginRun ( const edm::Run r,
const edm::EventSetup c 
) [protected, virtual]

Reimplemented from edm::EDAnalyzer.

Definition at line 437 of file DQMStoreStats.cc.

                                                                       {
}
int DQMStoreStats::calcstats ( int  mode = DQMStoreStats::considerAllME) [private]

do the stats here and produce output;

mode is coded in DQMStoreStats::statMode enum (select subsets of ME, e.g. those with getLumiFlag() == true)

Definition at line 60 of file DQMStoreStats.cc.

References DQMStoreStatsSubfolder::AddBinsD(), DQMStoreStatsSubfolder::AddBinsF(), DQMStoreStatsSubfolder::AddBinsS(), considerAllME, considerOnlyLumiProductME, gather_cfg::cout, dbe_, MonitorElement::DQM_KIND_TH1D, MonitorElement::DQM_KIND_TH1F, MonitorElement::DQM_KIND_TH1S, MonitorElement::DQM_KIND_TH2D, MonitorElement::DQM_KIND_TH2F, MonitorElement::DQM_KIND_TH2S, MonitorElement::DQM_KIND_TH3F, MonitorElement::DQM_KIND_TPROFILE, MonitorElement::DQM_KIND_TPROFILE2D, DQMStore::getMatchingContents(), maxbinsglobal_, maxbinssubsys_, mode, nbinsglobal_, nbinssubsys_, path(), pathnamematch_, runineventloop_, runonendjob_, runonendlumi_, runonendrun_, DQMStoreStatsSubfolder::subfolderName_, and DQMStoreStatsSubsystem::subsystemName_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                                                    {

  nbinsglobal_ = 0; 
  nbinssubsys_ = 0; 
  maxbinsglobal_ = 0; 
  maxbinssubsys_ = 0; 
  std::string path = "";
  std::string subsystemname = "";
  std::string subfoldername = "";
  size_t subsysStringEnd = 0, subfolderStringEnd  = 0;


  std::vector<MonitorElement*> melist;
  melist = dbe_->getMatchingContents( pathnamematch_ );

  DQMStoreStatsTopLevel dqmStoreStatsTopLevel;

  // loop all ME
  typedef std::vector <MonitorElement*>::iterator meIt;
  for(meIt it = melist.begin(); it != melist.end(); ++it) {

    // consider only ME with getLumiFlag() == true ?
    if( mode == DQMStoreStats::considerOnlyLumiProductME && 
        !( (*it)->getLumiFlag() ) ) continue;
    
    // figure out subsystem/subfolder names
    std::string path = (*it)->getPathname();

    // protection against ghost ME with empty paths
    if( 0 == path.size() ) continue;

    subsysStringEnd = path.find( '/', 0 );
    if( std::string::npos == subsysStringEnd ) subsysStringEnd = path.size(); // no subfolder

    // new subsystem?
    if( path.substr( 0, subsysStringEnd ) != subsystemname ) {
      DQMStoreStatsSubsystem aSubsystem;
      subsystemname = path.substr( 0, subsysStringEnd );
      aSubsystem.subsystemName_ = subsystemname;
      dqmStoreStatsTopLevel.push_back( aSubsystem );
    }

    // get subfolder name (if there is one..)
    if( path.size() == subsysStringEnd ) {
      // no subfolders in subsystem, make dummy
      DQMStoreStatsSubfolder aSubfolder;
      aSubfolder.subfolderName_ = subsystemname; // <-- for tagging this case
      dqmStoreStatsTopLevel.back().push_back( aSubfolder );
    }

    else {

      // there is a subfolder, get its name
      subfolderStringEnd = path.find( '/', subsysStringEnd + 1 );
      if( std::string::npos == subfolderStringEnd ) subfolderStringEnd = path.size();

      // new subfolder?
      if( path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 ) != subfoldername ) {
        subfoldername = path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 );
        DQMStoreStatsSubfolder aSubfolder;
        aSubfolder.subfolderName_ = subfoldername;
        dqmStoreStatsTopLevel.back().push_back( aSubfolder );
      }

    }

    // shortcut
    DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();

    switch( (*it)->kind() ) {
      
      // one-dim ME
      case MonitorElement::DQM_KIND_TH1F: currentSubfolder.AddBinsF( (*it)->getNbinsX() ); break;
      case MonitorElement::DQM_KIND_TH1S: currentSubfolder.AddBinsS( (*it)->getNbinsX() ); break;
      case MonitorElement::DQM_KIND_TH1D: currentSubfolder.AddBinsD( (*it)->getNbinsX() ); break;
      case MonitorElement::DQM_KIND_TPROFILE: currentSubfolder.AddBinsD( (*it)->getNbinsX() ); break;

      // two-dim ME
      case MonitorElement::DQM_KIND_TH2F: currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
      case MonitorElement::DQM_KIND_TH2S: currentSubfolder.AddBinsS( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
      case MonitorElement::DQM_KIND_TH2D: currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
      case MonitorElement::DQM_KIND_TPROFILE2D: currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
 
      // three-dim ME
      case MonitorElement::DQM_KIND_TH3F: 
        currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ() ); break;

      default: {}
        // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
        // which we don't care much about. Alternatively:

        //   std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: " 
        //             << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
        //             << "  in path: \"" << path << "\" not considered." << std::endl;
    }
      
  } 



  // OUTPUT

  std::cout << endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << endl;

  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Configuration:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << " > running ";
  if (runonendrun_) std::cout << "on run end." << std::endl;
  if (runonendlumi_) std::cout << "on lumi end." << std::endl;
  if (runonendjob_) std::cout << "on job end." << std::endl;
  if (runineventloop_) std::cout << "in event loop." << std::endl;
  std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
  std::cout << std::endl;

  // dump folder structure
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Top level folder tree:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
    std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
    
    for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
      std::cout << "  |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
    }
    
  }

  // dump mem/bin table

  unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;

  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Detailed ressource usage information ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "subsystem/folder                  histograms     bins      bins per      MB        kB per" << std::endl;
  std::cout << "                                   (total)     (total)    histogram    (total)   histogram  " << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
    std::cout << it0->subsystemName_ << std::endl;
    
    unsigned int nHistograms = 0, nBins = 0, nBytes = 0;

    for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {

      // fixed-size working copy
      std::string thisSubfolderName( it1->subfolderName_ );
      if( thisSubfolderName.size() > 30 ) {
        thisSubfolderName.resize( 30 );
        thisSubfolderName.replace( thisSubfolderName.size() - 3, 3, 3, '.' );
      }

      std::cout << " -> " << std::setw( 30 ) << std::left << thisSubfolderName;
      std::cout << std::setw( 7 ) << std::right << it1->totalHistos_;
      std::cout << std::setw( 12 ) << std::right << it1->totalBins_;

      // bins/histogram, need to catch nan if histos=0
      if( it1->totalHistos_ ) {
        std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << it1->totalBins_ / float( it1->totalHistos_ );
      } 
      else std::cout << std::setw( 12 ) << std::right << "-";

      std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / 1000.;

      // mem/histogram, need to catch nan if histos=0
      if( it1->totalHistos_ ) {
        std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / it1->totalHistos_;
      }
      else std::cout << std::setw( 12 ) << std::right << "-";

      std::cout << std::endl;

      // collect totals
      nHistograms += it1->totalHistos_; 
      nBins       += it1->totalBins_;   
      nBytes      += it1->totalMemory_; 

    }




    overallNHistograms += nHistograms;
    overallNBins       += nBins;
    overallNBytes      += nBytes;

    // display totals
    std::cout << "    " << std::setw( 30 ) << std::left << "SUBSYSTEM TOTAL";
    std::cout << std::setw( 7 ) << std::right << nHistograms;
    std::cout << std::setw( 12 ) << std::right << nBins;
    std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << nBins / float( nHistograms );
    std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / 1000.;
    std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / nHistograms;
    std::cout << std::endl;
      
    std::cout << ".........................................................................................." << std::endl;

  }


  // dump total
  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Grand total ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME:" << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
  std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether" << std::endl;
  std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB" << std::endl;



  std::cout << endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << "[DQMStoreStats::calcstats] -- End of output ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME." << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << endl;

  return 0;

}
void DQMStoreStats::dumpMemoryProfile ( void  ) [private]

Definition at line 300 of file DQMStoreStats.cc.

References gather_cfg::cout, dumpMemHistory_, isOpenProcFileSuccessful_, memoryHistoryVector_, download_sqlite_cfg::outputFile, and startingTime_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                            {

  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Memory profile:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;

  // determine virtual memory maximum
  std::pair<time_t, unsigned int> maxItem( 0, 0 );
  for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
       it < memoryHistoryVector_.end(); ++it ) {
    if( it->second > maxItem.second ) {
      maxItem = *it;
    }
  }

  std::stringstream rootOutputFileName;
  rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";

  // dump memory history to root file
  if( dumpMemHistory_ && isOpenProcFileSuccessful_ ) {

    TFile outputFile( rootOutputFileName.str().c_str(), "RECREATE" );

    int aTime;
    float aMb;

    TTree memHistoryTree( "dqmstorestats_memhistory", "memory history" );
    memHistoryTree.Branch( "seconds", &aTime, "seconds/I" );
    memHistoryTree.Branch( "megabytes", &aMb, "megabytes/F" );
    for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
         it < memoryHistoryVector_.end(); ++it ) {
      aTime = it->first - startingTime_;
      aMb = it->second / 1000.;
      memHistoryTree.Fill();
    }

    outputFile.Write();
    outputFile.Close();

  }

  std::cout << "Approx. maximum total virtual memory size of job: ";
  if( isOpenProcFileSuccessful_ && memoryHistoryVector_.size() ) {
    std::cout << maxItem.second / 1000.
              << " MB (reached " << maxItem.first - startingTime_ << " sec. after constructor called)," << std::endl;
    std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size() << " samples)" << std::endl;
  } else {
    std::cout << "(could not be determined)" << std::endl;
  }

  std::cout << std::endl << std::endl;
  
}
void DQMStoreStats::endJob ( void  ) [protected, virtual]
void DQMStoreStats::endLuminosityBlock ( const edm::LuminosityBlock lumiSeg,
const edm::EventSetup c 
) [protected, virtual]
void DQMStoreStats::endRun ( const edm::Run r,
const edm::EventSetup c 
) [protected, virtual]
void DQMStoreStats::print ( void  ) [private]

Definition at line 360 of file DQMStoreStats.cc.

References gather_cfg::cout, maxbinsmesubsys_, maxbinssubsys_, nbinssubsys_, nmesubsys_, subfolder_, and subsystem_.

                         {
  // subsystem info printout
  std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
  std::cout <<  "  " << subfolder_ << ": " ;
  std::cout <<  nmesubsys_ << " histograms with " 
            <<  nbinssubsys_  << " bins. " ; 
  if (nmesubsys_ > 0) std::cout <<  nbinssubsys_/nmesubsys_ << " bins/histogram " ;
  std::cout << std::endl;
  std::cout <<  "  Largest histogram: " << maxbinsmesubsys_ << " with " <<
                                         maxbinssubsys_ << " bins." <<  std::endl;
}
std::pair< unsigned int, unsigned int > DQMStoreStats::readMemoryEntry ( void  ) const [private]

read virtual memory size from /proc/<pid>/status file

Definition at line 378 of file DQMStoreStats.cc.

References recoMuon::in, isOpenProcFileSuccessful_, procFileName_, and cond::rpcobgas::time.

Referenced by analyze().

                                                                               {
  
  // see if initial test reading was successful
  if( isOpenProcFileSuccessful_ ) {

    std::ifstream procFile( procFileName_.str().c_str(), ios::in );

    std::string readBuffer( "" );
    unsigned int memSize = 0;

    // scan procfile
    while( !procFile.eof() ) {
      procFile >> readBuffer;
      if( std::string( "VmSize:" ) == readBuffer ) {
        procFile >> memSize;
        break;
      }
    }

    procFile.close();
    return std::pair<time_t, unsigned int>( time( 0 ), memSize );
  }

  return std::pair<time_t, unsigned int>( 0, 0 );

}

Member Data Documentation

Definition at line 116 of file DQMStoreStats.h.

Referenced by beginJob(), and calcstats().

Definition at line 143 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

Definition at line 136 of file DQMStoreStats.h.

Referenced by beginJob(), dumpMemoryProfile(), and readMemoryEntry().

Definition at line 125 of file DQMStoreStats.h.

Referenced by calcstats().

std::string DQMStoreStats::maxbinsmeglobal_ [private]

Definition at line 127 of file DQMStoreStats.h.

std::string DQMStoreStats::maxbinsmesubsys_ [private]

Definition at line 128 of file DQMStoreStats.h.

Referenced by print().

Definition at line 126 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

std::vector<std::pair<time_t, unsigned int> > DQMStoreStats::memoryHistoryVector_ [private]

Definition at line 134 of file DQMStoreStats.h.

Referenced by analyze(), and dumpMemoryProfile().

Definition at line 121 of file DQMStoreStats.h.

Referenced by calcstats().

Definition at line 122 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

Definition at line 123 of file DQMStoreStats.h.

Definition at line 124 of file DQMStoreStats.h.

Referenced by print().

Definition at line 117 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

std::string DQMStoreStats::pathnamematch_ [private]

Definition at line 131 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

std::stringstream DQMStoreStats::procFileName_ [private]

Definition at line 137 of file DQMStoreStats.h.

Referenced by beginJob(), and readMemoryEntry().

Definition at line 142 of file DQMStoreStats.h.

Referenced by analyze(), calcstats(), and DQMStoreStats().

Definition at line 140 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endJob().

Definition at line 141 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endLuminosityBlock().

Definition at line 139 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endRun().

time_t DQMStoreStats::startingTime_ [private]

Definition at line 135 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

Definition at line 130 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

std::string DQMStoreStats::subfolder_ [private]

Definition at line 120 of file DQMStoreStats.h.

Referenced by print().

std::string DQMStoreStats::subsystem_ [private]

Definition at line 119 of file DQMStoreStats.h.

Referenced by print().

int DQMStoreStats::verbose_ [private]

Definition at line 132 of file DQMStoreStats.h.

Referenced by DQMStoreStats().