DQMStoreStats Class Reference

#include <DQMStoreStats.h>

Inheritance diagram for DQMStoreStats:

Public Types
enum	statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 }
Public Types inherited from edm::EDAnalyzer
typedef EDAnalyzer	ModuleType
Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Public Member Functions
	DQMStoreStats (const edm::ParameterSet &)
	~DQMStoreStats () override
Public Member Functions inherited from edm::EDAnalyzer
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)
	EDAnalyzer ()
SerialTaskQueue *	globalLuminosityBlocksQueue ()
SerialTaskQueue *	globalRunsQueue ()
ModuleDescription const &	moduleDescription () const
std::string	workerType () const
	~EDAnalyzer () override
Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const
void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...
	EDConsumerBase ()
	EDConsumerBase (EDConsumerBase const &)=delete
	EDConsumerBase (EDConsumerBase &&)=default
ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const
void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const
void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const
std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const
void	labelsForToken (EDGetToken iToken, Labels &oLabels) const
void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const
EDConsumerBase const &	operator= (EDConsumerBase const &)=delete
EDConsumerBase &	operator= (EDConsumerBase &&)=default
bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const
bool	registeredToConsumeMany (TypeID const &, BranchType) const
ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const
void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)
virtual	~EDConsumerBase () noexcept(false)

Protected Member Functions
void	analyze (const edm::Event &e, const edm::EventSetup &c) override
void	beginJob () override
void	beginRun (const edm::Run &r, const edm::EventSetup &c) override
void	endJob () override
void	endLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) override
void	endRun (const edm::Run &r, const edm::EventSetup &c) override
Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)
EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)
template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)
ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...
template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()
void	consumesMany (const TypeToGet &id)
template<BranchType B>
void	consumesMany (const TypeToGet &id)
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)
template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Private Member Functions
void	calcIgProfDump (Folder &)
int	calcstats (int)
void	dumpMemoryProfile ()
void	print ()
std::pair< unsigned int, unsigned int >	readMemoryEntry () const

Private Attributes
DQMStore *	dbe_
bool	dumpMemHistory_
bool	dumpToFWJR_
bool	isOpenProcFileSuccessful_
int	maxbinsglobal_
std::string	maxbinsmeglobal_
std::string	maxbinsmesubsys_
int	maxbinssubsys_
std::vector< std::pair< time_t, unsigned int > >	memoryHistoryVector_
int	nbinsglobal_
int	nbinssubsys_
int	nmeglobal_
int	nmesubsys_
edm::ParameterSet	parameters_
std::string	pathnamematch_
std::stringstream	procFileName_
bool	runineventloop_
bool	runonendjob_
bool	runonendlumi_
bool	runonendrun_
time_t	startingTime_
int	statsdepth_
std::string	subfolder_
std::string	subsystem_
int	verbose_

Additional Inherited Members
Static Public Member Functions inherited from edm::EDAnalyzer
static const std::string &	baseType ()
static void	fillDescriptions (ConfigurationDescriptions &descriptions)
static void	prevalidate (ConfigurationDescriptions &)
static bool	wantsGlobalLuminosityBlocks ()
static bool	wantsGlobalRuns ()
static bool	wantsStreamLuminosityBlocks ()
static bool	wantsStreamRuns ()

Detailed Description

DQMStoreStats itself

• DQM Test Client

  Author

  Andreas Meyer CERN

  Jan Olzem DESY

Definition at line 294 of file DQMStoreStats.h.

Member Enumeration Documentation

enum DQMStoreStats::statsMode

Enumerator
considerAllME
considerOnlyLumiProductME

Definition at line 299 of file DQMStoreStats.h.

{ considerAllME = 0, considerOnlyLumiProductME = 1 };

Constructor & Destructor Documentation

DQMStoreStats::DQMStoreStats

(

const edm::ParameterSet &

ps

)

Definition at line 49 of file DQMStoreStats.cc.

References dumpMemHistory_, dumpToFWJR_, funct::false, edm::ParameterSet::getUntrackedParameter(), parameters_, pathnamematch_, runineventloop_, runonendjob_, runonendlumi_, runonendrun_, startingTime_, statsdepth_, AlCaHLTBitMon_QueryRunRegistry::string, ntuplemaker::time, funct::true, verbose_, and ~DQMStoreStats().

  : subsystem_ (""),
    subfolder_ (""),
    nbinsglobal_ (0),
    nbinssubsys_ (0),
    nmeglobal_ (0),
    nmesubsys_ (0),
    maxbinsglobal_ (0),
    maxbinssubsys_ (0),
    maxbinsmeglobal_ (""),
    maxbinsmesubsys_ (""),
    statsdepth_ (1),
    pathnamematch_ ("*"),
    verbose_ (0)
{
  parameters_ = ps;
  pathnamematch_ = ps.getUntrackedParameter<std::string>( "pathNameMatch", pathnamematch_ );
  statsdepth_ = ps.getUntrackedParameter<int>( "statsDepth", statsdepth_ );
  verbose_ = ps.getUntrackedParameter<int>( "verbose", verbose_ );
  dumpMemHistory_ = ps.getUntrackedParameter<bool>( "dumpMemoryHistory", false );
  runonendrun_    = ps.getUntrackedParameter<bool>( "runOnEndRun", true );
  runonendjob_    = ps.getUntrackedParameter<bool>( "runOnEndJob", false );
  runonendlumi_   = ps.getUntrackedParameter<bool>( "runOnEndLumi", false );
  runineventloop_ = ps.getUntrackedParameter<bool>( "runInEventLoop", false );
  dumpToFWJR_     = ps.getUntrackedParameter<bool>( "dumpToFWJR", false );

  startingTime_ = time( nullptr );
}

DQMStoreStats::~DQMStoreStats ( )

overridedefault

Referenced by DQMStoreStats().

Member Function Documentation

void DQMStoreStats::analyze ( const edm::Event &  e, const edm::EventSetup &  c  )

overrideprotected

Definition at line 658 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), memoryHistoryVector_, readMemoryEntry(), and runineventloop_.

                                                                         {

  //now read virtual memory size from proc folder
  memoryHistoryVector_.emplace_back(readMemoryEntry() );

  if (runineventloop_) {
    calcstats( DQMStoreStats::considerAllME );
    calcstats( DQMStoreStats::considerOnlyLumiProductME );
    dumpMemoryProfile();
  }

}

void DQMStoreStats::beginJob ( void  )

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 625 of file DQMStoreStats.cc.

References MessageLogger_cfi::cerr, dbe_, recoMuon::in, isOpenProcFileSuccessful_, Utilities::operator, and procFileName_.

                             {

  dbe_ = Service<DQMStore>().operator->();

  // access the proc/ folder for memory information
  procFileName_ << "/proc/" << getpid() << "/status";

  // open for a test
  std::ifstream procFile( procFileName_.str().c_str(), ios::in );

  if( procFile.good() ) {
    isOpenProcFileSuccessful_ = true;
  }
  else {
    std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
    std::cerr << "  Total memory profile will not be available." << std::endl;
    isOpenProcFileSuccessful_ = false;
  }

  procFile.close();

}

void DQMStoreStats::beginRun ( const edm::Run &  r, const edm::EventSetup &  c  )

overrideprotected

Definition at line 652 of file DQMStoreStats.cc.

                                                                       {
}

void DQMStoreStats::calcIgProfDump ( Folder &  root )

private

Definition at line 80 of file DQMStoreStats.cc.

References Folder::children(), Folder::CreateIterator(), VIterator< Item >::CurrentItem(), Folder::files(), VIterator< Item >::First(), cuy::ii, VIterator< Item >::IsDone(), Folder::mainrows(), Folder::mainrows_cumulative(), VIterator< Item >::Next(), Folder::parents(), AlCaHLTBitMon_QueryRunRegistry::string, Folder::summary(), and Folder::symbols().

Referenced by calcstats().

{
  std::ofstream stream("dqm-bin-stats.sql");
  stream << ""
"    PRAGMA journal_mode=OFF;"
"    PRAGMA count_changes=OFF;"
"    DROP TABLE IF EXISTS files;"
"    DROP TABLE IF EXISTS symbols;"
"    DROP TABLE IF EXISTS mainrows;"
"    DROP TABLE IF EXISTS children;"
"    DROP TABLE IF EXISTS parents;"
"    DROP TABLE IF EXISTS summary;"
"    CREATE TABLE children ("
"        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
"        parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
"        from_parent_count INTEGER,"
"        from_parent_calls INTEGER,"
"        from_parent_paths INTEGER,"
"        pct REAL"
"    );"
"    CREATE TABLE files ("
"        id,"
"        name TEXT"
"    );"
"    CREATE TABLE mainrows ("
"        id INTEGER PRIMARY KEY,"
"        symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
"        self_count INTEGER,"
"        cumulative_count INTEGER,"
"        kids INTEGER,"
"        self_calls INTEGER,"
"        total_calls INTEGER,"
"        self_paths INTEGER,"
"        total_paths INTEGER,"
"        pct REAL"
"    );"
"    CREATE TABLE parents ("
"        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
"        child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
"        to_child_count INTEGER,"
"        to_child_calls INTEGER,"
"        to_child_paths INTEGER,"
"        pct REAL"
"    );"
"    CREATE TABLE summary ("
"        counter TEXT,"
"        total_count INTEGER,"
"        total_freq INTEGER,"
"        tick_period REAL"
"    );"
"    CREATE TABLE symbols ("
"        id,"
"        name TEXT,"
"        filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
"    );"
"    CREATE UNIQUE INDEX fileIndex ON files (id);"
"    CREATE INDEX selfCountIndex ON mainrows(self_count);"
"    CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
"    CREATE INDEX totalCountIndex ON mainrows(cumulative_count);" << std::endl;

  std::string sql_statement("");

  root.files(sql_statement);
  root.symbols(sql_statement);
  root.mainrows_cumulative(sql_statement);
  root.summary(sql_statement);
  VIterator<Folder *> subsystems = root.CreateIterator() ;
  size_t ii=0;
  for(subsystems.First() ; !subsystems.IsDone() ; subsystems.Next(), ++ii)
  {
    subsystems.CurrentItem()->mainrows(sql_statement);
    subsystems.CurrentItem()->parents(sql_statement);
    subsystems.CurrentItem()->children(sql_statement);
  }
  stream << sql_statement << std::endl;
}

int DQMStoreStats::calcstats ( int  mode = DQMStoreStats::considerAllME )

private

do the stats here and produce output;

mode is coded in DQMStoreStats::statMode enum (select subsets of ME, e.g. those with getLumiFlag() == true)

Definition at line 163 of file DQMStoreStats.cc.

References DQMStoreStatsSubfolder::AddBinsD(), DQMStoreStatsSubfolder::AddBinsF(), DQMStoreStatsSubfolder::AddBinsS(), calcIgProfDump(), Folder::cd(), considerAllME, considerOnlyLumiProductME, gather_cfg::cout, dbe_, MonitorElement::DQM_KIND_TH1D, MonitorElement::DQM_KIND_TH1F, MonitorElement::DQM_KIND_TH1S, MonitorElement::DQM_KIND_TH2D, MonitorElement::DQM_KIND_TH2F, MonitorElement::DQM_KIND_TH2S, MonitorElement::DQM_KIND_TH3F, MonitorElement::DQM_KIND_TPROFILE, MonitorElement::DQM_KIND_TPROFILE2D, dumpToFWJR_, objects.autophobj::float, getEmptyMetric(), edm::Service< T >::isAvailable(), maxbinsglobal_, maxbinssubsys_, ALCARECOPromptCalibProdSiPixelAli0T_cff::mode, nbinsglobal_, nbinssubsys_, callgraph::path, pathnamematch_, edm::JobReport::reportAnalysisFile(), runineventloop_, runonendjob_, runonendlumi_, runonendrun_, AlCaHLTBitMon_QueryRunRegistry::string, DQMStoreStatsSubfolder::subfolderName_, DQMStoreStatsSubsystem::subsystemName_, and Folder::update().

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                                                    {

  nbinsglobal_ = 0; 
  nbinssubsys_ = 0; 
  maxbinsglobal_ = 0; 
  maxbinssubsys_ = 0; 
  std::string path = "";
  std::string subsystemname = "";
  std::string subfoldername = "";
  size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd  = 0;


  std::vector<MonitorElement*> melist;
  melist = dbe_->getMatchingContents( pathnamematch_ );

  Folder dbeFolder("root");
  DQMStoreStatsTopLevel dqmStoreStatsTopLevel;

  // loop all ME
  for(auto & it : melist) {

    // consider only ME with getLumiFlag() == true ?
    if( mode == DQMStoreStats::considerOnlyLumiProductME && 
        !( it->getLumiFlag() ) ) continue;
    
    // figure out subsystem/subfolder names
    const std::string& path = it->getPathname();

    subfolderStringBegin = 0;
    Folder * curr = &dbeFolder;
    while(true)
    {
      subfolderStringEnd = path.find( '/', subfolderStringBegin );
      if( std::string::npos == subfolderStringEnd )
      {
        curr = curr->cd(path.substr( subfolderStringBegin, path.size()-subfolderStringBegin ));
        break;
      }
      curr = curr->cd(path.substr( subfolderStringBegin, subfolderStringEnd-subfolderStringBegin ));
      subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
    }
    
    // protection against ghost ME with empty paths
    if( path.empty() ) continue;

    subsysStringEnd = path.find( '/', 0 );
    if( std::string::npos == subsysStringEnd ) subsysStringEnd = path.size(); // no subfolder

    // new subsystem?
    if( path.substr( 0, subsysStringEnd ) != subsystemname ) {
      DQMStoreStatsSubsystem aSubsystem;
      subsystemname = path.substr( 0, subsysStringEnd );
      aSubsystem.subsystemName_ = subsystemname;
      dqmStoreStatsTopLevel.push_back( aSubsystem );
      subfoldername = "";
    }

    // get subfolder name (if there is one..)
    if( path.size() == subsysStringEnd ) {
      // no subfolders in subsystem, make dummy
      DQMStoreStatsSubfolder aSubfolder;
      aSubfolder.subfolderName_ = subsystemname; // <-- for tagging this case
      dqmStoreStatsTopLevel.back().push_back( aSubfolder );
    }

    else {

      // there is a subfolder, get its name
      subfolderStringEnd = path.find( '/', subsysStringEnd + 1 );
      if( std::string::npos == subfolderStringEnd ) subfolderStringEnd = path.size();

      // new subfolder?
      if( path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 ) != subfoldername ) {
        subfoldername = path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 );
        DQMStoreStatsSubfolder aSubfolder;
        aSubfolder.subfolderName_ = subfoldername;
        dqmStoreStatsTopLevel.back().push_back( aSubfolder );
      }

    }

    // shortcut
    DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();

    switch( it->kind() ) {
      
      // one-dim ME
    case MonitorElement::DQM_KIND_TH1F:
      currentSubfolder.AddBinsF( it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0) );
      curr->update( it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0),   it->getNbinsX()*sizeof( float ) );
      break;
    case MonitorElement::DQM_KIND_TH1S:
      currentSubfolder.AddBinsS( it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0) );
      curr->update( it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0),   it->getNbinsX()*sizeof( short ) );
      break;
    case MonitorElement::DQM_KIND_TH1D:
      currentSubfolder.AddBinsD( it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0) );
      curr->update( it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0),   it->getNbinsX()*sizeof( double ) );
      break;
    case MonitorElement::DQM_KIND_TPROFILE:
      currentSubfolder.AddBinsD( it->getNbinsX(), getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0) );
      curr->update( it->getNbinsX(), getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0),   it->getNbinsX()*sizeof( double ) );
      break;

      // two-dim ME
    case MonitorElement::DQM_KIND_TH2F:
      currentSubfolder.AddBinsF( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0) );
      curr->update( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0),  it->getNbinsX() * it->getNbinsY()*sizeof(float) );
      break;
    case MonitorElement::DQM_KIND_TH2S:
      currentSubfolder.AddBinsS( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0) );
      curr->update( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0), it->getNbinsX() * it->getNbinsY()*sizeof(short) );
      break;
    case MonitorElement::DQM_KIND_TH2D:
      currentSubfolder.AddBinsD( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0) );
      curr->update( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0), it->getNbinsX() * it->getNbinsY()*sizeof(double) );
      break;
    case MonitorElement::DQM_KIND_TPROFILE2D:
      currentSubfolder.AddBinsD( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0) );
      curr->update( it->getNbinsX() * it->getNbinsY(), getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2, 0), it->getNbinsX() * it->getNbinsY()*sizeof(double) );
      break;
 
      // three-dim ME
    case MonitorElement::DQM_KIND_TH3F: 
      currentSubfolder.AddBinsF( it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(), getEmptyMetric( it->getTH3F()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2,  it->getNbinsZ()+2 ) );
      curr->update( it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
                    getEmptyMetric( it->getTH3F()->GetArray(), it->getNbinsX()+2,  it->getNbinsY()+2,  it->getNbinsZ()+2 ),
                    it->getNbinsX() * it->getNbinsY() * it->getNbinsZ()*sizeof(float));
      break;

    default: {}
      // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
      // which we don't care much about. Alternatively:

      //   std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: " 
      //               << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
      //               << "  in path: \"" << path << "\" not considered." << std::endl;
    }
  } 

  if( mode == DQMStoreStats::considerAllME )
    calcIgProfDump(dbeFolder);

  // OUTPUT

  std::cout << endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << endl;

  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Configuration:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << " > running ";
  if (runonendrun_) std::cout << "on run end." << std::endl;
  if (runonendlumi_) std::cout << "on lumi end." << std::endl;
  if (runonendjob_) std::cout << "on job end." << std::endl;
  if (runineventloop_) std::cout << "in event loop." << std::endl;
  std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
  std::cout << std::endl;

  // dump folder structure
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Top level folder tree:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  for( auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
    std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
    
    for( auto it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
      std::cout << "  |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
    }
    
  }

  // dump mem/bin table

  unsigned int overallNHistograms = 0, overallNBins = 0, overallNEmptyBins = 0, overallNBytes = 0;

  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Detailed ressource usage information ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "subsystem/folder                          histograms       bins        Empty bins     Empty/Total      bins per       MB         kB per" << std::endl;
  std::cout << "                                           (total)        (total)        (total)                      histogram     (total)    histogram  " << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  for( auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
    std::cout << it0->subsystemName_ << std::endl;
    
    unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;

    for( auto it1 = it0->begin(); it1 < it0->end(); ++it1 ) {

      // fixed-size working copy
      std::string thisSubfolderName( it1->subfolderName_ );
      if( thisSubfolderName.size() > 30 ) {
        thisSubfolderName.resize( 30 );
        thisSubfolderName.replace( thisSubfolderName.size() - 3, 3, 3, '.' );
      }

      std::cout << " -> " << std::setw( 30 ) << std::left << thisSubfolderName;
      std::cout << std::setw( 14 ) << std::right << it1->totalHistos_;
      std::cout << std::setw( 14 ) << std::right << it1->totalBins_;
      std::cout << std::setw( 14 ) << std::right << it1->totalEmptyBins_;
      std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << (float)it1->totalEmptyBins_/(float)it1->totalBins_;

      // bins/histogram, need to catch nan if histos=0
      if( it1->totalHistos_ ) {
        std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalBins_ / float( it1->totalHistos_ );
      } 
      else std::cout << std::setw( 14 ) << std::right << "-";

      std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / 1024.;

      // mem/histogram, need to catch nan if histos=0
      if( it1->totalHistos_ ) {
        std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / it1->totalHistos_;
      }
      else std::cout << std::setw( 14 ) << std::right << "-";

      std::cout << std::endl;

      // collect totals
      nHistograms += it1->totalHistos_; 
      nBins       += it1->totalBins_;   
      nEmptyBins  += it1->totalEmptyBins_;   
      nBytes      += it1->totalMemory_; 

    }




    overallNHistograms += nHistograms;
    overallNBins       += nBins;
    overallNEmptyBins  += nEmptyBins;
    overallNBytes      += nBytes;

    // display totals
    std::cout << "    " << std::setw( 30 ) << std::left << "SUBSYSTEM TOTAL";
    std::cout << std::setw( 14 ) << std::right << nHistograms;
    std::cout << std::setw( 14 ) << std::right << nBins;
    std::cout << std::setw( 14 ) << std::right << nEmptyBins;
    std::cout << std::setw( 14 ) << std::right << (float)nEmptyBins/(float)nBins;
    std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBins / float( nHistograms );
    std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / 1000.;
    std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / nHistograms;
    std::cout << std::endl;
      
    std::cout << ".........................................................................................." << std::endl;

  }


  // dump total
  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Grand total ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME:" << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
  std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether" << std::endl;
  std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB" << std::endl;



  std::cout << endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << "[DQMStoreStats::calcstats] -- End of output ";
  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME." << std::endl;
  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
  std::cout << "===========================================================================================" << std::endl;
  std::cout << endl;

  // Put together a simplified version of the complete dump that is
  // sent to std::cout. Just dump the very basic information,
  // i.e. summary for each folder, both for run and LS products.
  if (dumpToFWJR_)
  {
    edm::Service<edm::JobReport> jr;
    // Do not even try if the FWJR service is not available.
    if (!jr.isAvailable())
      return 0;
    // Prepare appropriate map to store FWJR output.
    std::map<std::string, std::string> jrInfo;
    unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;

    jrInfo["Source"] = "DQMServices/Components";
    jrInfo["FileClass"] = "DQMStoreStats";
    if (runonendrun_)
      jrInfo["DumpType"] = "EndRun";
    if (runonendlumi_)
      jrInfo["DumpType"] = "EndLumi";
    if (runonendjob_)
      jrInfo["DumpType"] = "EndJob";
    if (runineventloop_)
      jrInfo["DumpType"] = "EventLoop";
    if( mode == DQMStoreStats::considerAllME )
      jrInfo["Type"] = "RunProduct";
    else if( mode == DQMStoreStats::considerOnlyLumiProductME )
      jrInfo["Type"] = "LumiProduct";


    jrInfo["pathNameMatch"] = pathnamematch_;

    for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 )
    {
      unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
      for( auto it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
        // collect totals
        nHistograms += it1->totalHistos_; 
        nBins       += it1->totalBins_;   
        nEmptyBins  += it1->totalEmptyBins_;   
        nBytes      += it1->totalMemory_; 
      }
      overallNHistograms += nHistograms;
      overallNBins       += nBins;
      overallNBytes      += nBytes;
      std::stringstream iss("");
      iss << nHistograms;
      jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
      iss.str("");iss<<nBins;
      jrInfo[it0->subsystemName_ + std::string("_b")]  = iss.str();
      iss.str("");iss<<nEmptyBins;
      jrInfo[it0->subsystemName_ + std::string("_be")]  = iss.str();
      iss.str("");iss<< ((float)nEmptyBins/(float)nBins);
      jrInfo[it0->subsystemName_ + std::string("_fbe")]  = iss.str();
      iss.str("");iss<< ((float)nBins/(float)nHistograms);
      jrInfo[it0->subsystemName_ + std::string("_b_h")]    = iss.str();
      iss.str("");iss<<nBytes / 1024. / 1024.;
      jrInfo[it0->subsystemName_ + std::string("_MB")]    = iss.str();
      iss.str("");iss<<nBytes / 1024. / nHistograms;
      jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
    }
    jr->reportAnalysisFile("DQMStatsReport", jrInfo);
  }

  return 0;

}

void DQMStoreStats::dumpMemoryProfile ( )

private

Definition at line 515 of file DQMStoreStats.cc.

References gather_cfg::cout, dumpMemHistory_, isOpenProcFileSuccessful_, memoryHistoryVector_, download_sqlite_cfg::outputFile, and startingTime_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                       {

  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
  std::cout << "Memory profile:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------" << std::endl;

  // determine virtual memory maximum
  std::pair<time_t, unsigned int> maxItem( 0, 0 );
  for( auto it = memoryHistoryVector_.begin();
       it < memoryHistoryVector_.end(); ++it ) {
    if( it->second > maxItem.second ) {
      maxItem = *it;
    }
  }

  std::stringstream rootOutputFileName;
  rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";

  // dump memory history to root file
  if( dumpMemHistory_ && isOpenProcFileSuccessful_ ) {

    TFile outputFile( rootOutputFileName.str().c_str(), "RECREATE" );

    int aTime;
    float aMb;

    TTree memHistoryTree( "dqmstorestats_memhistory", "memory history" );
    memHistoryTree.Branch( "seconds", &aTime, "seconds/I" );
    memHistoryTree.Branch( "megabytes", &aMb, "megabytes/F" );
    for( auto it = memoryHistoryVector_.begin();
         it < memoryHistoryVector_.end(); ++it ) {
      aTime = it->first - startingTime_;
      aMb = it->second / 1000.;
      memHistoryTree.Fill();
    }

    outputFile.Write();
    outputFile.Close();

  }

  std::cout << "Approx. maximum total virtual memory size of job: ";
  if( isOpenProcFileSuccessful_ && !memoryHistoryVector_.empty() ) {
    std::cout << maxItem.second / 1000.
              << " MB (reached " << maxItem.first - startingTime_ << " sec. after constructor called)," << std::endl;
    std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size() << " samples)" << std::endl;
  } else {
    std::cout << "(could not be determined)" << std::endl;
  }

  std::cout << std::endl << std::endl;
  
}

void DQMStoreStats::endJob ( void  )

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 701 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendjob_.

                           {

  if (runonendjob_) {
    calcstats( DQMStoreStats::considerAllME );
    calcstats( DQMStoreStats::considerOnlyLumiProductME );
    dumpMemoryProfile();
  }

}

void DQMStoreStats::endLuminosityBlock ( const edm::LuminosityBlock &  lumiSeg, const edm::EventSetup &  c  )

overrideprotected

Definition at line 675 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendlumi_.

                                                                     {
  if (runonendlumi_) { 
    calcstats( DQMStoreStats::considerAllME );
    calcstats( DQMStoreStats::considerOnlyLumiProductME );
    dumpMemoryProfile();
  }

}

void DQMStoreStats::endRun ( const edm::Run &  r, const edm::EventSetup &  c  )

overrideprotected

Definition at line 688 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendrun_.

                                                                  {

  if (runonendrun_) {
    calcstats( DQMStoreStats::considerAllME );
    calcstats( DQMStoreStats::considerOnlyLumiProductME );
    dumpMemoryProfile();
  }

}

void DQMStoreStats::print ( void  )

private

Definition at line 575 of file DQMStoreStats.cc.

References gather_cfg::cout, maxbinsmesubsys_, maxbinssubsys_, nbinssubsys_, nmesubsys_, subfolder_, and subsystem_.

                         {
  // subsystem info printout
  std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
  std::cout <<  "  " << subfolder_ << ": " ;
  std::cout <<  nmesubsys_ << " histograms with " 
            <<  nbinssubsys_  << " bins. " ; 
  if (nmesubsys_ > 0) std::cout <<  nbinssubsys_/nmesubsys_ << " bins/histogram " ;
  std::cout << std::endl;
  std::cout <<  "  Largest histogram: " << maxbinsmesubsys_ << " with " <<
                                         maxbinssubsys_ << " bins." <<  std::endl;
}

std::pair< unsigned int, unsigned int > DQMStoreStats::readMemoryEntry ( ) const

private

read virtual memory size from /proc/<pid>/status file

Definition at line 593 of file DQMStoreStats.cc.

References recoMuon::in, isOpenProcFileSuccessful_, procFileName_, AlCaHLTBitMon_QueryRunRegistry::string, and ntuplemaker::time.

Referenced by analyze().

                                                                          {
  
  // see if initial test reading was successful
  if( isOpenProcFileSuccessful_ ) {

    std::ifstream procFile( procFileName_.str().c_str(), ios::in );

    std::string readBuffer( "" );
    unsigned int memSize = 0;

    // scan procfile
    while( !procFile.eof() ) {
      procFile >> readBuffer;
      if( std::string( "VmSize:" ) == readBuffer ) {
        procFile >> memSize;
        break;
      }
    }

    procFile.close();
    return std::pair<time_t, unsigned int>( time( nullptr ), memSize );
  }

  return std::pair<time_t, unsigned int>( 0, 0 );

}

Member Data Documentation

DQMStore* DQMStoreStats::dbe_

private

Definition at line 331 of file DQMStoreStats.h.

Referenced by beginJob(), and calcstats().

bool DQMStoreStats::dumpMemHistory_

private

Definition at line 358 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

bool DQMStoreStats::dumpToFWJR_

private

Definition at line 359 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

bool DQMStoreStats::isOpenProcFileSuccessful_

private

Definition at line 351 of file DQMStoreStats.h.

Referenced by beginJob(), dumpMemoryProfile(), and readMemoryEntry().

int DQMStoreStats::maxbinsglobal_

private

Definition at line 340 of file DQMStoreStats.h.

Referenced by calcstats().

std::string DQMStoreStats::maxbinsmeglobal_

private

Definition at line 342 of file DQMStoreStats.h.

std::string DQMStoreStats::maxbinsmesubsys_

private

Definition at line 343 of file DQMStoreStats.h.

Referenced by print().

int DQMStoreStats::maxbinssubsys_

private

Definition at line 341 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

std::vector<std::pair<time_t, unsigned int> > DQMStoreStats::memoryHistoryVector_

private

Definition at line 349 of file DQMStoreStats.h.

Referenced by analyze(), and dumpMemoryProfile().

int DQMStoreStats::nbinsglobal_

private

Definition at line 336 of file DQMStoreStats.h.

Referenced by calcstats().

int DQMStoreStats::nbinssubsys_

private

Definition at line 337 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

int DQMStoreStats::nmeglobal_

private

Definition at line 338 of file DQMStoreStats.h.

int DQMStoreStats::nmesubsys_

private

Definition at line 339 of file DQMStoreStats.h.

Referenced by print().

edm::ParameterSet DQMStoreStats::parameters_

private

Definition at line 332 of file DQMStoreStats.h.

Referenced by Mixins._TypedParameterizable::clone(), Types.PSet::clone(), Mixins._TypedParameterizable::copy(), and DQMStoreStats().

std::string DQMStoreStats::pathnamematch_

private

Definition at line 346 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

std::stringstream DQMStoreStats::procFileName_

private

Definition at line 352 of file DQMStoreStats.h.

Referenced by beginJob(), and readMemoryEntry().

bool DQMStoreStats::runineventloop_

private

Definition at line 357 of file DQMStoreStats.h.

Referenced by analyze(), calcstats(), and DQMStoreStats().

bool DQMStoreStats::runonendjob_

private

Definition at line 355 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endJob().

bool DQMStoreStats::runonendlumi_

private

Definition at line 356 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endLuminosityBlock().

bool DQMStoreStats::runonendrun_

private

Definition at line 354 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endRun().

time_t DQMStoreStats::startingTime_

private

Definition at line 350 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

int DQMStoreStats::statsdepth_

private

Definition at line 345 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

std::string DQMStoreStats::subfolder_

private

Definition at line 335 of file DQMStoreStats.h.

Referenced by print().

std::string DQMStoreStats::subsystem_

private

Definition at line 334 of file DQMStoreStats.h.

Referenced by print().

int DQMStoreStats::verbose_

private

Definition at line 347 of file DQMStoreStats.h.

Referenced by DQMStoreStats().