DQMStoreStats Class Reference

Inheritance diagram for DQMStoreStats:

Public Types
enum	statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 }
Public Types inherited from edm::one::EDAnalyzerBase
typedef EDAnalyzerBase	ModuleType
Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Public Member Functions
	DQMStoreStats (const edm::ParameterSet &)
	~DQMStoreStats () override=default
Public Member Functions inherited from edm::one::EDAnalyzer< edm::one::WatchRuns, edm::one::WatchLuminosityBlocks >
	EDAnalyzer ()=default
	EDAnalyzer (const EDAnalyzer &)=delete
SerialTaskQueue *	globalLuminosityBlocksQueue () final
SerialTaskQueue *	globalRunsQueue () final
const EDAnalyzer &	operator= (const EDAnalyzer &)=delete
bool	wantsGlobalLuminosityBlocks () const noexcept final
bool	wantsGlobalRuns () const noexcept final
bool	wantsInputProcessBlocks () const noexcept final
bool	wantsProcessBlocks () const noexcept final
Public Member Functions inherited from edm::one::EDAnalyzerBase
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)
	EDAnalyzerBase ()
ModuleDescription const &	moduleDescription () const
bool	wantsStreamLuminosityBlocks () const noexcept
bool	wantsStreamRuns () const noexcept
	~EDAnalyzerBase () override
Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const
void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...
	EDConsumerBase ()
	EDConsumerBase (EDConsumerBase const &)=delete
	EDConsumerBase (EDConsumerBase &&)=default
ESResolverIndex const *	esGetTokenIndices (edm::Transition iTrans) const
std::vector< ESResolverIndex > const &	esGetTokenIndicesVector (edm::Transition iTrans) const
std::vector< ESRecordIndex > const &	esGetTokenRecordIndicesVector (edm::Transition iTrans) const
ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const
void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const
void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const
std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const
void	labelsForToken (EDGetToken iToken, Labels &oLabels) const
void	modulesWhoseProductsAreConsumed (std::array< std::vector< ModuleDescription const *> *, NumBranchTypes > &modulesAll, std::vector< ModuleProcessName > &modulesInPreviousProcesses, ProductRegistry const &preg, std::map< std::string, ModuleDescription const *> const &labelsToDesc, std::string const &processName) const
EDConsumerBase const &	operator= (EDConsumerBase const &)=delete
EDConsumerBase &	operator= (EDConsumerBase &&)=default
bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const
void	selectInputProcessBlocks (ProductRegistry const &productRegistry, ProcessBlockHelperBase const &processBlockHelperBase)
ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const
void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)
void	updateLookup (eventsetup::ESRecordsToProductResolverIndices const &)
virtual	~EDConsumerBase () noexcept(false)

Protected Member Functions
void	analyze (const edm::Event &e, const edm::EventSetup &c) override
void	beginJob () override
void	beginLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) override
void	beginRun (const edm::Run &r, const edm::EventSetup &c) override
void	endJob () override
void	endLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) override
void	endRun (const edm::Run &r, const edm::EventSetup &c) override
Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)
template<BranchType B = InEvent>
EDConsumerBaseAdaptor< B >	consumes (edm::InputTag tag) noexcept
EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)
template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)
ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...
template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes ()
template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag const &tag)
template<Transition Tr = Transition::Event>
constexpr auto	esConsumes ()
template<Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag tag)
template<Transition Tr = Transition::Event>
ESGetTokenGeneric	esConsumes (eventsetup::EventSetupRecordKey const &iRecord, eventsetup::DataKey const &iKey)
	Used with EventSetupRecord::doGet. More...
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)
template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)
void	resetItemsToGetFrom (BranchType iType)

Private Member Functions
void	calcIgProfDump (Folder &)
int	calcstats (int)
void	dumpMemoryProfile ()
void	print ()
std::pair< unsigned int, unsigned int >	readMemoryEntry () const

Private Attributes
DQMStore *	dbe_
bool	dumpMemHistory_
bool	dumpToFWJR_
bool	isOpenProcFileSuccessful_
int	maxbinsglobal_
std::string	maxbinsmeglobal_
std::string	maxbinsmesubsys_
int	maxbinssubsys_
std::vector< std::pair< time_t, unsigned int > >	memoryHistoryVector_
int	nbinsglobal_
int	nbinssubsys_
int	nmeglobal_
int	nmesubsys_
edm::ParameterSet	parameters_
std::string	pathnamematch_
std::stringstream	procFileName_
bool	runineventloop_
bool	runonendjob_
bool	runonendlumi_
bool	runonendrun_
time_t	startingTime_
int	statsdepth_
std::string	subfolder_
std::string	subsystem_
int	verbose_

Additional Inherited Members
Static Public Member Functions inherited from edm::one::EDAnalyzerBase
static const std::string &	baseType ()
static void	fillDescriptions (ConfigurationDescriptions &descriptions)
static void	prevalidate (ConfigurationDescriptions &descriptions)

Detailed Description

DQMStoreStats itself

• DQM Test Client

  Author

  Andreas Meyer CERN

  Jan Olzem DESY

Definition at line 312 of file DQMStoreStats.cc.

Member Enumeration Documentation

statsMode

enum DQMStoreStats::statsMode

Enumerator
considerAllME
considerOnlyLumiProductME

Definition at line 317 of file DQMStoreStats.cc.

{ considerAllME = 0, considerOnlyLumiProductME = 1 };

Constructor & Destructor Documentation

DQMStoreStats()

DQMStoreStats::DQMStoreStats

(

const edm::ParameterSet &

ps

)

Definition at line 413 of file DQMStoreStats.cc.

References dumpMemHistory_, dumpToFWJR_, edm::ParameterSet::getUntrackedParameter(), parameters_, pathnamematch_, runineventloop_, runonendjob_, runonendlumi_, runonendrun_, startingTime_, statsdepth_, AlCaHLTBitMon_QueryRunRegistry::string, hcalRecHitTable_cff::time, and verbose_.

    : subsystem_(""),
      subfolder_(""),
      nbinsglobal_(0),
      nbinssubsys_(0),
      nmeglobal_(0),
      nmesubsys_(0),
      maxbinsglobal_(0),
      maxbinssubsys_(0),
      maxbinsmeglobal_(""),
      maxbinsmesubsys_(""),
      statsdepth_(1),
      pathnamematch_(""),
      verbose_(0) {
  parameters_ = ps;
  pathnamematch_ = ps.getUntrackedParameter<std::string>("pathNameMatch", pathnamematch_);
  statsdepth_ = ps.getUntrackedParameter<int>("statsDepth", statsdepth_);
  verbose_ = ps.getUntrackedParameter<int>("verbose", verbose_);
  dumpMemHistory_ = ps.getUntrackedParameter<bool>("dumpMemoryHistory", false);
  runonendrun_ = ps.getUntrackedParameter<bool>("runOnEndRun", true);
  runonendjob_ = ps.getUntrackedParameter<bool>("runOnEndJob", false);
  runonendlumi_ = ps.getUntrackedParameter<bool>("runOnEndLumi", false);
  runineventloop_ = ps.getUntrackedParameter<bool>("runInEventLoop", false);
  dumpToFWJR_ = ps.getUntrackedParameter<bool>("dumpToFWJR", false);

  startingTime_ = time(nullptr);
}

~DQMStoreStats()

DQMStoreStats::~DQMStoreStats ( )

overridedefault

Member Function Documentation

analyze()

void DQMStoreStats::analyze ( const edm::Event &  e, const edm::EventSetup &  c  )

overrideprotectedvirtual

Implements edm::one::EDAnalyzerBase.

Definition at line 1057 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), memoryHistoryVector_, readMemoryEntry(), and runineventloop_.

                                                                         {
  //now read virtual memory size from proc folder
  memoryHistoryVector_.emplace_back(readMemoryEntry());

  if (runineventloop_) {
    calcstats(DQMStoreStats::considerAllME);
    calcstats(DQMStoreStats::considerOnlyLumiProductME);
    dumpMemoryProfile();
  }
}

beginJob()

void DQMStoreStats::beginJob ( void  )

overrideprotectedvirtual

Reimplemented from edm::one::EDAnalyzerBase.

Definition at line 1028 of file DQMStoreStats.cc.

References DMR_cfg::cerr, dbe_, recoMuon::in, isOpenProcFileSuccessful_, Utilities::operator, and procFileName_.

                             {
  dbe_ = Service<DQMStore>().operator->();

  // access the proc/ folder for memory information
  procFileName_ << "/proc/" << getpid() << "/status";

  // open for a test
  std::ifstream procFile(procFileName_.str().c_str(), ios::in);

  if (procFile.good()) {
    isOpenProcFileSuccessful_ = true;
  } else {
    std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
    std::cerr << "  Total memory profile will not be available." << std::endl;
    isOpenProcFileSuccessful_ = false;
  }

  procFile.close();
}

beginLuminosityBlock()

void DQMStoreStats::beginLuminosityBlock ( const edm::LuminosityBlock &  lumiSeg, const edm::EventSetup &  c  )

overrideprotected

Definition at line 1082 of file DQMStoreStats.cc.

{}

beginRun()

void DQMStoreStats::beginRun ( const edm::Run &  r, const edm::EventSetup &  c  )

overrideprotected

Definition at line 1052 of file DQMStoreStats.cc.

{}

calcIgProfDump()

void DQMStoreStats::calcIgProfDump ( Folder &  root )

private

Definition at line 441 of file DQMStoreStats.cc.

References Folder::children(), VIterator< Item >::CurrentItem(), VIterator< Item >::First(), cuy::ii, VIterator< Item >::IsDone(), Folder::mainrows(), VIterator< Item >::Next(), Folder::parents(), cms::cuda::stream, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by calcstats().

                                               {
  std::ofstream stream("dqm-bin-stats.sql");
  stream << ""
            "    PRAGMA journal_mode=OFF;"
            "    PRAGMA count_changes=OFF;"
            "    DROP TABLE IF EXISTS files;"
            "    DROP TABLE IF EXISTS symbols;"
            "    DROP TABLE IF EXISTS mainrows;"
            "    DROP TABLE IF EXISTS children;"
            "    DROP TABLE IF EXISTS parents;"
            "    DROP TABLE IF EXISTS summary;"
            "    CREATE TABLE children ("
            "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
            "        parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
            "        from_parent_count INTEGER,"
            "        from_parent_calls INTEGER,"
            "        from_parent_paths INTEGER,"
            "        pct REAL"
            "    );"
            "    CREATE TABLE files ("
            "        id,"
            "        name TEXT"
            "    );"
            "    CREATE TABLE mainrows ("
            "        id INTEGER PRIMARY KEY,"
            "        symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
            "        self_count INTEGER,"
            "        cumulative_count INTEGER,"
            "        kids INTEGER,"
            "        self_calls INTEGER,"
            "        total_calls INTEGER,"
            "        self_paths INTEGER,"
            "        total_paths INTEGER,"
            "        pct REAL"
            "    );"
            "    CREATE TABLE parents ("
            "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
            "        child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
            "        to_child_count INTEGER,"
            "        to_child_calls INTEGER,"
            "        to_child_paths INTEGER,"
            "        pct REAL"
            "    );"
            "    CREATE TABLE summary ("
            "        counter TEXT,"
            "        total_count INTEGER,"
            "        total_freq INTEGER,"
            "        tick_period REAL"
            "    );"
            "    CREATE TABLE symbols ("
            "        id,"
            "        name TEXT,"
            "        filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
            "    );"
            "    CREATE UNIQUE INDEX fileIndex ON files (id);"
            "    CREATE INDEX selfCountIndex ON mainrows(self_count);"
            "    CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
            "    CREATE INDEX totalCountIndex ON mainrows(cumulative_count);"
         << std::endl;

  std::string sql_statement("");

  root.files(sql_statement);
  root.symbols(sql_statement);
  root.mainrows_cumulative(sql_statement);
  root.summary(sql_statement);
  VIterator<Folder*> subsystems = root.CreateIterator();
  size_t ii = 0;
  for (subsystems.First(); !subsystems.IsDone(); subsystems.Next(), ++ii) {
    subsystems.CurrentItem()->mainrows(sql_statement);
    subsystems.CurrentItem()->parents(sql_statement);
    subsystems.CurrentItem()->children(sql_statement);
  }
  stream << sql_statement << std::endl;
}

calcstats()

int DQMStoreStats::calcstats ( int  mode = DQMStoreStats::considerAllME )

private

do the stats here and produce output;

mode is coded in DQMStoreStats::statMode enum (select subsets of ME, e.g. those with getLumiFlag() == true)

Definition at line 523 of file DQMStoreStats.cc.

References DQMStoreStatsSubfolder::AddBinsD(), DQMStoreStatsSubfolder::AddBinsF(), DQMStoreStatsSubfolder::AddBinsI(), DQMStoreStatsSubfolder::AddBinsS(), calcIgProfDump(), Folder::cd(), considerAllME, considerOnlyLumiProductME, gather_cfg::cout, dbe_, dumpToFWJR_, nano_mu_digi_cff::float, dqm::implementation::IGetter::getAllContents(), getEmptyMetric(), createfilelist::int, edm::Service< T >::isAvailable(), ALPAKA_ACCELERATOR_NAMESPACE::vertexFinder::it, maxbinsglobal_, maxbinssubsys_, ALCARECOPromptCalibProdSiPixelAli0T_cff::mode, seedmultiplicitymonitor_newtracking_cfi::nBins, nbinsglobal_, nbinssubsys_, castor_dqm_sourceclient_file_cfg::path, pathnamematch_, edm::JobReport::reportAnalysisFile(), runineventloop_, runonendjob_, runonendlumi_, runonendrun_, AlCaHLTBitMon_QueryRunRegistry::string, DQMStoreStatsSubfolder::subfolderName_, DQMStoreStatsSubsystem::subsystemName_, MonitorElementData::TH1D, MonitorElementData::TH1F, MonitorElementData::TH1I, MonitorElementData::TH1S, MonitorElementData::TH2D, MonitorElementData::TH2F, MonitorElementData::TH2I, MonitorElementData::TH2S, MonitorElementData::TH3F, MonitorElementData::TPROFILE, MonitorElementData::TPROFILE2D, and Folder::update().

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                                                  {
  nbinsglobal_ = 0;
  nbinssubsys_ = 0;
  maxbinsglobal_ = 0;
  maxbinssubsys_ = 0;
  std::string path = "";
  std::string subsystemname = "";
  std::string subfoldername = "";
  size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;

  std::vector<MonitorElement*> melist;
  melist = dbe_->getAllContents(pathnamematch_);

  Folder dbeFolder("root");
  DQMStoreStatsTopLevel dqmStoreStatsTopLevel;

  // loop all ME
  for (auto& it : melist) {
    // consider only ME with getLumiFlag() == true ?
    if (mode == DQMStoreStats::considerOnlyLumiProductME && !(it->getLumiFlag()))
      continue;

    // figure out subsystem/subfolder names
    const std::string& path = it->getPathname();

    subfolderStringBegin = 0;
    Folder* curr = &dbeFolder;
    while (true) {
      subfolderStringEnd = path.find('/', subfolderStringBegin);
      if (std::string::npos == subfolderStringEnd) {
        curr = curr->cd(path.substr(subfolderStringBegin, path.size() - subfolderStringBegin));
        break;
      }
      curr = curr->cd(path.substr(subfolderStringBegin, subfolderStringEnd - subfolderStringBegin));
      subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
    }

    // protection against ghost ME with empty paths
    if (path.empty())
      continue;

    subsysStringEnd = path.find('/', 0);
    if (std::string::npos == subsysStringEnd)
      subsysStringEnd = path.size();  // no subfolder

    // new subsystem?
    if (path.substr(0, subsysStringEnd) != subsystemname) {
      DQMStoreStatsSubsystem aSubsystem;
      subsystemname = path.substr(0, subsysStringEnd);
      aSubsystem.subsystemName_ = subsystemname;
      dqmStoreStatsTopLevel.push_back(aSubsystem);
      subfoldername = "";
    }

    // get subfolder name (if there is one..)
    if (path.size() == subsysStringEnd) {
      // no subfolders in subsystem, make dummy
      DQMStoreStatsSubfolder aSubfolder;
      aSubfolder.subfolderName_ = subsystemname;  // <-- for tagging this case
      dqmStoreStatsTopLevel.back().push_back(aSubfolder);
    }

    else {
      // there is a subfolder, get its name
      subfolderStringEnd = path.find('/', subsysStringEnd + 1);
      if (std::string::npos == subfolderStringEnd)
        subfolderStringEnd = path.size();

      // new subfolder?
      if (path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1) != subfoldername) {
        subfoldername = path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1);
        DQMStoreStatsSubfolder aSubfolder;
        aSubfolder.subfolderName_ = subfoldername;
        dqmStoreStatsTopLevel.back().push_back(aSubfolder);
      }
    }

    // shortcut
    DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();

    switch (it->kind()) {
        // one-dim ME
      case MonitorElement::Kind::TH1F:
        currentSubfolder.AddBinsF(it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0));
        curr->update(it->getNbinsX(),
                     getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0),
                     it->getNbinsX() * sizeof(float));
        break;
      case MonitorElement::Kind::TH1S:
        currentSubfolder.AddBinsS(it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0));
        curr->update(it->getNbinsX(),
                     getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0),
                     it->getNbinsX() * sizeof(short));
        break;
      case MonitorElement::Kind::TH1D:
        currentSubfolder.AddBinsD(it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0));
        curr->update(it->getNbinsX(),
                     getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0),
                     it->getNbinsX() * sizeof(double));
        break;
      case MonitorElement::Kind::TH1I:
        currentSubfolder.AddBinsI(it->getNbinsX(), getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0));
        curr->update(it->getNbinsX(),
                     getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0),
                     it->getNbinsX() * sizeof(int));
        break;
      case MonitorElement::Kind::TPROFILE:
        currentSubfolder.AddBinsD(it->getNbinsX(),
                                  getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0));
        curr->update(it->getNbinsX(),
                     getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0),
                     it->getNbinsX() * sizeof(double));
        break;

        // two-dim ME
      case MonitorElement::Kind::TH2F:
        currentSubfolder.AddBinsF(
            it->getNbinsX() * it->getNbinsY(),
            getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
        curr->update(it->getNbinsX() * it->getNbinsY(),
                     getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                     it->getNbinsX() * it->getNbinsY() * sizeof(float));
        break;
      case MonitorElement::Kind::TH2S:
        currentSubfolder.AddBinsS(
            it->getNbinsX() * it->getNbinsY(),
            getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
        curr->update(it->getNbinsX() * it->getNbinsY(),
                     getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                     it->getNbinsX() * it->getNbinsY() * sizeof(short));
        break;
      case MonitorElement::Kind::TH2D:
        currentSubfolder.AddBinsD(
            it->getNbinsX() * it->getNbinsY(),
            getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
        curr->update(it->getNbinsX() * it->getNbinsY(),
                     getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                     it->getNbinsX() * it->getNbinsY() * sizeof(double));
        break;
      case MonitorElement::Kind::TH2I:
        currentSubfolder.AddBinsI(
            it->getNbinsX() * it->getNbinsY(),
            getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
        curr->update(it->getNbinsX() * it->getNbinsY(),
                     getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                     it->getNbinsX() * it->getNbinsY() * sizeof(int));
        break;
      case MonitorElement::Kind::TPROFILE2D:
        currentSubfolder.AddBinsD(
            it->getNbinsX() * it->getNbinsY(),
            getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
        curr->update(it->getNbinsX() * it->getNbinsY(),
                     getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                     it->getNbinsX() * it->getNbinsY() * sizeof(double));
        break;

        // three-dim ME
      case MonitorElement::Kind::TH3F:
        currentSubfolder.AddBinsF(
            it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
            getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2));
        curr->update(
            it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
            getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2),
            it->getNbinsX() * it->getNbinsY() * it->getNbinsZ() * sizeof(float));
        break;

      default: {
      }
        // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
        // which we don't care much about. Alternatively:

        //   std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: "
        //               << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
        //               << "  in path: \"" << path << "\" not considered." << std::endl;
    }
  }

  if (mode == DQMStoreStats::considerAllME)
    calcIgProfDump(dbeFolder);

  // OUTPUT

  std::cout << endl;
  std::cout << "==========================================================================================="
            << std::endl;
  std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
  if (mode == DQMStoreStats::considerAllME)
    std::cout << "FOR ALL ME" << std::endl;
  else if (mode == DQMStoreStats::considerOnlyLumiProductME)
    std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
  std::cout << "==========================================================================================="
            << std::endl;
  std::cout << endl;

  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << "Configuration:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << " > running ";
  if (runonendrun_)
    std::cout << "on run end." << std::endl;
  if (runonendlumi_)
    std::cout << "on lumi end." << std::endl;
  if (runonendjob_)
    std::cout << "on job end." << std::endl;
  if (runineventloop_)
    std::cout << "in event loop." << std::endl;
  std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
  std::cout << std::endl;

  // dump folder structure
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << "Top level folder tree:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
    std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;

    for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
      std::cout << "  |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
    }
  }

  // dump mem/bin table

  unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;

  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << "Detailed ressource usage information ";
  if (mode == DQMStoreStats::considerAllME)
    std::cout << "FOR ALL ME" << std::endl;
  else if (mode == DQMStoreStats::considerOnlyLumiProductME)
    std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << "subsystem/folder                          histograms       bins        Empty bins     Empty/Total      "
               "bins per       MB         kB per"
            << std::endl;
  std::cout << "                                           (total)        (total)        (total)                      "
               "histogram     (total)    histogram  "
            << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
    std::cout << it0->subsystemName_ << std::endl;

    unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;

    for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
      // fixed-size working copy
      std::string thisSubfolderName(it1->subfolderName_);
      if (thisSubfolderName.size() > 30) {
        thisSubfolderName.resize(30);
        thisSubfolderName.replace(thisSubfolderName.size() - 3, 3, 3, '.');
      }

      std::cout << " -> " << std::setw(30) << std::left << thisSubfolderName;
      std::cout << std::setw(14) << std::right << it1->totalHistos_;
      std::cout << std::setw(14) << std::right << it1->totalBins_;
      std::cout << std::setw(14) << std::right << it1->totalEmptyBins_;
      std::cout << std::setw(14) << std::right << std::setprecision(3)
                << (float)it1->totalEmptyBins_ / (float)it1->totalBins_;

      // bins/histogram, need to catch nan if histos=0
      if (it1->totalHistos_) {
        std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalBins_ / float(it1->totalHistos_);
      } else
        std::cout << std::setw(14) << std::right << "-";

      std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalMemory_ / 1024. / 1024.;

      // mem/histogram, need to catch nan if histos=0
      if (it1->totalHistos_) {
        std::cout << std::setw(14) << std::right << std::setprecision(3)
                  << it1->totalMemory_ / 1024. / it1->totalHistos_;
      } else
        std::cout << std::setw(14) << std::right << "-";

      std::cout << std::endl;

      // collect totals
      nHistograms += it1->totalHistos_;
      nBins += it1->totalBins_;
      nEmptyBins += it1->totalEmptyBins_;
      nBytes += it1->totalMemory_;
    }

    overallNHistograms += nHistograms;
    overallNBins += nBins;
    overallNBytes += nBytes;

    // display totals
    std::cout << "    " << std::setw(30) << std::left << "SUBSYSTEM TOTAL";
    std::cout << std::setw(14) << std::right << nHistograms;
    std::cout << std::setw(14) << std::right << nBins;
    std::cout << std::setw(14) << std::right << nEmptyBins;
    std::cout << std::setw(14) << std::right << (float)nEmptyBins / (float)nBins;
    std::cout << std::setw(14) << std::right << std::setprecision(3) << nBins / float(nHistograms);
    std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / 1000.;
    std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / nHistograms;
    std::cout << std::endl;

    std::cout << ".........................................................................................."
              << std::endl;
  }

  // dump total
  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << "Grand total ";
  if (mode == DQMStoreStats::considerAllME)
    std::cout << "FOR ALL ME:" << std::endl;
  else if (mode == DQMStoreStats::considerOnlyLumiProductME)
    std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
  std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether"
            << std::endl;
  std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB"
            << std::endl;

  std::cout << endl;
  std::cout << "==========================================================================================="
            << std::endl;
  std::cout << "[DQMStoreStats::calcstats] -- End of output ";
  if (mode == DQMStoreStats::considerAllME)
    std::cout << "FOR ALL ME." << std::endl;
  else if (mode == DQMStoreStats::considerOnlyLumiProductME)
    std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
  std::cout << "==========================================================================================="
            << std::endl;
  std::cout << endl;

  // Put together a simplified version of the complete dump that is
  // sent to std::cout. Just dump the very basic information,
  // i.e. summary for each folder, both for run and LS products.
  if (dumpToFWJR_) {
    edm::Service<edm::JobReport> jr;
    // Do not even try if the FWJR service is not available.
    if (!jr.isAvailable())
      return 0;
    // Prepare appropriate map to store FWJR output.
    std::map<std::string, std::string> jrInfo;

    jrInfo["Source"] = "DQMServices/Components";
    jrInfo["FileClass"] = "DQMStoreStats";
    if (runonendrun_)
      jrInfo["DumpType"] = "EndRun";
    if (runonendlumi_)
      jrInfo["DumpType"] = "EndLumi";
    if (runonendjob_)
      jrInfo["DumpType"] = "EndJob";
    if (runineventloop_)
      jrInfo["DumpType"] = "EventLoop";
    if (mode == DQMStoreStats::considerAllME)
      jrInfo["Type"] = "RunProduct";
    else if (mode == DQMStoreStats::considerOnlyLumiProductME)
      jrInfo["Type"] = "LumiProduct";

    jrInfo["pathNameMatch"] = pathnamematch_;

    for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
      unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
      for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
        // collect totals
        nHistograms += it1->totalHistos_;
        nBins += it1->totalBins_;
        nEmptyBins += it1->totalEmptyBins_;
        nBytes += it1->totalMemory_;
      }
      std::stringstream iss("");
      iss << nHistograms;
      jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
      iss.str("");
      iss << nBins;
      jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
      iss.str("");
      iss << nEmptyBins;
      jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
      iss.str("");
      iss << ((float)nEmptyBins / (float)nBins);
      jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
      iss.str("");
      iss << ((float)nBins / (float)nHistograms);
      jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
      iss.str("");
      iss << nBytes / 1024. / 1024.;
      jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
      iss.str("");
      iss << nBytes / 1024. / nHistograms;
      jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
    }
    jr->reportAnalysisFile("DQMStatsReport", jrInfo);
  }

  return 0;
}

dumpMemoryProfile()

void DQMStoreStats::dumpMemoryProfile ( )

private

Definition at line 932 of file DQMStoreStats.cc.

References gather_cfg::cout, dumpMemHistory_, isOpenProcFileSuccessful_, ALPAKA_ACCELERATOR_NAMESPACE::vertexFinder::it, memoryHistoryVector_, download_sqlite_cfg::outputFile, and startingTime_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                      {
  std::cout << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;
  std::cout << "Memory profile:" << std::endl;
  std::cout << "------------------------------------------------------------------------------------------"
            << std::endl;

  // determine virtual memory maximum
  std::pair<time_t, unsigned int> maxItem(0, 0);
  for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
    if (it->second > maxItem.second) {
      maxItem = *it;
    }
  }

  std::stringstream rootOutputFileName;
  rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";

  // dump memory history to root file
  if (dumpMemHistory_ && isOpenProcFileSuccessful_) {
    TFile outputFile(rootOutputFileName.str().c_str(), "RECREATE");

    int aTime;
    float aMb;

    TTree memHistoryTree("dqmstorestats_memhistory", "memory history");
    memHistoryTree.Branch("seconds", &aTime, "seconds/I");
    memHistoryTree.Branch("megabytes", &aMb, "megabytes/F");
    for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
      aTime = it->first - startingTime_;
      aMb = it->second / 1000.;
      memHistoryTree.Fill();
    }

    outputFile.Write();
    outputFile.Close();
  }

  std::cout << "Approx. maximum total virtual memory size of job: ";
  if (isOpenProcFileSuccessful_ && !memoryHistoryVector_.empty()) {
    std::cout << maxItem.second / 1000. << " MB (reached " << maxItem.first - startingTime_
              << " sec. after constructor called)," << std::endl;
    std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size()
              << " samples)" << std::endl;
  } else {
    std::cout << "(could not be determined)" << std::endl;
  }

  std::cout << std::endl << std::endl;
}

endJob()

void DQMStoreStats::endJob ( void  )

overrideprotectedvirtual

Reimplemented from edm::one::EDAnalyzerBase.

Definition at line 1098 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendjob_.

                           {
  if (runonendjob_) {
    calcstats(DQMStoreStats::considerAllME);
    calcstats(DQMStoreStats::considerOnlyLumiProductME);
    dumpMemoryProfile();
  }
}

endLuminosityBlock()

void DQMStoreStats::endLuminosityBlock ( const edm::LuminosityBlock &  lumiSeg, const edm::EventSetup &  c  )

overrideprotected

Definition at line 1071 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendlumi_.

                                                                                                {
  if (runonendlumi_) {
    calcstats(DQMStoreStats::considerAllME);
    calcstats(DQMStoreStats::considerOnlyLumiProductME);
    dumpMemoryProfile();
  }
}

endRun()

void DQMStoreStats::endRun ( const edm::Run &  r, const edm::EventSetup &  c  )

overrideprotected

Definition at line 1087 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendrun_.

                                                                  {
  if (runonendrun_) {
    calcstats(DQMStoreStats::considerAllME);
    calcstats(DQMStoreStats::considerOnlyLumiProductME);
    dumpMemoryProfile();
  }
}

print()

void DQMStoreStats::print ( void  )

private

Definition at line 987 of file DQMStoreStats.cc.

References gather_cfg::cout, maxbinsmesubsys_, maxbinssubsys_, nbinssubsys_, nmesubsys_, subfolder_, and subsystem_.

                          {
  // subsystem info printout
  std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
  std::cout << "  " << subfolder_ << ": ";
  std::cout << nmesubsys_ << " histograms with " << nbinssubsys_ << " bins. ";
  if (nmesubsys_ > 0)
    std::cout << nbinssubsys_ / nmesubsys_ << " bins/histogram ";
  std::cout << std::endl;
  std::cout << "  Largest histogram: " << maxbinsmesubsys_ << " with " << maxbinssubsys_ << " bins." << std::endl;
}

readMemoryEntry()

std::pair< unsigned int, unsigned int > DQMStoreStats::readMemoryEntry ( ) const

private

read virtual memory size from /proc/<pid>/status file

Definition at line 1001 of file DQMStoreStats.cc.

References recoMuon::in, isOpenProcFileSuccessful_, procFileName_, AlCaHLTBitMon_QueryRunRegistry::string, and hcalRecHitTable_cff::time.

Referenced by analyze().

                                                                         {
  // see if initial test reading was successful
  if (isOpenProcFileSuccessful_) {
    std::ifstream procFile(procFileName_.str().c_str(), ios::in);

    std::string readBuffer("");
    unsigned int memSize = 0;

    // scan procfile
    while (!procFile.eof()) {
      procFile >> readBuffer;
      if (std::string("VmSize:") == readBuffer) {
        procFile >> memSize;
        break;
      }
    }

    procFile.close();
    return std::pair<time_t, unsigned int>(time(nullptr), memSize);
  }

  return std::pair<time_t, unsigned int>(0, 0);
}

Member Data Documentation

dbe_

DQMStore* DQMStoreStats::dbe_

private

Definition at line 346 of file DQMStoreStats.cc.

Referenced by beginJob(), and calcstats().

dumpMemHistory_

bool DQMStoreStats::dumpMemHistory_

private

Definition at line 373 of file DQMStoreStats.cc.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

dumpToFWJR_

bool DQMStoreStats::dumpToFWJR_

private

Definition at line 374 of file DQMStoreStats.cc.

Referenced by calcstats(), and DQMStoreStats().

isOpenProcFileSuccessful_

bool DQMStoreStats::isOpenProcFileSuccessful_

private

Definition at line 366 of file DQMStoreStats.cc.

Referenced by beginJob(), dumpMemoryProfile(), and readMemoryEntry().

maxbinsglobal_

int DQMStoreStats::maxbinsglobal_

private

Definition at line 355 of file DQMStoreStats.cc.

Referenced by calcstats().

maxbinsmeglobal_

std::string DQMStoreStats::maxbinsmeglobal_

private

Definition at line 357 of file DQMStoreStats.cc.

maxbinsmesubsys_

std::string DQMStoreStats::maxbinsmesubsys_

private

Definition at line 358 of file DQMStoreStats.cc.

Referenced by print().

maxbinssubsys_

int DQMStoreStats::maxbinssubsys_

private

Definition at line 356 of file DQMStoreStats.cc.

Referenced by calcstats(), and print().

memoryHistoryVector_

std::vector<std::pair<time_t, unsigned int> > DQMStoreStats::memoryHistoryVector_

private

Definition at line 364 of file DQMStoreStats.cc.

Referenced by analyze(), and dumpMemoryProfile().

nbinsglobal_

int DQMStoreStats::nbinsglobal_

private

Definition at line 351 of file DQMStoreStats.cc.

Referenced by calcstats().

nbinssubsys_

int DQMStoreStats::nbinssubsys_

private

Definition at line 352 of file DQMStoreStats.cc.

Referenced by calcstats(), and print().

nmeglobal_

int DQMStoreStats::nmeglobal_

private

Definition at line 353 of file DQMStoreStats.cc.

nmesubsys_

int DQMStoreStats::nmesubsys_

private

Definition at line 354 of file DQMStoreStats.cc.

Referenced by print().

parameters_

edm::ParameterSet DQMStoreStats::parameters_

private

Definition at line 347 of file DQMStoreStats.cc.

Referenced by Mixins._TypedParameterizable::clone(), Types.PSet::clone(), Types.EDAlias::clone(), Mixins._TypedParameterizable::copy(), and DQMStoreStats().

pathnamematch_

std::string DQMStoreStats::pathnamematch_

private

Definition at line 361 of file DQMStoreStats.cc.

Referenced by calcstats(), and DQMStoreStats().

procFileName_

std::stringstream DQMStoreStats::procFileName_

private

Definition at line 367 of file DQMStoreStats.cc.

Referenced by beginJob(), and readMemoryEntry().

runineventloop_

bool DQMStoreStats::runineventloop_

private

Definition at line 372 of file DQMStoreStats.cc.

Referenced by analyze(), calcstats(), and DQMStoreStats().

runonendjob_

bool DQMStoreStats::runonendjob_

private

Definition at line 370 of file DQMStoreStats.cc.

Referenced by calcstats(), DQMStoreStats(), and endJob().

runonendlumi_

bool DQMStoreStats::runonendlumi_

private

Definition at line 371 of file DQMStoreStats.cc.

Referenced by calcstats(), DQMStoreStats(), and endLuminosityBlock().

runonendrun_

bool DQMStoreStats::runonendrun_

private

Definition at line 369 of file DQMStoreStats.cc.

Referenced by calcstats(), DQMStoreStats(), and endRun().

startingTime_

time_t DQMStoreStats::startingTime_

private

Definition at line 365 of file DQMStoreStats.cc.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

statsdepth_

int DQMStoreStats::statsdepth_

private

Definition at line 360 of file DQMStoreStats.cc.

Referenced by DQMStoreStats().

subfolder_

std::string DQMStoreStats::subfolder_

private

Definition at line 350 of file DQMStoreStats.cc.

Referenced by print().

subsystem_

std::string DQMStoreStats::subsystem_

private

Definition at line 349 of file DQMStoreStats.cc.

Referenced by print().

verbose_

int DQMStoreStats::verbose_

private

Definition at line 362 of file DQMStoreStats.cc.

Referenced by DQMStoreStats().