Inheritance diagram for DQMStoreStats:

Public Types
enum	statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 }

Public Types inherited from edm::one::EDAnalyzerBase
typedef EDAnalyzerBase	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Public Member Functions
	DQMStoreStats (const edm::ParameterSet &)

	~DQMStoreStats () override=default

Public Member Functions inherited from edm::one::EDAnalyzer< edm::one::WatchRuns, edm::one::WatchLuminosityBlocks >
	EDAnalyzer ()=default

	EDAnalyzer (const EDAnalyzer &)=delete

SerialTaskQueue *	globalLuminosityBlocksQueue () final

SerialTaskQueue *	globalRunsQueue () final

const EDAnalyzer &	operator= (const EDAnalyzer &)=delete

bool	wantsGlobalLuminosityBlocks () const final

bool	wantsGlobalRuns () const final

bool	wantsInputProcessBlocks () const final

bool	wantsProcessBlocks () const final

Public Member Functions inherited from edm::one::EDAnalyzerBase
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)

	EDAnalyzerBase ()

ModuleDescription const &	moduleDescription () const

bool	wantsStreamLuminosityBlocks () const

bool	wantsStreamRuns () const

	~EDAnalyzerBase () override

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ESResolverIndex const *	esGetTokenIndices (edm::Transition iTrans) const

std::vector< ESResolverIndex > const &	esGetTokenIndicesVector (edm::Transition iTrans) const

std::vector< ESRecordIndex > const &	esGetTokenRecordIndicesVector (edm::Transition iTrans) const

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::array< std::vector< ModuleDescription const > , NumBranchTypes > &modulesAll, std::vector< ModuleProcessName > &modulesInPreviousProcesses, ProductRegistry const &preg, std::map< std::string, ModuleDescription const *> const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

void	selectInputProcessBlocks (ProductRegistry const &productRegistry, ProcessBlockHelperBase const &processBlockHelperBase)

ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

void	updateLookup (eventsetup::ESRecordsToProductResolverIndices const &)

virtual	~EDConsumerBase () noexcept(false)

Protected Member Functions
void	analyze (const edm::Event &e, const edm::EventSetup &c) override

void	beginJob () override

void	beginLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) override

void	beginRun (const edm::Run &r, const edm::EventSetup &c) override

void	endJob () override

void	endLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) override

void	endRun (const edm::Run &r, const edm::EventSetup &c) override

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

template<BranchType B = InEvent>
EDConsumerBaseAdaptor< B >	consumes (edm::InputTag tag) noexcept

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes ()

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag const &tag)

template<Transition Tr = Transition::Event>
constexpr auto	esConsumes ()

template<Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag tag)

template<Transition Tr = Transition::Event>
ESGetTokenGeneric	esConsumes (eventsetup::EventSetupRecordKey const &iRecord, eventsetup::DataKey const &iKey)
	Used with EventSetupRecord::doGet. More...

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

void	resetItemsToGetFrom (BranchType iType)

Private Member Functions
void	calcIgProfDump (Folder &)

int	calcstats (int)

void	dumpMemoryProfile ()

void	print ()

std::pair< unsigned int, unsigned int >	readMemoryEntry () const

Private Attributes
DQMStore *	dbe_

bool	dumpMemHistory_

bool	dumpToFWJR_

bool	isOpenProcFileSuccessful_

int	maxbinsglobal_

std::string	maxbinsmeglobal_

std::string	maxbinsmesubsys_

int	maxbinssubsys_

std::vector< std::pair< time_t, unsigned int > >	memoryHistoryVector_

int	nbinsglobal_

int	nbinssubsys_

int	nmeglobal_

int	nmesubsys_

edm::ParameterSet	parameters_

std::string	pathnamematch_

std::stringstream	procFileName_

bool	runineventloop_

bool	runonendjob_

bool	runonendlumi_

bool	runonendrun_

time_t	startingTime_

int	statsdepth_

std::string	subfolder_

std::string	subsystem_

int	verbose_

Additional Inherited Members
Static Public Member Functions inherited from edm::one::EDAnalyzerBase
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &descriptions)

Detailed Description

DQMStoreStats itself

DQM Test Client

Author
Andreas Meyer CERN

Jan Olzem DESY

Definition at line 312 of file DQMStoreStats.cc.

Member Enumeration Documentation

◆ statsMode

enum DQMStoreStats::statsMode

Enumerator
considerAllME
considerOnlyLumiProductME

Definition at line 317 of file DQMStoreStats.cc.

317 { considerAllME = 0, considerOnlyLumiProductME = 1 };

DQMStoreStats::considerAllME

Definition: DQMStoreStats.cc:317

DQMStoreStats::considerOnlyLumiProductME

Definition: DQMStoreStats.cc:317

Constructor & Destructor Documentation

◆ DQMStoreStats()

DQMStoreStats::DQMStoreStats ( const edm::ParameterSet & ps )

Definition at line 413 of file DQMStoreStats.cc.

References dumpMemHistory_, dumpToFWJR_, edm::ParameterSet::getUntrackedParameter(), parameters_, pathnamematch_, runineventloop_, runonendjob_, runonendlumi_, runonendrun_, startingTime_, statsdepth_, AlCaHLTBitMon_QueryRunRegistry::string, hcalRecHitTable_cff::time, and verbose_.

     : subsystem_(""),
       subfolder_(""),
       nbinsglobal_(0),
       nbinssubsys_(0),
       nmeglobal_(0),
       nmesubsys_(0),
       maxbinsglobal_(0),
       maxbinssubsys_(0),
       maxbinsmeglobal_(""),
       maxbinsmesubsys_(""),
       statsdepth_(1),
       pathnamematch_(""),
       verbose_(0) {
   parameters_ = ps;
   pathnamematch_ = ps.getUntrackedParameter<std::string>("pathNameMatch", pathnamematch_);
   statsdepth_ = ps.getUntrackedParameter<int>("statsDepth", statsdepth_);
   verbose_ = ps.getUntrackedParameter<int>("verbose", verbose_);
   dumpMemHistory_ = ps.getUntrackedParameter<bool>("dumpMemoryHistory", false);
   runonendrun_ = ps.getUntrackedParameter<bool>("runOnEndRun", true);
   runonendjob_ = ps.getUntrackedParameter<bool>("runOnEndJob", false);
   runonendlumi_ = ps.getUntrackedParameter<bool>("runOnEndLumi", false);
   runineventloop_ = ps.getUntrackedParameter<bool>("runInEventLoop", false);
   dumpToFWJR_ = ps.getUntrackedParameter<bool>("dumpToFWJR", false);
 
   startingTime_ = time(nullptr);
 }

◆ ~DQMStoreStats()

DQMStoreStats::~DQMStoreStats ( )

overridedefault

Member Function Documentation

◆ analyze()

void DQMStoreStats::analyze	(	const edm::Event &	e,
		const edm::EventSetup &	c
	)

overrideprotectedvirtual

Implements edm::one::EDAnalyzerBase.

Definition at line 1057 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), memoryHistoryVector_, readMemoryEntry(), and runineventloop_.

                                                                          {
   //now read virtual memory size from proc folder
   memoryHistoryVector_.emplace_back(readMemoryEntry());
 
   if (runineventloop_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

◆ beginJob()

void DQMStoreStats::beginJob ( void )

overrideprotectedvirtual

Reimplemented from edm::one::EDAnalyzerBase.

Definition at line 1028 of file DQMStoreStats.cc.

References DMR_cfg::cerr, dbe_, recoMuon::in, isOpenProcFileSuccessful_, Utilities::operator, and procFileName_.

                              {
   dbe_ = Service<DQMStore>().operator->();
 
   // access the proc/ folder for memory information
   procFileName_ << "/proc/" << getpid() << "/status";
 
   // open for a test
   std::ifstream procFile(procFileName_.str().c_str(), ios::in);
 
   if (procFile.good()) {
     isOpenProcFileSuccessful_ = true;
   } else {
     std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
     std::cerr << "  Total memory profile will not be available." << std::endl;
     isOpenProcFileSuccessful_ = false;
   }
 
   procFile.close();
 }

◆ beginLuminosityBlock()

void DQMStoreStats::beginLuminosityBlock	(	const edm::LuminosityBlock &	lumiSeg,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 1082 of file DQMStoreStats.cc.

1082 {}

◆ beginRun()

void DQMStoreStats::beginRun	(	const edm::Run &	r,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 1052 of file DQMStoreStats.cc.

1052 {}

◆ calcIgProfDump()

void DQMStoreStats::calcIgProfDump ( Folder & root )

private

Definition at line 441 of file DQMStoreStats.cc.

References Folder::children(), VIterator< Item >::CurrentItem(), VIterator< Item >::First(), cuy::ii, VIterator< Item >::IsDone(), Folder::mainrows(), VIterator< Item >::Next(), Folder::parents(), cms::cuda::stream, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by calcstats().

                                                {
   std::ofstream stream("dqm-bin-stats.sql");
   stream << ""
             "    PRAGMA journal_mode=OFF;"
             "    PRAGMA count_changes=OFF;"
             "    DROP TABLE IF EXISTS files;"
             "    DROP TABLE IF EXISTS symbols;"
             "    DROP TABLE IF EXISTS mainrows;"
             "    DROP TABLE IF EXISTS children;"
             "    DROP TABLE IF EXISTS parents;"
             "    DROP TABLE IF EXISTS summary;"
             "    CREATE TABLE children ("
             "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
             "        parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
             "        from_parent_count INTEGER,"
             "        from_parent_calls INTEGER,"
             "        from_parent_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE files ("
             "        id,"
             "        name TEXT"
             "    );"
             "    CREATE TABLE mainrows ("
             "        id INTEGER PRIMARY KEY,"
             "        symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
             "        self_count INTEGER,"
             "        cumulative_count INTEGER,"
             "        kids INTEGER,"
             "        self_calls INTEGER,"
             "        total_calls INTEGER,"
             "        self_paths INTEGER,"
             "        total_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE parents ("
             "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
             "        child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
             "        to_child_count INTEGER,"
             "        to_child_calls INTEGER,"
             "        to_child_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE summary ("
             "        counter TEXT,"
             "        total_count INTEGER,"
             "        total_freq INTEGER,"
             "        tick_period REAL"
             "    );"
             "    CREATE TABLE symbols ("
             "        id,"
             "        name TEXT,"
             "        filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
             "    );"
             "    CREATE UNIQUE INDEX fileIndex ON files (id);"
             "    CREATE INDEX selfCountIndex ON mainrows(self_count);"
             "    CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
             "    CREATE INDEX totalCountIndex ON mainrows(cumulative_count);"
          << std::endl;
 
   std::string sql_statement("");
 
   root.files(sql_statement);
   root.symbols(sql_statement);
   root.mainrows_cumulative(sql_statement);
   root.summary(sql_statement);
   VIterator<Folder*> subsystems = root.CreateIterator();
   size_t ii = 0;
   for (subsystems.First(); !subsystems.IsDone(); subsystems.Next(), ++ii) {
     subsystems.CurrentItem()->mainrows(sql_statement);
     subsystems.CurrentItem()->parents(sql_statement);
     subsystems.CurrentItem()->children(sql_statement);
   }
   stream << sql_statement << std::endl;
 }

◆ calcstats()

int DQMStoreStats::calcstats ( int mode = DQMStoreStats::considerAllME )

private

do the stats here and produce output;

mode is coded in DQMStoreStats::statMode enum (select subsets of ME, e.g. those with getLumiFlag() == true)

Definition at line 523 of file DQMStoreStats.cc.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                                                   {
   nbinsglobal_ = 0;
   nbinssubsys_ = 0;
   maxbinsglobal_ = 0;
   maxbinssubsys_ = 0;
   std::string path = "";
   std::string subsystemname = "";
   std::string subfoldername = "";
   size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;
 
   std::vector<MonitorElement*> melist;
   melist = dbe_->getAllContents(pathnamematch_);
 
   Folder dbeFolder("root");
   DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
 
   // loop all ME
   for (auto& it : melist) {
     // consider only ME with getLumiFlag() == true ?
     if (mode == DQMStoreStats::considerOnlyLumiProductME && !(it->getLumiFlag()))
       continue;
 
     // figure out subsystem/subfolder names
     const std::string& path = it->getPathname();
 
     subfolderStringBegin = 0;
     Folder* curr = &dbeFolder;
     while (true) {
       subfolderStringEnd = path.find('/', subfolderStringBegin);
       if (std::string::npos == subfolderStringEnd) {
         curr = curr->cd(path.substr(subfolderStringBegin, path.size() - subfolderStringBegin));
         break;
       }
       curr = curr->cd(path.substr(subfolderStringBegin, subfolderStringEnd - subfolderStringBegin));
       subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
     }
 
     // protection against ghost ME with empty paths
     if (path.empty())
       continue;
 
     subsysStringEnd = path.find('/', 0);
     if (std::string::npos == subsysStringEnd)
       subsysStringEnd = path.size();  // no subfolder
 
     // new subsystem?
     if (path.substr(0, subsysStringEnd) != subsystemname) {
       DQMStoreStatsSubsystem aSubsystem;
       subsystemname = path.substr(0, subsysStringEnd);
       aSubsystem.subsystemName_ = subsystemname;
       dqmStoreStatsTopLevel.push_back(aSubsystem);
       subfoldername = "";
     }
 
     // get subfolder name (if there is one..)
     if (path.size() == subsysStringEnd) {
       // no subfolders in subsystem, make dummy
       DQMStoreStatsSubfolder aSubfolder;
       aSubfolder.subfolderName_ = subsystemname;  // <-- for tagging this case
       dqmStoreStatsTopLevel.back().push_back(aSubfolder);
     }
 
     else {
       // there is a subfolder, get its name
       subfolderStringEnd = path.find('/', subsysStringEnd + 1);
       if (std::string::npos == subfolderStringEnd)
         subfolderStringEnd = path.size();
 
       // new subfolder?
       if (path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1) != subfoldername) {
         subfoldername = path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1);
         DQMStoreStatsSubfolder aSubfolder;
         aSubfolder.subfolderName_ = subfoldername;
         dqmStoreStatsTopLevel.back().push_back(aSubfolder);
       }
     }
 
     // shortcut
     DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
 
     switch (it->kind()) {
         // one-dim ME
       case MonitorElement::Kind::TH1F:
         currentSubfolder.AddBinsF(it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0),
                      it->getNbinsX() * sizeof(float));
         break;
       case MonitorElement::Kind::TH1S:
         currentSubfolder.AddBinsS(it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0),
                      it->getNbinsX() * sizeof(short));
         break;
       case MonitorElement::Kind::TH1D:
         currentSubfolder.AddBinsD(it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0),
                      it->getNbinsX() * sizeof(double));
         break;
       case MonitorElement::Kind::TH1I:
         currentSubfolder.AddBinsI(it->getNbinsX(), getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0),
                      it->getNbinsX() * sizeof(int));
         break;
       case MonitorElement::Kind::TPROFILE:
         currentSubfolder.AddBinsD(it->getNbinsX(),
                                   getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0),
                      it->getNbinsX() * sizeof(double));
         break;
 
         // two-dim ME
       case MonitorElement::Kind::TH2F:
         currentSubfolder.AddBinsF(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(float));
         break;
       case MonitorElement::Kind::TH2S:
         currentSubfolder.AddBinsS(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(short));
         break;
       case MonitorElement::Kind::TH2D:
         currentSubfolder.AddBinsD(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(double));
         break;
       case MonitorElement::Kind::TH2I:
         currentSubfolder.AddBinsI(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(int));
         break;
       case MonitorElement::Kind::TPROFILE2D:
         currentSubfolder.AddBinsD(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(double));
         break;
 
         // three-dim ME
       case MonitorElement::Kind::TH3F:
         currentSubfolder.AddBinsF(
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
             getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2));
         curr->update(
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
             getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2),
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ() * sizeof(float));
         break;
 
       default: {
       }
         // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
         // which we don't care much about. Alternatively:
 
         //   std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: "
         //               << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
         //               << "  in path: \"" << path << "\" not considered." << std::endl;
     }
   }
 
   if (mode == DQMStoreStats::considerAllME)
     calcIgProfDump(dbeFolder);
 
   // OUTPUT
 
   std::cout << endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << endl;
 
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Configuration:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << " > running ";
   if (runonendrun_)
     std::cout << "on run end." << std::endl;
   if (runonendlumi_)
     std::cout << "on lumi end." << std::endl;
   if (runonendjob_)
     std::cout << "on job end." << std::endl;
   if (runineventloop_)
     std::cout << "in event loop." << std::endl;
   std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
   std::cout << std::endl;
 
   // dump folder structure
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Top level folder tree:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
     std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
 
     for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
       std::cout << "  |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
     }
   }
 
   // dump mem/bin table
 
   unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
 
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Detailed ressource usage information ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "subsystem/folder                          histograms       bins        Empty bins     Empty/Total      "
                "bins per       MB         kB per"
             << std::endl;
   std::cout << "                                           (total)        (total)        (total)                      "
                "histogram     (total)    histogram  "
             << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
     std::cout << it0->subsystemName_ << std::endl;
 
     unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
 
     for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
       // fixed-size working copy
       std::string thisSubfolderName(it1->subfolderName_);
       if (thisSubfolderName.size() > 30) {
         thisSubfolderName.resize(30);
         thisSubfolderName.replace(thisSubfolderName.size() - 3, 3, 3, '.');
       }
 
       std::cout << " -> " << std::setw(30) << std::left << thisSubfolderName;
       std::cout << std::setw(14) << std::right << it1->totalHistos_;
       std::cout << std::setw(14) << std::right << it1->totalBins_;
       std::cout << std::setw(14) << std::right << it1->totalEmptyBins_;
       std::cout << std::setw(14) << std::right << std::setprecision(3)
                 << (float)it1->totalEmptyBins_ / (float)it1->totalBins_;
 
       // bins/histogram, need to catch nan if histos=0
       if (it1->totalHistos_) {
         std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalBins_ / float(it1->totalHistos_);
       } else
         std::cout << std::setw(14) << std::right << "-";
 
       std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalMemory_ / 1024. / 1024.;
 
       // mem/histogram, need to catch nan if histos=0
       if (it1->totalHistos_) {
         std::cout << std::setw(14) << std::right << std::setprecision(3)
                   << it1->totalMemory_ / 1024. / it1->totalHistos_;
       } else
         std::cout << std::setw(14) << std::right << "-";
 
       std::cout << std::endl;
 
       // collect totals
       nHistograms += it1->totalHistos_;
       nBins += it1->totalBins_;
       nEmptyBins += it1->totalEmptyBins_;
       nBytes += it1->totalMemory_;
     }
 
     overallNHistograms += nHistograms;
     overallNBins += nBins;
     overallNBytes += nBytes;
 
     // display totals
     std::cout << "    " << std::setw(30) << std::left << "SUBSYSTEM TOTAL";
     std::cout << std::setw(14) << std::right << nHistograms;
     std::cout << std::setw(14) << std::right << nBins;
     std::cout << std::setw(14) << std::right << nEmptyBins;
     std::cout << std::setw(14) << std::right << (float)nEmptyBins / (float)nBins;
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBins / float(nHistograms);
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / 1000.;
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / nHistograms;
     std::cout << std::endl;
 
     std::cout << ".........................................................................................."
               << std::endl;
   }
 
   // dump total
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Grand total ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME:" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
   std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether"
             << std::endl;
   std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB"
             << std::endl;
 
   std::cout << endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << "[DQMStoreStats::calcstats] -- End of output ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME." << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << endl;
 
   // Put together a simplified version of the complete dump that is
   // sent to std::cout. Just dump the very basic information,
   // i.e. summary for each folder, both for run and LS products.
   if (dumpToFWJR_) {
     edm::Service<edm::JobReport> jr;
     // Do not even try if the FWJR service is not available.
     if (!jr.isAvailable())
       return 0;
     // Prepare appropriate map to store FWJR output.
     std::map<std::string, std::string> jrInfo;
 
     jrInfo["Source"] = "DQMServices/Components";
     jrInfo["FileClass"] = "DQMStoreStats";
     if (runonendrun_)
       jrInfo["DumpType"] = "EndRun";
     if (runonendlumi_)
       jrInfo["DumpType"] = "EndLumi";
     if (runonendjob_)
       jrInfo["DumpType"] = "EndJob";
     if (runineventloop_)
       jrInfo["DumpType"] = "EventLoop";
     if (mode == DQMStoreStats::considerAllME)
       jrInfo["Type"] = "RunProduct";
     else if (mode == DQMStoreStats::considerOnlyLumiProductME)
       jrInfo["Type"] = "LumiProduct";
 
     jrInfo["pathNameMatch"] = pathnamematch_;
 
     for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
       unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
       for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
         // collect totals
         nHistograms += it1->totalHistos_;
         nBins += it1->totalBins_;
         nEmptyBins += it1->totalEmptyBins_;
         nBytes += it1->totalMemory_;
       }
       std::stringstream iss("");
       iss << nHistograms;
       jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
       iss.str("");
       iss << nBins;
       jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
       iss.str("");
       iss << nEmptyBins;
       jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
       iss.str("");
       iss << ((float)nEmptyBins / (float)nBins);
       jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
       iss.str("");
       iss << ((float)nBins / (float)nHistograms);
       jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
       iss.str("");
       iss << nBytes / 1024. / 1024.;
       jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
       iss.str("");
       iss << nBytes / 1024. / nHistograms;
       jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
     }
     jr->reportAnalysisFile("DQMStatsReport", jrInfo);
   }
 
   return 0;
 }

◆ dumpMemoryProfile()

void DQMStoreStats::dumpMemoryProfile ( )

private

Definition at line 932 of file DQMStoreStats.cc.

References gather_cfg::cout, dumpMemHistory_, isOpenProcFileSuccessful_, memoryHistoryVector_, download_sqlite_cfg::outputFile, and startingTime_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                       {
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Memory profile:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
 
   // determine virtual memory maximum
   std::pair<time_t, unsigned int> maxItem(0, 0);
   for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
     if (it->second > maxItem.second) {
       maxItem = *it;
     }
   }
 
   std::stringstream rootOutputFileName;
   rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
 
   // dump memory history to root file
   if (dumpMemHistory_ && isOpenProcFileSuccessful_) {
     TFile outputFile(rootOutputFileName.str().c_str(), "RECREATE");
 
     int aTime;
     float aMb;
 
     TTree memHistoryTree("dqmstorestats_memhistory", "memory history");
     memHistoryTree.Branch("seconds", &aTime, "seconds/I");
     memHistoryTree.Branch("megabytes", &aMb, "megabytes/F");
     for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
       aTime = it->first - startingTime_;
       aMb = it->second / 1000.;
       memHistoryTree.Fill();
     }
 
     outputFile.Write();
     outputFile.Close();
   }
 
   std::cout << "Approx. maximum total virtual memory size of job: ";
   if (isOpenProcFileSuccessful_ && !memoryHistoryVector_.empty()) {
     std::cout << maxItem.second / 1000. << " MB (reached " << maxItem.first - startingTime_
               << " sec. after constructor called)," << std::endl;
     std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size()
               << " samples)" << std::endl;
   } else {
     std::cout << "(could not be determined)" << std::endl;
   }
 
   std::cout << std::endl << std::endl;
 }

◆ endJob()

void DQMStoreStats::endJob ( void )

overrideprotectedvirtual

Reimplemented from edm::one::EDAnalyzerBase.

Definition at line 1098 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendjob_.

                            {
   if (runonendjob_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

◆ endLuminosityBlock()

void DQMStoreStats::endLuminosityBlock	(	const edm::LuminosityBlock &	lumiSeg,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 1071 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendlumi_.

                                                                                                 {
   if (runonendlumi_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

◆ endRun()

void DQMStoreStats::endRun	(	const edm::Run &	r,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 1087 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendrun_.

                                                                   {
   if (runonendrun_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

◆ print()

void DQMStoreStats::print ( void )

private

Definition at line 987 of file DQMStoreStats.cc.

References gather_cfg::cout, maxbinsmesubsys_, maxbinssubsys_, nbinssubsys_, nmesubsys_, subfolder_, and subsystem_.

                           {
   // subsystem info printout
   std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
   std::cout << "  " << subfolder_ << ": ";
   std::cout << nmesubsys_ << " histograms with " << nbinssubsys_ << " bins. ";
   if (nmesubsys_ > 0)
     std::cout << nbinssubsys_ / nmesubsys_ << " bins/histogram ";
   std::cout << std::endl;
   std::cout << "  Largest histogram: " << maxbinsmesubsys_ << " with " << maxbinssubsys_ << " bins." << std::endl;
 }

◆ readMemoryEntry()

std::pair< unsigned int, unsigned int > DQMStoreStats::readMemoryEntry ( ) const

private

read virtual memory size from /proc/<pid>/status file

Definition at line 1001 of file DQMStoreStats.cc.

References recoMuon::in, isOpenProcFileSuccessful_, procFileName_, AlCaHLTBitMon_QueryRunRegistry::string, and hcalRecHitTable_cff::time.

Referenced by analyze().

                                                                          {
   // see if initial test reading was successful
   if (isOpenProcFileSuccessful_) {
     std::ifstream procFile(procFileName_.str().c_str(), ios::in);
 
     std::string readBuffer("");
     unsigned int memSize = 0;
 
     // scan procfile
     while (!procFile.eof()) {
       procFile >> readBuffer;
       if (std::string("VmSize:") == readBuffer) {
         procFile >> memSize;
         break;
       }
     }
 
     procFile.close();
     return std::pair<time_t, unsigned int>(time(nullptr), memSize);
   }
 
   return std::pair<time_t, unsigned int>(0, 0);
 }

Member Data Documentation

◆ dbe_

DQMStore* DQMStoreStats::dbe_

private

Definition at line 346 of file DQMStoreStats.cc.

Referenced by beginJob(), and calcstats().

◆ dumpMemHistory_

bool DQMStoreStats::dumpMemHistory_

private

Definition at line 373 of file DQMStoreStats.cc.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

◆ dumpToFWJR_

bool DQMStoreStats::dumpToFWJR_

private

Definition at line 374 of file DQMStoreStats.cc.

Referenced by calcstats(), and DQMStoreStats().

◆ isOpenProcFileSuccessful_

bool DQMStoreStats::isOpenProcFileSuccessful_

private

Definition at line 366 of file DQMStoreStats.cc.

Referenced by beginJob(), dumpMemoryProfile(), and readMemoryEntry().

◆ maxbinsglobal_

int DQMStoreStats::maxbinsglobal_

private

Definition at line 355 of file DQMStoreStats.cc.

Referenced by calcstats().

◆ maxbinsmeglobal_

std::string DQMStoreStats::maxbinsmeglobal_

private

Definition at line 357 of file DQMStoreStats.cc.

◆ maxbinsmesubsys_

std::string DQMStoreStats::maxbinsmesubsys_

private

Definition at line 358 of file DQMStoreStats.cc.

Referenced by print().

◆ maxbinssubsys_

int DQMStoreStats::maxbinssubsys_

private

Definition at line 356 of file DQMStoreStats.cc.

Referenced by calcstats(), and print().

◆ memoryHistoryVector_

std::vector<std::pair<time_t, unsigned int> > DQMStoreStats::memoryHistoryVector_

private

Definition at line 364 of file DQMStoreStats.cc.

Referenced by analyze(), and dumpMemoryProfile().

◆ nbinsglobal_

int DQMStoreStats::nbinsglobal_

private

Definition at line 351 of file DQMStoreStats.cc.

Referenced by calcstats().

◆ nbinssubsys_

int DQMStoreStats::nbinssubsys_

private

Definition at line 352 of file DQMStoreStats.cc.

Referenced by calcstats(), and print().

◆ nmeglobal_

int DQMStoreStats::nmeglobal_

private

Definition at line 353 of file DQMStoreStats.cc.

◆ nmesubsys_

int DQMStoreStats::nmesubsys_

private

Definition at line 354 of file DQMStoreStats.cc.

Referenced by print().

◆ parameters_

edm::ParameterSet DQMStoreStats::parameters_

private

Definition at line 347 of file DQMStoreStats.cc.

Referenced by Mixins._TypedParameterizable::clone(), Types.PSet::clone(), Types.EDAlias::clone(), Mixins._TypedParameterizable::copy(), and DQMStoreStats().

◆ pathnamematch_

std::string DQMStoreStats::pathnamematch_

private

Definition at line 361 of file DQMStoreStats.cc.

Referenced by calcstats(), and DQMStoreStats().

◆ procFileName_

std::stringstream DQMStoreStats::procFileName_

private

Definition at line 367 of file DQMStoreStats.cc.

Referenced by beginJob(), and readMemoryEntry().

◆ runineventloop_

bool DQMStoreStats::runineventloop_

private

Definition at line 372 of file DQMStoreStats.cc.

Referenced by analyze(), calcstats(), and DQMStoreStats().

◆ runonendjob_

bool DQMStoreStats::runonendjob_

private

Definition at line 370 of file DQMStoreStats.cc.

Referenced by calcstats(), DQMStoreStats(), and endJob().

◆ runonendlumi_

bool DQMStoreStats::runonendlumi_

private

Definition at line 371 of file DQMStoreStats.cc.

Referenced by calcstats(), DQMStoreStats(), and endLuminosityBlock().

◆ runonendrun_

bool DQMStoreStats::runonendrun_

private

Definition at line 369 of file DQMStoreStats.cc.

Referenced by calcstats(), DQMStoreStats(), and endRun().

◆ startingTime_

time_t DQMStoreStats::startingTime_

private

Definition at line 365 of file DQMStoreStats.cc.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

◆ statsdepth_

int DQMStoreStats::statsdepth_

private

Definition at line 360 of file DQMStoreStats.cc.

Referenced by DQMStoreStats().

◆ subfolder_

std::string DQMStoreStats::subfolder_

private

Definition at line 350 of file DQMStoreStats.cc.

Referenced by print().

◆ subsystem_

std::string DQMStoreStats::subsystem_

private

Definition at line 349 of file DQMStoreStats.cc.

Referenced by print().

◆ verbose_

int DQMStoreStats::verbose_

private

Definition at line 362 of file DQMStoreStats.cc.

Referenced by DQMStoreStats().

Public Types

Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

◆ statsMode

Constructor & Destructor Documentation

◆ DQMStoreStats()

◆ ~DQMStoreStats()

Member Function Documentation

◆ analyze()

◆ beginJob()

◆ beginLuminosityBlock()

◆ beginRun()

◆ calcIgProfDump()

◆ calcstats()

◆ dumpMemoryProfile()

◆ endJob()

◆ endLuminosityBlock()

◆ endRun()

◆ print()

◆ readMemoryEntry()

Member Data Documentation

◆ dbe_

◆ dumpMemHistory_

◆ dumpToFWJR_

◆ isOpenProcFileSuccessful_

◆ maxbinsglobal_

◆ maxbinsmeglobal_

◆ maxbinsmesubsys_

◆ maxbinssubsys_

◆ memoryHistoryVector_

◆ nbinsglobal_

◆ nbinssubsys_

◆ nmeglobal_

◆ nmesubsys_

◆ parameters_

◆ pathnamematch_

◆ procFileName_

◆ runineventloop_

◆ runonendjob_

◆ runonendlumi_

◆ runonendrun_

◆ startingTime_

◆ statsdepth_

◆ subfolder_

◆ subsystem_

◆ verbose_