#include <DQMStoreStats.h>

Inheritance diagram for DQMStoreStats:

Public Types
enum	statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 }

Public Types inherited from edm::EDAnalyzer
typedef EDAnalyzer	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Public Member Functions
	DQMStoreStats (const edm::ParameterSet &)

	~DQMStoreStats () override

Public Member Functions inherited from edm::EDAnalyzer
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)

	EDAnalyzer ()

SerialTaskQueue *	globalLuminosityBlocksQueue ()

SerialTaskQueue *	globalRunsQueue ()

ModuleDescription const &	moduleDescription () const

std::string	workerType () const

	~EDAnalyzer () override

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase &&)=default

	EDConsumerBase (EDConsumerBase const &)=delete

ESProxyIndex const *	esGetTokenIndices (edm::Transition iTrans) const

std::vector< ESProxyIndex > const &	esGetTokenIndicesVector (edm::Transition iTrans) const

std::vector< ESRecordIndex > const &	esGetTokenRecordIndicesVector (edm::Transition iTrans) const

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::array< std::vector< ModuleDescription const * > , NumBranchTypes > &modulesAll, std::vector< ModuleProcessName > &modulesInPreviousProcesses, ProductRegistry const &preg, std::map< std::string, ModuleDescription const > const &labelsToDesc, std::string const &processName) const

EDConsumerBase &	operator= (EDConsumerBase &&)=default

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

bool	registeredToConsumeMany (TypeID const &, BranchType) const

void	selectInputProcessBlocks (ProductRegistry const &productRegistry, ProcessBlockHelperBase const &processBlockHelperBase)

ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

void	updateLookup (eventsetup::ESRecordsToProxyIndices const &)

virtual	~EDConsumerBase () noexcept(false)

Protected Member Functions
void	analyze (const edm::Event &e, const edm::EventSetup &c) override

void	beginJob () override

void	beginRun (const edm::Run &r, const edm::EventSetup &c) override

void	endJob () override

void	endLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) override

void	endRun (const edm::Run &r, const edm::EventSetup &c) override

Protected Member Functions inherited from edm::EDConsumerBase
EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

template<BranchType B = InEvent>
EDConsumerBaseAdaptor< B >	consumes (edm::InputTag tag) noexcept

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()

void	consumesMany (const TypeToGet &id)

template<BranchType B>
void	consumesMany (const TypeToGet &id)

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes ()

template<Transition Tr = Transition::Event>
constexpr auto	esConsumes () noexcept

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag const &tag)

template<Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag tag) noexcept

template<Transition Tr = Transition::Event>
ESGetTokenGeneric	esConsumes (eventsetup::EventSetupRecordKey const &iRecord, eventsetup::DataKey const &iKey)
	Used with EventSetupRecord::doGet. More...

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

Private Member Functions
void	calcIgProfDump (Folder &)

int	calcstats (int)

void	dumpMemoryProfile ()

void	print ()

std::pair< unsigned int, unsigned int >	readMemoryEntry () const

Private Attributes
DQMStore *	dbe_

bool	dumpMemHistory_

bool	dumpToFWJR_

bool	isOpenProcFileSuccessful_

int	maxbinsglobal_

std::string	maxbinsmeglobal_

std::string	maxbinsmesubsys_

int	maxbinssubsys_

std::vector< std::pair< time_t, unsigned int > >	memoryHistoryVector_

int	nbinsglobal_

int	nbinssubsys_

int	nmeglobal_

int	nmesubsys_

edm::ParameterSet	parameters_

std::string	pathnamematch_

std::stringstream	procFileName_

bool	runineventloop_

bool	runonendjob_

bool	runonendlumi_

bool	runonendrun_

time_t	startingTime_

int	statsdepth_

std::string	subfolder_

std::string	subsystem_

int	verbose_

Additional Inherited Members
Static Public Member Functions inherited from edm::EDAnalyzer
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &)

static bool	wantsGlobalLuminosityBlocks ()

static bool	wantsGlobalRuns ()

static bool	wantsInputProcessBlocks ()

static bool	wantsProcessBlocks ()

static bool	wantsStreamLuminosityBlocks ()

static bool	wantsStreamRuns ()

Detailed Description

DQMStoreStats itself

DQM Test Client

Author
Andreas Meyer CERN

Jan Olzem DESY

Definition at line 299 of file DQMStoreStats.h.

Member Enumeration Documentation

◆ statsMode

enum DQMStoreStats::statsMode

Enumerator
considerAllME
considerOnlyLumiProductME

Definition at line 304 of file DQMStoreStats.h.

304 { considerAllME = 0, considerOnlyLumiProductME = 1 };

Constructor & Destructor Documentation

◆ DQMStoreStats()

DQMStoreStats::DQMStoreStats ( const edm::ParameterSet & ps )

Definition at line 47 of file DQMStoreStats.cc.

     : subsystem_(""),
       subfolder_(""),
       nbinsglobal_(0),
       nbinssubsys_(0),
       nmeglobal_(0),
       nmesubsys_(0),
       maxbinsglobal_(0),
       maxbinssubsys_(0),
       maxbinsmeglobal_(""),
       maxbinsmesubsys_(""),
       statsdepth_(1),
       pathnamematch_(""),
       verbose_(0) {
   parameters_ = ps;
   pathnamematch_ = ps.getUntrackedParameter<std::string>("pathNameMatch", pathnamematch_);
   statsdepth_ = ps.getUntrackedParameter<int>("statsDepth", statsdepth_);
   verbose_ = ps.getUntrackedParameter<int>("verbose", verbose_);
   dumpMemHistory_ = ps.getUntrackedParameter<bool>("dumpMemoryHistory", false);
   runonendrun_ = ps.getUntrackedParameter<bool>("runOnEndRun", true);
   runonendjob_ = ps.getUntrackedParameter<bool>("runOnEndJob", false);
   runonendlumi_ = ps.getUntrackedParameter<bool>("runOnEndLumi", false);
   runineventloop_ = ps.getUntrackedParameter<bool>("runInEventLoop", false);
   dumpToFWJR_ = ps.getUntrackedParameter<bool>("dumpToFWJR", false);
  
   startingTime_ = time(nullptr);
 }

References dumpMemHistory_, dumpToFWJR_, edm::ParameterSet::getUntrackedParameter(), parameters_, pathnamematch_, runineventloop_, runonendjob_, runonendlumi_, runonendrun_, startingTime_, statsdepth_, AlCaHLTBitMon_QueryRunRegistry::string, protons_cff::time, and verbose_.

◆ ~DQMStoreStats()

DQMStoreStats::~DQMStoreStats ( )

overridedefault

Member Function Documentation

◆ analyze()

void DQMStoreStats::analyze	(	const edm::Event &	e,
		const edm::EventSetup &	c
	)

overrideprotectedvirtual

Implements edm::EDAnalyzer.

Definition at line 684 of file DQMStoreStats.cc.

                                                                          {
   //now read virtual memory size from proc folder
   memoryHistoryVector_.emplace_back(readMemoryEntry());
  
   if (runineventloop_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), memoryHistoryVector_, readMemoryEntry(), and runineventloop_.

◆ beginJob()

void DQMStoreStats::beginJob ( void )

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 655 of file DQMStoreStats.cc.

                              {
   dbe_ = Service<DQMStore>().operator->();
  
   // access the proc/ folder for memory information
   procFileName_ << "/proc/" << getpid() << "/status";
  
   // open for a test
   std::ifstream procFile(procFileName_.str().c_str(), ios::in);
  
   if (procFile.good()) {
     isOpenProcFileSuccessful_ = true;
   } else {
     std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
     std::cerr << "  Total memory profile will not be available." << std::endl;
     isOpenProcFileSuccessful_ = false;
   }
  
   procFile.close();
 }

References EcnaPython_AdcPeg12_S1_10_R170298_1_0_150_Dee0::cerr, dbe_, recoMuon::in, isOpenProcFileSuccessful_, and procFileName_.

◆ beginRun()

void DQMStoreStats::beginRun	(	const edm::Run &	r,
		const edm::EventSetup &	c
	)

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 679 of file DQMStoreStats.cc.

679 {}

◆ calcIgProfDump()

void DQMStoreStats::calcIgProfDump ( Folder & root )

private

Definition at line 77 of file DQMStoreStats.cc.

                                                {
   std::ofstream stream("dqm-bin-stats.sql");
   stream << ""
             "    PRAGMA journal_mode=OFF;"
             "    PRAGMA count_changes=OFF;"
             "    DROP TABLE IF EXISTS files;"
             "    DROP TABLE IF EXISTS symbols;"
             "    DROP TABLE IF EXISTS mainrows;"
             "    DROP TABLE IF EXISTS children;"
             "    DROP TABLE IF EXISTS parents;"
             "    DROP TABLE IF EXISTS summary;"
             "    CREATE TABLE children ("
             "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
             "        parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
             "        from_parent_count INTEGER,"
             "        from_parent_calls INTEGER,"
             "        from_parent_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE files ("
             "        id,"
             "        name TEXT"
             "    );"
             "    CREATE TABLE mainrows ("
             "        id INTEGER PRIMARY KEY,"
             "        symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
             "        self_count INTEGER,"
             "        cumulative_count INTEGER,"
             "        kids INTEGER,"
             "        self_calls INTEGER,"
             "        total_calls INTEGER,"
             "        self_paths INTEGER,"
             "        total_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE parents ("
             "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
             "        child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
             "        to_child_count INTEGER,"
             "        to_child_calls INTEGER,"
             "        to_child_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE summary ("
             "        counter TEXT,"
             "        total_count INTEGER,"
             "        total_freq INTEGER,"
             "        tick_period REAL"
             "    );"
             "    CREATE TABLE symbols ("
             "        id,"
             "        name TEXT,"
             "        filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
             "    );"
             "    CREATE UNIQUE INDEX fileIndex ON files (id);"
             "    CREATE INDEX selfCountIndex ON mainrows(self_count);"
             "    CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
             "    CREATE INDEX totalCountIndex ON mainrows(cumulative_count);"
          << std::endl;
  
   std::string sql_statement("");
  
   root.files(sql_statement);
   root.symbols(sql_statement);
   root.mainrows_cumulative(sql_statement);
   root.summary(sql_statement);
   VIterator<Folder*> subsystems = root.CreateIterator();
   size_t ii = 0;
   for (subsystems.First(); !subsystems.IsDone(); subsystems.Next(), ++ii) {
     subsystems.CurrentItem()->mainrows(sql_statement);
     subsystems.CurrentItem()->parents(sql_statement);
     subsystems.CurrentItem()->children(sql_statement);
   }
   stream << sql_statement << std::endl;
 }

References Folder::children(), VIterator< Item >::CurrentItem(), VIterator< Item >::First(), cuy::ii, VIterator< Item >::IsDone(), Folder::mainrows(), VIterator< Item >::Next(), Folder::parents(), cms::cuda::stream, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by calcstats().

◆ calcstats()

int DQMStoreStats::calcstats ( int mode = DQMStoreStats::considerAllME )

private

do the stats here and produce output;

mode is coded in DQMStoreStats::statMode enum (select subsets of ME, e.g. those with getLumiFlag() == true)

Definition at line 159 of file DQMStoreStats.cc.

                                                                   {
   nbinsglobal_ = 0;
   nbinssubsys_ = 0;
   maxbinsglobal_ = 0;
   maxbinssubsys_ = 0;
   std::string path = "";
   std::string subsystemname = "";
   std::string subfoldername = "";
   size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;
  
   std::vector<MonitorElement*> melist;
   melist = dbe_->getAllContents(pathnamematch_);
  
   Folder dbeFolder("root");
   DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
  
   // loop all ME
   for (auto& it : melist) {
     // consider only ME with getLumiFlag() == true ?
     if (mode == DQMStoreStats::considerOnlyLumiProductME && !(it->getLumiFlag()))
       continue;
  
     // figure out subsystem/subfolder names
     const std::string& path = it->getPathname();
  
     subfolderStringBegin = 0;
     Folder* curr = &dbeFolder;
     while (true) {
       subfolderStringEnd = path.find('/', subfolderStringBegin);
       if (std::string::npos == subfolderStringEnd) {
         curr = curr->cd(path.substr(subfolderStringBegin, path.size() - subfolderStringBegin));
         break;
       }
       curr = curr->cd(path.substr(subfolderStringBegin, subfolderStringEnd - subfolderStringBegin));
       subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
     }
  
     // protection against ghost ME with empty paths
     if (path.empty())
       continue;
  
     subsysStringEnd = path.find('/', 0);
     if (std::string::npos == subsysStringEnd)
       subsysStringEnd = path.size();  // no subfolder
  
     // new subsystem?
     if (path.substr(0, subsysStringEnd) != subsystemname) {
       DQMStoreStatsSubsystem aSubsystem;
       subsystemname = path.substr(0, subsysStringEnd);
       aSubsystem.subsystemName_ = subsystemname;
       dqmStoreStatsTopLevel.push_back(aSubsystem);
       subfoldername = "";
     }
  
     // get subfolder name (if there is one..)
     if (path.size() == subsysStringEnd) {
       // no subfolders in subsystem, make dummy
       DQMStoreStatsSubfolder aSubfolder;
       aSubfolder.subfolderName_ = subsystemname;  // <-- for tagging this case
       dqmStoreStatsTopLevel.back().push_back(aSubfolder);
     }
  
     else {
       // there is a subfolder, get its name
       subfolderStringEnd = path.find('/', subsysStringEnd + 1);
       if (std::string::npos == subfolderStringEnd)
         subfolderStringEnd = path.size();
  
       // new subfolder?
       if (path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1) != subfoldername) {
         subfoldername = path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1);
         DQMStoreStatsSubfolder aSubfolder;
         aSubfolder.subfolderName_ = subfoldername;
         dqmStoreStatsTopLevel.back().push_back(aSubfolder);
       }
     }
  
     // shortcut
     DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
  
     switch (it->kind()) {
         // one-dim ME
       case MonitorElement::Kind::TH1F:
         currentSubfolder.AddBinsF(it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0),
                      it->getNbinsX() * sizeof(float));
         break;
       case MonitorElement::Kind::TH1S:
         currentSubfolder.AddBinsS(it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0),
                      it->getNbinsX() * sizeof(short));
         break;
       case MonitorElement::Kind::TH1D:
         currentSubfolder.AddBinsD(it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0),
                      it->getNbinsX() * sizeof(double));
         break;
       case MonitorElement::Kind::TPROFILE:
         currentSubfolder.AddBinsD(it->getNbinsX(),
                                   getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0),
                      it->getNbinsX() * sizeof(double));
         break;
  
         // two-dim ME
       case MonitorElement::Kind::TH2F:
         currentSubfolder.AddBinsF(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(float));
         break;
       case MonitorElement::Kind::TH2S:
         currentSubfolder.AddBinsS(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(short));
         break;
       case MonitorElement::Kind::TH2D:
         currentSubfolder.AddBinsD(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(double));
         break;
       case MonitorElement::Kind::TPROFILE2D:
         currentSubfolder.AddBinsD(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(double));
         break;
  
         // three-dim ME
       case MonitorElement::Kind::TH3F:
         currentSubfolder.AddBinsF(
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
             getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2));
         curr->update(
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
             getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2),
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ() * sizeof(float));
         break;
  
       default: {
       }
         // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
         // which we don't care much about. Alternatively:
  
         //   std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: "
         //               << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
         //               << "  in path: \"" << path << "\" not considered." << std::endl;
     }
   }
  
   if (mode == DQMStoreStats::considerAllME)
     calcIgProfDump(dbeFolder);
  
   // OUTPUT
  
   std::cout << endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << endl;
  
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Configuration:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << " > running ";
   if (runonendrun_)
     std::cout << "on run end." << std::endl;
   if (runonendlumi_)
     std::cout << "on lumi end." << std::endl;
   if (runonendjob_)
     std::cout << "on job end." << std::endl;
   if (runineventloop_)
     std::cout << "in event loop." << std::endl;
   std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
   std::cout << std::endl;
  
   // dump folder structure
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Top level folder tree:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
     std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
  
     for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
       std::cout << "  |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
     }
   }
  
   // dump mem/bin table
  
   unsigned int overallNHistograms = 0, overallNBins = 0, overallNEmptyBins = 0, overallNBytes = 0;
  
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Detailed ressource usage information ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "subsystem/folder                          histograms       bins        Empty bins     Empty/Total      "
                "bins per       MB         kB per"
             << std::endl;
   std::cout << "                                           (total)        (total)        (total)                      "
                "histogram     (total)    histogram  "
             << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
     std::cout << it0->subsystemName_ << std::endl;
  
     unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
  
     for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
       // fixed-size working copy
       std::string thisSubfolderName(it1->subfolderName_);
       if (thisSubfolderName.size() > 30) {
         thisSubfolderName.resize(30);
         thisSubfolderName.replace(thisSubfolderName.size() - 3, 3, 3, '.');
       }
  
       std::cout << " -> " << std::setw(30) << std::left << thisSubfolderName;
       std::cout << std::setw(14) << std::right << it1->totalHistos_;
       std::cout << std::setw(14) << std::right << it1->totalBins_;
       std::cout << std::setw(14) << std::right << it1->totalEmptyBins_;
       std::cout << std::setw(14) << std::right << std::setprecision(3)
                 << (float)it1->totalEmptyBins_ / (float)it1->totalBins_;
  
       // bins/histogram, need to catch nan if histos=0
       if (it1->totalHistos_) {
         std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalBins_ / float(it1->totalHistos_);
       } else
         std::cout << std::setw(14) << std::right << "-";
  
       std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalMemory_ / 1024. / 1024.;
  
       // mem/histogram, need to catch nan if histos=0
       if (it1->totalHistos_) {
         std::cout << std::setw(14) << std::right << std::setprecision(3)
                   << it1->totalMemory_ / 1024. / it1->totalHistos_;
       } else
         std::cout << std::setw(14) << std::right << "-";
  
       std::cout << std::endl;
  
       // collect totals
       nHistograms += it1->totalHistos_;
       nBins += it1->totalBins_;
       nEmptyBins += it1->totalEmptyBins_;
       nBytes += it1->totalMemory_;
     }
  
     overallNHistograms += nHistograms;
     overallNBins += nBins;
     overallNEmptyBins += nEmptyBins;
     overallNBytes += nBytes;
  
     // display totals
     std::cout << "    " << std::setw(30) << std::left << "SUBSYSTEM TOTAL";
     std::cout << std::setw(14) << std::right << nHistograms;
     std::cout << std::setw(14) << std::right << nBins;
     std::cout << std::setw(14) << std::right << nEmptyBins;
     std::cout << std::setw(14) << std::right << (float)nEmptyBins / (float)nBins;
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBins / float(nHistograms);
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / 1000.;
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / nHistograms;
     std::cout << std::endl;
  
     std::cout << ".........................................................................................."
               << std::endl;
   }
  
   // dump total
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Grand total ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME:" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
   std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether"
             << std::endl;
   std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB"
             << std::endl;
  
   std::cout << endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << "[DQMStoreStats::calcstats] -- End of output ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME." << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << endl;
  
   // Put together a simplified version of the complete dump that is
   // sent to std::cout. Just dump the very basic information,
   // i.e. summary for each folder, both for run and LS products.
   if (dumpToFWJR_) {
     edm::Service<edm::JobReport> jr;
     // Do not even try if the FWJR service is not available.
     if (!jr.isAvailable())
       return 0;
     // Prepare appropriate map to store FWJR output.
     std::map<std::string, std::string> jrInfo;
     unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
  
     jrInfo["Source"] = "DQMServices/Components";
     jrInfo["FileClass"] = "DQMStoreStats";
     if (runonendrun_)
       jrInfo["DumpType"] = "EndRun";
     if (runonendlumi_)
       jrInfo["DumpType"] = "EndLumi";
     if (runonendjob_)
       jrInfo["DumpType"] = "EndJob";
     if (runineventloop_)
       jrInfo["DumpType"] = "EventLoop";
     if (mode == DQMStoreStats::considerAllME)
       jrInfo["Type"] = "RunProduct";
     else if (mode == DQMStoreStats::considerOnlyLumiProductME)
       jrInfo["Type"] = "LumiProduct";
  
     jrInfo["pathNameMatch"] = pathnamematch_;
  
     for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
       unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
       for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
         // collect totals
         nHistograms += it1->totalHistos_;
         nBins += it1->totalBins_;
         nEmptyBins += it1->totalEmptyBins_;
         nBytes += it1->totalMemory_;
       }
       overallNHistograms += nHistograms;
       overallNBins += nBins;
       overallNBytes += nBytes;
       std::stringstream iss("");
       iss << nHistograms;
       jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
       iss.str("");
       iss << nBins;
       jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
       iss.str("");
       iss << nEmptyBins;
       jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
       iss.str("");
       iss << ((float)nEmptyBins / (float)nBins);
       jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
       iss.str("");
       iss << ((float)nBins / (float)nHistograms);
       jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
       iss.str("");
       iss << nBytes / 1024. / 1024.;
       jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
       iss.str("");
       iss << nBytes / 1024. / nHistograms;
       jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
     }
     jr->reportAnalysisFile("DQMStatsReport", jrInfo);
   }
  
   return 0;
 }

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

◆ dumpMemoryProfile()

void DQMStoreStats::dumpMemoryProfile ( )

private

Definition at line 559 of file DQMStoreStats.cc.

                                       {
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Memory profile:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
  
   // determine virtual memory maximum
   std::pair<time_t, unsigned int> maxItem(0, 0);
   for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
     if (it->second > maxItem.second) {
       maxItem = *it;
     }
   }
  
   std::stringstream rootOutputFileName;
   rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
  
   // dump memory history to root file
   if (dumpMemHistory_ && isOpenProcFileSuccessful_) {
     TFile outputFile(rootOutputFileName.str().c_str(), "RECREATE");
  
     int aTime;
     float aMb;
  
     TTree memHistoryTree("dqmstorestats_memhistory", "memory history");
     memHistoryTree.Branch("seconds", &aTime, "seconds/I");
     memHistoryTree.Branch("megabytes", &aMb, "megabytes/F");
     for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
       aTime = it->first - startingTime_;
       aMb = it->second / 1000.;
       memHistoryTree.Fill();
     }
  
     outputFile.Write();
     outputFile.Close();
   }
  
   std::cout << "Approx. maximum total virtual memory size of job: ";
   if (isOpenProcFileSuccessful_ && !memoryHistoryVector_.empty()) {
     std::cout << maxItem.second / 1000. << " MB (reached " << maxItem.first - startingTime_
               << " sec. after constructor called)," << std::endl;
     std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size()
               << " samples)" << std::endl;
   } else {
     std::cout << "(could not be determined)" << std::endl;
   }
  
   std::cout << std::endl << std::endl;
 }

References gather_cfg::cout, dumpMemHistory_, isOpenProcFileSuccessful_, memoryHistoryVector_, download_sqlite_cfg::outputFile, and startingTime_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

◆ endJob()

void DQMStoreStats::endJob ( void )

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 720 of file DQMStoreStats.cc.

                            {
   if (runonendjob_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendjob_.

◆ endLuminosityBlock()

void DQMStoreStats::endLuminosityBlock	(	const edm::LuminosityBlock &	lumiSeg,
		const edm::EventSetup &	c
	)

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 698 of file DQMStoreStats.cc.

                                                                                                 {
   if (runonendlumi_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendlumi_.

◆ endRun()

void DQMStoreStats::endRun	(	const edm::Run &	r,
		const edm::EventSetup &	c
	)

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 709 of file DQMStoreStats.cc.

                                                                   {
   if (runonendrun_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendrun_.

◆ print()

void DQMStoreStats::print ( void )

private

Definition at line 614 of file DQMStoreStats.cc.

                           {
   // subsystem info printout
   std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
   std::cout << "  " << subfolder_ << ": ";
   std::cout << nmesubsys_ << " histograms with " << nbinssubsys_ << " bins. ";
   if (nmesubsys_ > 0)
     std::cout << nbinssubsys_ / nmesubsys_ << " bins/histogram ";
   std::cout << std::endl;
   std::cout << "  Largest histogram: " << maxbinsmesubsys_ << " with " << maxbinssubsys_ << " bins." << std::endl;
 }

References gather_cfg::cout, maxbinsmesubsys_, maxbinssubsys_, nbinssubsys_, nmesubsys_, subfolder_, and subsystem_.

◆ readMemoryEntry()

std::pair< unsigned int, unsigned int > DQMStoreStats::readMemoryEntry ( ) const

private

read virtual memory size from /proc/<pid>/status file

Definition at line 628 of file DQMStoreStats.cc.

                                                                          {
   // see if initial test reading was successful
   if (isOpenProcFileSuccessful_) {
     std::ifstream procFile(procFileName_.str().c_str(), ios::in);
  
     std::string readBuffer("");
     unsigned int memSize = 0;
  
     // scan procfile
     while (!procFile.eof()) {
       procFile >> readBuffer;
       if (std::string("VmSize:") == readBuffer) {
         procFile >> memSize;
         break;
       }
     }
  
     procFile.close();
     return std::pair<time_t, unsigned int>(time(nullptr), memSize);
   }
  
   return std::pair<time_t, unsigned int>(0, 0);
 }

References recoMuon::in, isOpenProcFileSuccessful_, procFileName_, AlCaHLTBitMon_QueryRunRegistry::string, and protons_cff::time.

Referenced by analyze().

Member Data Documentation

◆ dbe_

DQMStore* DQMStoreStats::dbe_

private

Definition at line 332 of file DQMStoreStats.h.

Referenced by beginJob(), and calcstats().

◆ dumpMemHistory_

bool DQMStoreStats::dumpMemHistory_

private

Definition at line 359 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

◆ dumpToFWJR_

bool DQMStoreStats::dumpToFWJR_

private

Definition at line 360 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

◆ isOpenProcFileSuccessful_

bool DQMStoreStats::isOpenProcFileSuccessful_

private

Definition at line 352 of file DQMStoreStats.h.

Referenced by beginJob(), dumpMemoryProfile(), and readMemoryEntry().

◆ maxbinsglobal_

int DQMStoreStats::maxbinsglobal_

private

Definition at line 341 of file DQMStoreStats.h.

Referenced by calcstats().

◆ maxbinsmeglobal_

std::string DQMStoreStats::maxbinsmeglobal_

private

Definition at line 343 of file DQMStoreStats.h.

◆ maxbinsmesubsys_

std::string DQMStoreStats::maxbinsmesubsys_

private

Definition at line 344 of file DQMStoreStats.h.

Referenced by print().

◆ maxbinssubsys_

int DQMStoreStats::maxbinssubsys_

private

Definition at line 342 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

◆ memoryHistoryVector_

std::vector<std::pair<time_t, unsigned int> > DQMStoreStats::memoryHistoryVector_

private

Definition at line 350 of file DQMStoreStats.h.

Referenced by analyze(), and dumpMemoryProfile().

◆ nbinsglobal_

int DQMStoreStats::nbinsglobal_

private

Definition at line 337 of file DQMStoreStats.h.

Referenced by calcstats().

◆ nbinssubsys_

int DQMStoreStats::nbinssubsys_

private

Definition at line 338 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

◆ nmeglobal_

int DQMStoreStats::nmeglobal_

private

Definition at line 339 of file DQMStoreStats.h.

◆ nmesubsys_

int DQMStoreStats::nmesubsys_

private

Definition at line 340 of file DQMStoreStats.h.

Referenced by print().

◆ parameters_

edm::ParameterSet DQMStoreStats::parameters_

private

Definition at line 333 of file DQMStoreStats.h.

Referenced by Mixins._TypedParameterizable::clone(), Types.PSet::clone(), Types.EDAlias::clone(), Mixins._TypedParameterizable::copy(), and DQMStoreStats().

◆ pathnamematch_

std::string DQMStoreStats::pathnamematch_

private

Definition at line 347 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

◆ procFileName_

std::stringstream DQMStoreStats::procFileName_

private

Definition at line 353 of file DQMStoreStats.h.

Referenced by beginJob(), and readMemoryEntry().

◆ runineventloop_

bool DQMStoreStats::runineventloop_

private

Definition at line 358 of file DQMStoreStats.h.

Referenced by analyze(), calcstats(), and DQMStoreStats().

◆ runonendjob_

bool DQMStoreStats::runonendjob_

private

Definition at line 356 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endJob().

◆ runonendlumi_

bool DQMStoreStats::runonendlumi_

private

Definition at line 357 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endLuminosityBlock().

◆ runonendrun_

bool DQMStoreStats::runonendrun_

private

Definition at line 355 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endRun().

◆ startingTime_

time_t DQMStoreStats::startingTime_

private

Definition at line 351 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

◆ statsdepth_

int DQMStoreStats::statsdepth_

private

Definition at line 346 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

◆ subfolder_

std::string DQMStoreStats::subfolder_

private

Definition at line 336 of file DQMStoreStats.h.

Referenced by print().

◆ subsystem_

std::string DQMStoreStats::subsystem_

private

Definition at line 335 of file DQMStoreStats.h.

Referenced by print().

◆ verbose_

int DQMStoreStats::verbose_

private

Definition at line 348 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

Public Types

Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

◆ statsMode

Constructor & Destructor Documentation

◆ DQMStoreStats()

◆ ~DQMStoreStats()

Member Function Documentation

◆ analyze()

◆ beginJob()

◆ beginRun()

◆ calcIgProfDump()

◆ calcstats()

◆ dumpMemoryProfile()

◆ endJob()

◆ endLuminosityBlock()

◆ endRun()

◆ print()

◆ readMemoryEntry()

Member Data Documentation

◆ dbe_

◆ dumpMemHistory_

◆ dumpToFWJR_

◆ isOpenProcFileSuccessful_

◆ maxbinsglobal_

◆ maxbinsmeglobal_

◆ maxbinsmesubsys_

◆ maxbinssubsys_

◆ memoryHistoryVector_

◆ nbinsglobal_

◆ nbinssubsys_

◆ nmeglobal_

◆ nmesubsys_

◆ parameters_

◆ pathnamematch_

◆ procFileName_

◆ runineventloop_

◆ runonendjob_

◆ runonendlumi_

◆ runonendrun_

◆ startingTime_

◆ statsdepth_

◆ subfolder_

◆ subsystem_

◆ verbose_