#include <DQMStoreStats.h>

Inheritance diagram for DQMStoreStats:

Public Types
enum	statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 }

Public Types inherited from edm::EDAnalyzer
typedef EDAnalyzer	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Public Member Functions
	DQMStoreStats (const edm::ParameterSet &)

	~DQMStoreStats () override

Public Member Functions inherited from edm::EDAnalyzer
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)

	EDAnalyzer ()

SerialTaskQueue *	globalLuminosityBlocksQueue ()

SerialTaskQueue *	globalRunsQueue ()

ModuleDescription const &	moduleDescription () const

std::string	workerType () const

	~EDAnalyzer () override

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ESProxyIndex const *	esGetTokenIndices (edm::Transition iTrans) const

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

bool	registeredToConsumeMany (TypeID const &, BranchType) const

ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

void	updateLookup (eventsetup::ESRecordsToProxyIndices const &)

virtual	~EDConsumerBase () noexcept(false)

Protected Member Functions
void	analyze (const edm::Event &e, const edm::EventSetup &c) override

void	beginJob () override

void	beginRun (const edm::Run &r, const edm::EventSetup &c) override

void	endJob () override

void	endLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) override

void	endRun (const edm::Run &r, const edm::EventSetup &c) override

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()

void	consumesMany (const TypeToGet &id)

template<BranchType B>
void	consumesMany (const TypeToGet &id)

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes ()

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag const &tag)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Private Member Functions
void	calcIgProfDump (Folder &)

int	calcstats (int)

void	dumpMemoryProfile ()

void	print ()

std::pair< unsigned int, unsigned int >	readMemoryEntry () const

Private Attributes
DQMStore *	dbe_

bool	dumpMemHistory_

bool	dumpToFWJR_

bool	isOpenProcFileSuccessful_

int	maxbinsglobal_

std::string	maxbinsmeglobal_

std::string	maxbinsmesubsys_

int	maxbinssubsys_

std::vector< std::pair< time_t, unsigned int > >	memoryHistoryVector_

int	nbinsglobal_

int	nbinssubsys_

int	nmeglobal_

int	nmesubsys_

edm::ParameterSet	parameters_

std::string	pathnamematch_

std::stringstream	procFileName_

bool	runineventloop_

bool	runonendjob_

bool	runonendlumi_

bool	runonendrun_

time_t	startingTime_

int	statsdepth_

std::string	subfolder_

std::string	subsystem_

int	verbose_

Additional Inherited Members
Static Public Member Functions inherited from edm::EDAnalyzer
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &)

static bool	wantsGlobalLuminosityBlocks ()

static bool	wantsGlobalRuns ()

static bool	wantsStreamLuminosityBlocks ()

static bool	wantsStreamRuns ()

Detailed Description

DQMStoreStats itself

DQM Test Client

Author
Andreas Meyer CERN

Jan Olzem DESY

Definition at line 299 of file DQMStoreStats.h.

Member Enumeration Documentation

enum DQMStoreStats::statsMode

Enumerator
considerAllME
considerOnlyLumiProductME

Definition at line 304 of file DQMStoreStats.h.

304 { considerAllME = 0, considerOnlyLumiProductME = 1 };

DQMStoreStats::considerAllME

Definition: DQMStoreStats.h:304

DQMStoreStats::considerOnlyLumiProductME

Definition: DQMStoreStats.h:304

Constructor & Destructor Documentation

DQMStoreStats::DQMStoreStats ( const edm::ParameterSet & ps )

Definition at line 47 of file DQMStoreStats.cc.

References dumpMemHistory_, dumpToFWJR_, edm::ParameterSet::getUntrackedParameter(), parameters_, pathnamematch_, runineventloop_, runonendjob_, runonendlumi_, runonendrun_, startingTime_, statsdepth_, AlCaHLTBitMon_QueryRunRegistry::string, ntuplemaker::time, verbose_, and ~DQMStoreStats().

     : subsystem_(""),
       subfolder_(""),
       nbinsglobal_(0),
       nbinssubsys_(0),
       nmeglobal_(0),
       nmesubsys_(0),
       maxbinsglobal_(0),
       maxbinssubsys_(0),
       maxbinsmeglobal_(""),
       maxbinsmesubsys_(""),
       statsdepth_(1),
       pathnamematch_("*"),
       verbose_(0) {
   parameters_ = ps;
   pathnamematch_ = ps.getUntrackedParameter<std::string>("pathNameMatch", pathnamematch_);
   statsdepth_ = ps.getUntrackedParameter<int>("statsDepth", statsdepth_);
   verbose_ = ps.getUntrackedParameter<int>("verbose", verbose_);
   dumpMemHistory_ = ps.getUntrackedParameter<bool>("dumpMemoryHistory", false);
   runonendrun_ = ps.getUntrackedParameter<bool>("runOnEndRun", true);
   runonendjob_ = ps.getUntrackedParameter<bool>("runOnEndJob", false);
   runonendlumi_ = ps.getUntrackedParameter<bool>("runOnEndLumi", false);
   runineventloop_ = ps.getUntrackedParameter<bool>("runInEventLoop", false);
   dumpToFWJR_ = ps.getUntrackedParameter<bool>("dumpToFWJR", false);
 
   startingTime_ = time(nullptr);
 }

DQMStoreStats::~DQMStoreStats ( )

overridedefault

Referenced by DQMStoreStats().

Member Function Documentation

void DQMStoreStats::analyze	(	const edm::Event &	e,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 683 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), memoryHistoryVector_, readMemoryEntry(), and runineventloop_.

                                                                          {
   //now read virtual memory size from proc folder
   memoryHistoryVector_.emplace_back(readMemoryEntry());
 
   if (runineventloop_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

void DQMStoreStats::beginJob ( void )

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 654 of file DQMStoreStats.cc.

References beam_dqm_sourceclient-live_cfg::cerr, dbe_, recoMuon::in, isOpenProcFileSuccessful_, Utilities::operator, and procFileName_.

                              {
   dbe_ = Service<DQMStore>().operator->();
 
   // access the proc/ folder for memory information
   procFileName_ << "/proc/" << getpid() << "/status";
 
   // open for a test
   std::ifstream procFile(procFileName_.str().c_str(), ios::in);
 
   if (procFile.good()) {
     isOpenProcFileSuccessful_ = true;
   } else {
     std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
     std::cerr << "  Total memory profile will not be available." << std::endl;
     isOpenProcFileSuccessful_ = false;
   }
 
   procFile.close();
 }

void DQMStoreStats::beginRun	(	const edm::Run &	r,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 678 of file DQMStoreStats.cc.

678 {}

void DQMStoreStats::calcIgProfDump ( Folder & root )

private

Definition at line 77 of file DQMStoreStats.cc.

References Folder::children(), Folder::CreateIterator(), VIterator< Item >::CurrentItem(), Folder::files(), VIterator< Item >::First(), cuy::ii, VIterator< Item >::IsDone(), Folder::mainrows(), Folder::mainrows_cumulative(), VIterator< Item >::Next(), Folder::parents(), AlCaHLTBitMon_QueryRunRegistry::string, Folder::summary(), and Folder::symbols().

Referenced by calcstats().

                                                {
   std::ofstream stream("dqm-bin-stats.sql");
   stream << ""
             "    PRAGMA journal_mode=OFF;"
             "    PRAGMA count_changes=OFF;"
             "    DROP TABLE IF EXISTS files;"
             "    DROP TABLE IF EXISTS symbols;"
             "    DROP TABLE IF EXISTS mainrows;"
             "    DROP TABLE IF EXISTS children;"
             "    DROP TABLE IF EXISTS parents;"
             "    DROP TABLE IF EXISTS summary;"
             "    CREATE TABLE children ("
             "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
             "        parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
             "        from_parent_count INTEGER,"
             "        from_parent_calls INTEGER,"
             "        from_parent_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE files ("
             "        id,"
             "        name TEXT"
             "    );"
             "    CREATE TABLE mainrows ("
             "        id INTEGER PRIMARY KEY,"
             "        symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
             "        self_count INTEGER,"
             "        cumulative_count INTEGER,"
             "        kids INTEGER,"
             "        self_calls INTEGER,"
             "        total_calls INTEGER,"
             "        self_paths INTEGER,"
             "        total_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE parents ("
             "        self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
             "        child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
             "        to_child_count INTEGER,"
             "        to_child_calls INTEGER,"
             "        to_child_paths INTEGER,"
             "        pct REAL"
             "    );"
             "    CREATE TABLE summary ("
             "        counter TEXT,"
             "        total_count INTEGER,"
             "        total_freq INTEGER,"
             "        tick_period REAL"
             "    );"
             "    CREATE TABLE symbols ("
             "        id,"
             "        name TEXT,"
             "        filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
             "    );"
             "    CREATE UNIQUE INDEX fileIndex ON files (id);"
             "    CREATE INDEX selfCountIndex ON mainrows(self_count);"
             "    CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
             "    CREATE INDEX totalCountIndex ON mainrows(cumulative_count);"
          << std::endl;
 
   std::string sql_statement("");
 
   root.files(sql_statement);
   root.symbols(sql_statement);
   root.mainrows_cumulative(sql_statement);
   root.summary(sql_statement);
   VIterator<Folder*> subsystems = root.CreateIterator();
   size_t ii = 0;
   for (subsystems.First(); !subsystems.IsDone(); subsystems.Next(), ++ii) {
     subsystems.CurrentItem()->mainrows(sql_statement);
     subsystems.CurrentItem()->parents(sql_statement);
     subsystems.CurrentItem()->children(sql_statement);
   }
   stream << sql_statement << std::endl;
 }

int DQMStoreStats::calcstats ( int mode = DQMStoreStats::considerAllME )

private

do the stats here and produce output;

mode is coded in DQMStoreStats::statMode enum (select subsets of ME, e.g. those with getLumiFlag() == true)

Definition at line 159 of file DQMStoreStats.cc.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                                                   {
   nbinsglobal_ = 0;
   nbinssubsys_ = 0;
   maxbinsglobal_ = 0;
   maxbinssubsys_ = 0;
   std::string path = "";
   std::string subsystemname = "";
   std::string subfoldername = "";
   size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;
 
   std::vector<MonitorElement*> melist;
   melist = dbe_->getMatchingContents(pathnamematch_);
 
   Folder dbeFolder("root");
   DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
 
   // loop all ME
   for (auto& it : melist) {
     // consider only ME with getLumiFlag() == true ?
     if (mode == DQMStoreStats::considerOnlyLumiProductME && !(it->getLumiFlag()))
       continue;
 
     // figure out subsystem/subfolder names
     const std::string& path = it->getPathname();
 
     subfolderStringBegin = 0;
     Folder* curr = &dbeFolder;
     while (true) {
       subfolderStringEnd = path.find('/', subfolderStringBegin);
       if (std::string::npos == subfolderStringEnd) {
         curr = curr->cd(path.substr(subfolderStringBegin, path.size() - subfolderStringBegin));
         break;
       }
       curr = curr->cd(path.substr(subfolderStringBegin, subfolderStringEnd - subfolderStringBegin));
       subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
     }
 
     // protection against ghost ME with empty paths
     if (path.empty())
       continue;
 
     subsysStringEnd = path.find('/', 0);
     if (std::string::npos == subsysStringEnd)
       subsysStringEnd = path.size();  // no subfolder
 
     // new subsystem?
     if (path.substr(0, subsysStringEnd) != subsystemname) {
       DQMStoreStatsSubsystem aSubsystem;
       subsystemname = path.substr(0, subsysStringEnd);
       aSubsystem.subsystemName_ = subsystemname;
       dqmStoreStatsTopLevel.push_back(aSubsystem);
       subfoldername = "";
     }
 
     // get subfolder name (if there is one..)
     if (path.size() == subsysStringEnd) {
       // no subfolders in subsystem, make dummy
       DQMStoreStatsSubfolder aSubfolder;
       aSubfolder.subfolderName_ = subsystemname;  // <-- for tagging this case
       dqmStoreStatsTopLevel.back().push_back(aSubfolder);
     }
 
     else {
       // there is a subfolder, get its name
       subfolderStringEnd = path.find('/', subsysStringEnd + 1);
       if (std::string::npos == subfolderStringEnd)
         subfolderStringEnd = path.size();
 
       // new subfolder?
       if (path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1) != subfoldername) {
         subfoldername = path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1);
         DQMStoreStatsSubfolder aSubfolder;
         aSubfolder.subfolderName_ = subfoldername;
         dqmStoreStatsTopLevel.back().push_back(aSubfolder);
       }
     }
 
     // shortcut
     DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
 
     switch (it->kind()) {
         // one-dim ME
       case MonitorElement::Kind::TH1F:
         currentSubfolder.AddBinsF(it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0),
                      it->getNbinsX() * sizeof(float));
         break;
       case MonitorElement::Kind::TH1S:
         currentSubfolder.AddBinsS(it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0),
                      it->getNbinsX() * sizeof(short));
         break;
       case MonitorElement::Kind::TH1D:
         currentSubfolder.AddBinsD(it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0),
                      it->getNbinsX() * sizeof(double));
         break;
       case MonitorElement::Kind::TPROFILE:
         currentSubfolder.AddBinsD(it->getNbinsX(),
                                   getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0));
         curr->update(it->getNbinsX(),
                      getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0),
                      it->getNbinsX() * sizeof(double));
         break;
 
         // two-dim ME
       case MonitorElement::Kind::TH2F:
         currentSubfolder.AddBinsF(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(float));
         break;
       case MonitorElement::Kind::TH2S:
         currentSubfolder.AddBinsS(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(short));
         break;
       case MonitorElement::Kind::TH2D:
         currentSubfolder.AddBinsD(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(double));
         break;
       case MonitorElement::Kind::TPROFILE2D:
         currentSubfolder.AddBinsD(
             it->getNbinsX() * it->getNbinsY(),
             getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
         curr->update(it->getNbinsX() * it->getNbinsY(),
                      getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
                      it->getNbinsX() * it->getNbinsY() * sizeof(double));
         break;
 
         // three-dim ME
       case MonitorElement::Kind::TH3F:
         currentSubfolder.AddBinsF(
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
             getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2));
         curr->update(
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
             getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2),
             it->getNbinsX() * it->getNbinsY() * it->getNbinsZ() * sizeof(float));
         break;
 
       default: {}
         // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
         // which we don't care much about. Alternatively:
 
         //   std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: "
         //               << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
         //               << "  in path: \"" << path << "\" not considered." << std::endl;
     }
   }
 
   if (mode == DQMStoreStats::considerAllME)
     calcIgProfDump(dbeFolder);
 
   // OUTPUT
 
   std::cout << endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << endl;
 
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Configuration:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << " > running ";
   if (runonendrun_)
     std::cout << "on run end." << std::endl;
   if (runonendlumi_)
     std::cout << "on lumi end." << std::endl;
   if (runonendjob_)
     std::cout << "on job end." << std::endl;
   if (runineventloop_)
     std::cout << "in event loop." << std::endl;
   std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
   std::cout << std::endl;
 
   // dump folder structure
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Top level folder tree:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
     std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
 
     for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
       std::cout << "  |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
     }
   }
 
   // dump mem/bin table
 
   unsigned int overallNHistograms = 0, overallNBins = 0, overallNEmptyBins = 0, overallNBytes = 0;
 
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Detailed ressource usage information ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "subsystem/folder                          histograms       bins        Empty bins     Empty/Total      "
                "bins per       MB         kB per"
             << std::endl;
   std::cout << "                                           (total)        (total)        (total)                      "
                "histogram     (total)    histogram  "
             << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
     std::cout << it0->subsystemName_ << std::endl;
 
     unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
 
     for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
       // fixed-size working copy
       std::string thisSubfolderName(it1->subfolderName_);
       if (thisSubfolderName.size() > 30) {
         thisSubfolderName.resize(30);
         thisSubfolderName.replace(thisSubfolderName.size() - 3, 3, 3, '.');
       }
 
       std::cout << " -> " << std::setw(30) << std::left << thisSubfolderName;
       std::cout << std::setw(14) << std::right << it1->totalHistos_;
       std::cout << std::setw(14) << std::right << it1->totalBins_;
       std::cout << std::setw(14) << std::right << it1->totalEmptyBins_;
       std::cout << std::setw(14) << std::right << std::setprecision(3)
                 << (float)it1->totalEmptyBins_ / (float)it1->totalBins_;
 
       // bins/histogram, need to catch nan if histos=0
       if (it1->totalHistos_) {
         std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalBins_ / float(it1->totalHistos_);
       } else
         std::cout << std::setw(14) << std::right << "-";
 
       std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalMemory_ / 1024. / 1024.;
 
       // mem/histogram, need to catch nan if histos=0
       if (it1->totalHistos_) {
         std::cout << std::setw(14) << std::right << std::setprecision(3)
                   << it1->totalMemory_ / 1024. / it1->totalHistos_;
       } else
         std::cout << std::setw(14) << std::right << "-";
 
       std::cout << std::endl;
 
       // collect totals
       nHistograms += it1->totalHistos_;
       nBins += it1->totalBins_;
       nEmptyBins += it1->totalEmptyBins_;
       nBytes += it1->totalMemory_;
     }
 
     overallNHistograms += nHistograms;
     overallNBins += nBins;
     overallNEmptyBins += nEmptyBins;
     overallNBytes += nBytes;
 
     // display totals
     std::cout << "    " << std::setw(30) << std::left << "SUBSYSTEM TOTAL";
     std::cout << std::setw(14) << std::right << nHistograms;
     std::cout << std::setw(14) << std::right << nBins;
     std::cout << std::setw(14) << std::right << nEmptyBins;
     std::cout << std::setw(14) << std::right << (float)nEmptyBins / (float)nBins;
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBins / float(nHistograms);
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / 1000.;
     std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / nHistograms;
     std::cout << std::endl;
 
     std::cout << ".........................................................................................."
               << std::endl;
   }
 
   // dump total
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Grand total ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME:" << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
   std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether"
             << std::endl;
   std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB"
             << std::endl;
 
   std::cout << endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << "[DQMStoreStats::calcstats] -- End of output ";
   if (mode == DQMStoreStats::considerAllME)
     std::cout << "FOR ALL ME." << std::endl;
   else if (mode == DQMStoreStats::considerOnlyLumiProductME)
     std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
   std::cout << "==========================================================================================="
             << std::endl;
   std::cout << endl;
 
   // Put together a simplified version of the complete dump that is
   // sent to std::cout. Just dump the very basic information,
   // i.e. summary for each folder, both for run and LS products.
   if (dumpToFWJR_) {
     edm::Service<edm::JobReport> jr;
     // Do not even try if the FWJR service is not available.
     if (!jr.isAvailable())
       return 0;
     // Prepare appropriate map to store FWJR output.
     std::map<std::string, std::string> jrInfo;
     unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
 
     jrInfo["Source"] = "DQMServices/Components";
     jrInfo["FileClass"] = "DQMStoreStats";
     if (runonendrun_)
       jrInfo["DumpType"] = "EndRun";
     if (runonendlumi_)
       jrInfo["DumpType"] = "EndLumi";
     if (runonendjob_)
       jrInfo["DumpType"] = "EndJob";
     if (runineventloop_)
       jrInfo["DumpType"] = "EventLoop";
     if (mode == DQMStoreStats::considerAllME)
       jrInfo["Type"] = "RunProduct";
     else if (mode == DQMStoreStats::considerOnlyLumiProductME)
       jrInfo["Type"] = "LumiProduct";
 
     jrInfo["pathNameMatch"] = pathnamematch_;
 
     for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
       unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
       for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
         // collect totals
         nHistograms += it1->totalHistos_;
         nBins += it1->totalBins_;
         nEmptyBins += it1->totalEmptyBins_;
         nBytes += it1->totalMemory_;
       }
       overallNHistograms += nHistograms;
       overallNBins += nBins;
       overallNBytes += nBytes;
       std::stringstream iss("");
       iss << nHistograms;
       jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
       iss.str("");
       iss << nBins;
       jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
       iss.str("");
       iss << nEmptyBins;
       jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
       iss.str("");
       iss << ((float)nEmptyBins / (float)nBins);
       jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
       iss.str("");
       iss << ((float)nBins / (float)nHistograms);
       jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
       iss.str("");
       iss << nBytes / 1024. / 1024.;
       jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
       iss.str("");
       iss << nBytes / 1024. / nHistograms;
       jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
     }
     jr->reportAnalysisFile("DQMStatsReport", jrInfo);
   }
 
   return 0;
 }

void DQMStoreStats::dumpMemoryProfile ( )

private

Definition at line 558 of file DQMStoreStats.cc.

References gather_cfg::cout, dumpMemHistory_, isOpenProcFileSuccessful_, memoryHistoryVector_, download_sqlite_cfg::outputFile, and startingTime_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

                                       {
   std::cout << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
   std::cout << "Memory profile:" << std::endl;
   std::cout << "------------------------------------------------------------------------------------------"
             << std::endl;
 
   // determine virtual memory maximum
   std::pair<time_t, unsigned int> maxItem(0, 0);
   for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
     if (it->second > maxItem.second) {
       maxItem = *it;
     }
   }
 
   std::stringstream rootOutputFileName;
   rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
 
   // dump memory history to root file
   if (dumpMemHistory_ && isOpenProcFileSuccessful_) {
     TFile outputFile(rootOutputFileName.str().c_str(), "RECREATE");
 
     int aTime;
     float aMb;
 
     TTree memHistoryTree("dqmstorestats_memhistory", "memory history");
     memHistoryTree.Branch("seconds", &aTime, "seconds/I");
     memHistoryTree.Branch("megabytes", &aMb, "megabytes/F");
     for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
       aTime = it->first - startingTime_;
       aMb = it->second / 1000.;
       memHistoryTree.Fill();
     }
 
     outputFile.Write();
     outputFile.Close();
   }
 
   std::cout << "Approx. maximum total virtual memory size of job: ";
   if (isOpenProcFileSuccessful_ && !memoryHistoryVector_.empty()) {
     std::cout << maxItem.second / 1000. << " MB (reached " << maxItem.first - startingTime_
               << " sec. after constructor called)," << std::endl;
     std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size()
               << " samples)" << std::endl;
   } else {
     std::cout << "(could not be determined)" << std::endl;
   }
 
   std::cout << std::endl << std::endl;
 }

void DQMStoreStats::endJob ( void )

overrideprotectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 719 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendjob_.

                            {
   if (runonendjob_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

void DQMStoreStats::endLuminosityBlock	(	const edm::LuminosityBlock &	lumiSeg,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 697 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendlumi_.

                                                                                                 {
   if (runonendlumi_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

void DQMStoreStats::endRun	(	const edm::Run &	r,
		const edm::EventSetup &	c
	)

overrideprotected

Definition at line 708 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), and runonendrun_.

                                                                   {
   if (runonendrun_) {
     calcstats(DQMStoreStats::considerAllME);
     calcstats(DQMStoreStats::considerOnlyLumiProductME);
     dumpMemoryProfile();
   }
 }

void DQMStoreStats::print ( void )

private

Definition at line 613 of file DQMStoreStats.cc.

References gather_cfg::cout, maxbinsmesubsys_, maxbinssubsys_, nbinssubsys_, nmesubsys_, subfolder_, and subsystem_.

                           {
   // subsystem info printout
   std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
   std::cout << "  " << subfolder_ << ": ";
   std::cout << nmesubsys_ << " histograms with " << nbinssubsys_ << " bins. ";
   if (nmesubsys_ > 0)
     std::cout << nbinssubsys_ / nmesubsys_ << " bins/histogram ";
   std::cout << std::endl;
   std::cout << "  Largest histogram: " << maxbinsmesubsys_ << " with " << maxbinssubsys_ << " bins." << std::endl;
 }

std::pair< unsigned int, unsigned int > DQMStoreStats::readMemoryEntry ( ) const

private

read virtual memory size from /proc/<pid>/status file

Definition at line 627 of file DQMStoreStats.cc.

References recoMuon::in, isOpenProcFileSuccessful_, procFileName_, AlCaHLTBitMon_QueryRunRegistry::string, and ntuplemaker::time.

Referenced by analyze().

                                                                          {
   // see if initial test reading was successful
   if (isOpenProcFileSuccessful_) {
     std::ifstream procFile(procFileName_.str().c_str(), ios::in);
 
     std::string readBuffer("");
     unsigned int memSize = 0;
 
     // scan procfile
     while (!procFile.eof()) {
       procFile >> readBuffer;
       if (std::string("VmSize:") == readBuffer) {
         procFile >> memSize;
         break;
       }
     }
 
     procFile.close();
     return std::pair<time_t, unsigned int>(time(nullptr), memSize);
   }
 
   return std::pair<time_t, unsigned int>(0, 0);
 }

Member Data Documentation

DQMStore* DQMStoreStats::dbe_

private

Definition at line 332 of file DQMStoreStats.h.

Referenced by beginJob(), and calcstats().

bool DQMStoreStats::dumpMemHistory_

private

Definition at line 359 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

bool DQMStoreStats::dumpToFWJR_

private

Definition at line 360 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

bool DQMStoreStats::isOpenProcFileSuccessful_

private

Definition at line 352 of file DQMStoreStats.h.

Referenced by beginJob(), dumpMemoryProfile(), and readMemoryEntry().

int DQMStoreStats::maxbinsglobal_

private

Definition at line 341 of file DQMStoreStats.h.

Referenced by calcstats().

std::string DQMStoreStats::maxbinsmeglobal_

private

Definition at line 343 of file DQMStoreStats.h.

std::string DQMStoreStats::maxbinsmesubsys_

private

Definition at line 344 of file DQMStoreStats.h.

Referenced by print().

int DQMStoreStats::maxbinssubsys_

private

Definition at line 342 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

std::vector<std::pair<time_t, unsigned int> > DQMStoreStats::memoryHistoryVector_

private

Definition at line 350 of file DQMStoreStats.h.

Referenced by analyze(), and dumpMemoryProfile().

int DQMStoreStats::nbinsglobal_

private

Definition at line 337 of file DQMStoreStats.h.

Referenced by calcstats().

int DQMStoreStats::nbinssubsys_

private

Definition at line 338 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

int DQMStoreStats::nmeglobal_

private

Definition at line 339 of file DQMStoreStats.h.

int DQMStoreStats::nmesubsys_

private

Definition at line 340 of file DQMStoreStats.h.

Referenced by print().

edm::ParameterSet DQMStoreStats::parameters_

private

Definition at line 333 of file DQMStoreStats.h.

Referenced by Mixins._TypedParameterizable::clone(), Types.PSet::clone(), Types.EDAlias::clone(), Mixins._TypedParameterizable::copy(), and DQMStoreStats().

std::string DQMStoreStats::pathnamematch_

private

Definition at line 347 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

std::stringstream DQMStoreStats::procFileName_

private

Definition at line 353 of file DQMStoreStats.h.

Referenced by beginJob(), and readMemoryEntry().

bool DQMStoreStats::runineventloop_

private

Definition at line 358 of file DQMStoreStats.h.

Referenced by analyze(), calcstats(), and DQMStoreStats().

bool DQMStoreStats::runonendjob_

private

Definition at line 356 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endJob().

bool DQMStoreStats::runonendlumi_

private

Definition at line 357 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endLuminosityBlock().

bool DQMStoreStats::runonendrun_

private

Definition at line 355 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endRun().

time_t DQMStoreStats::startingTime_

private

Definition at line 351 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

int DQMStoreStats::statsdepth_

private

Definition at line 346 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

std::string DQMStoreStats::subfolder_

private

Definition at line 336 of file DQMStoreStats.h.

Referenced by print().

std::string DQMStoreStats::subsystem_

private

Definition at line 335 of file DQMStoreStats.h.

Referenced by print().

int DQMStoreStats::verbose_

private

Definition at line 348 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

Public Types

Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation