CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Types | Public Member Functions | Protected Member Functions | Private Member Functions | Private Attributes
DQMStoreStats Class Reference

#include <DQMStoreStats.h>

Inheritance diagram for DQMStoreStats:
edm::EDAnalyzer edm::EDConsumerBase

Public Types

enum  statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 }
 
- Public Types inherited from edm::EDAnalyzer
typedef EDAnalyzer ModuleType
 

Public Member Functions

 DQMStoreStats (const edm::ParameterSet &)
 
 ~DQMStoreStats ()
 
- Public Member Functions inherited from edm::EDAnalyzer
void callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)
 
 EDAnalyzer ()
 
ModuleDescription const & moduleDescription () const
 
std::string workerType () const
 
virtual ~EDAnalyzer ()
 
- Public Member Functions inherited from edm::EDConsumerBase
 EDConsumerBase ()
 
ProductHolderIndexAndSkipBit indexFrom (EDGetToken, BranchType, TypeID const &) const
 
void itemsMayGet (BranchType, std::vector< ProductHolderIndexAndSkipBit > &) const
 
void itemsToGet (BranchType, std::vector< ProductHolderIndexAndSkipBit > &) const
 
std::vector
< ProductHolderIndexAndSkipBit >
const & 
itemsToGetFromEvent () const
 
void labelsForToken (EDGetToken iToken, Labels &oLabels) const
 
void modulesDependentUpon (const std::string &iProcessName, std::vector< const char * > &oModuleLabels) const
 
bool registeredToConsume (ProductHolderIndex, bool, BranchType) const
 
bool registeredToConsumeMany (TypeID const &, BranchType) const
 
void updateLookup (BranchType iBranchType, ProductHolderIndexHelper const &)
 
virtual ~EDConsumerBase ()
 

Protected Member Functions

void analyze (const edm::Event &e, const edm::EventSetup &c)
 
void beginJob ()
 
void beginLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &context)
 
void beginRun (const edm::Run &r, const edm::EventSetup &c)
 
void endJob ()
 
void endLuminosityBlock (const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c)
 
void endRun (const edm::Run &r, const edm::EventSetup &c)
 
- Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType > consumes (edm::InputTag const &tag)
 
EDGetToken consumes (const TypeToGet &id, edm::InputTag const &tag)
 
template<BranchType B>
EDGetToken consumes (TypeToGet const &id, edm::InputTag const &tag)
 
ConsumesCollector consumesCollector ()
 Use a ConsumesCollector to gather consumes information from helper functions. More...
 
template<typename ProductType , BranchType B = InEvent>
void consumesMany ()
 
void consumesMany (const TypeToGet &id)
 
template<BranchType B>
void consumesMany (const TypeToGet &id)
 
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType > mayConsume (edm::InputTag const &tag)
 
EDGetToken mayConsume (const TypeToGet &id, edm::InputTag const &tag)
 
template<BranchType B>
EDGetToken mayConsume (const TypeToGet &id, edm::InputTag const &tag)
 

Private Member Functions

void calcIgProfDump (Folder &)
 
int calcstats (int)
 
void dumpMemoryProfile (void)
 
void print ()
 
std::pair< unsigned int,
unsigned int > 
readMemoryEntry (void) const
 

Private Attributes

DQMStoredbe_
 
bool dumpMemHistory_
 
bool dumpToFWJR_
 
bool isOpenProcFileSuccessful_
 
int maxbinsglobal_
 
std::string maxbinsmeglobal_
 
std::string maxbinsmesubsys_
 
int maxbinssubsys_
 
std::vector< std::pair< time_t,
unsigned int > > 
memoryHistoryVector_
 
int nbinsglobal_
 
int nbinssubsys_
 
int nmeglobal_
 
int nmesubsys_
 
edm::ParameterSet parameters_
 
std::string pathnamematch_
 
std::stringstream procFileName_
 
bool runineventloop_
 
bool runonendjob_
 
bool runonendlumi_
 
bool runonendrun_
 
time_t startingTime_
 
int statsdepth_
 
std::string subfolder_
 
std::string subsystem_
 
int verbose_
 

Additional Inherited Members

- Static Public Member Functions inherited from edm::EDAnalyzer
static const std::string & baseType ()
 
static void fillDescriptions (ConfigurationDescriptions &descriptions)
 
static void prevalidate (ConfigurationDescriptions &)
 

Detailed Description

DQMStoreStats itself

Definition at line 292 of file DQMStoreStats.h.

Member Enumeration Documentation

Enumerator
considerAllME 
considerOnlyLumiProductME 

Definition at line 297 of file DQMStoreStats.h.

Constructor & Destructor Documentation

DQMStoreStats::DQMStoreStats ( const edm::ParameterSet ps)

Definition at line 49 of file DQMStoreStats.cc.

References dumpMemHistory_, dumpToFWJR_, edm::false, edm::ParameterSet::getUntrackedParameter(), parameters_, pathnamematch_, runineventloop_, runonendjob_, runonendlumi_, runonendrun_, startingTime_, statsdepth_, AlCaHLTBitMon_QueryRunRegistry::string, cond::rpcobgas::time, funct::true, and verbose_.

50  : subsystem_ (""),
51  subfolder_ (""),
52  nbinsglobal_ (0),
53  nbinssubsys_ (0),
54  nmeglobal_ (0),
55  nmesubsys_ (0),
56  maxbinsglobal_ (0),
57  maxbinssubsys_ (0),
58  maxbinsmeglobal_ (""),
59  maxbinsmesubsys_ (""),
60  statsdepth_ (1),
61  pathnamematch_ ("*"),
62  verbose_ (0)
63 {
64  parameters_ = ps;
66  statsdepth_ = ps.getUntrackedParameter<int>( "statsDepth", statsdepth_ );
67  verbose_ = ps.getUntrackedParameter<int>( "verbose", verbose_ );
68  dumpMemHistory_ = ps.getUntrackedParameter<bool>( "dumpMemoryHistory", false );
69  runonendrun_ = ps.getUntrackedParameter<bool>( "runOnEndRun", true );
70  runonendjob_ = ps.getUntrackedParameter<bool>( "runOnEndJob", false );
71  runonendlumi_ = ps.getUntrackedParameter<bool>( "runOnEndLumi", false );
72  runineventloop_ = ps.getUntrackedParameter<bool>( "runInEventLoop", false );
73  dumpToFWJR_ = ps.getUntrackedParameter<bool>( "dumpToFWJR", false );
74 
75  startingTime_ = time( 0 );
76 }
T getUntrackedParameter(std::string const &, T const &) const
time_t startingTime_
std::string subsystem_
std::string maxbinsmeglobal_
std::string pathnamematch_
std::string maxbinsmesubsys_
std::string subfolder_
edm::ParameterSet parameters_
volatile std::atomic< bool > shutdown_flag false
DQMStoreStats::~DQMStoreStats ( )

Definition at line 78 of file DQMStoreStats.cc.

78  {
79 }

Member Function Documentation

void DQMStoreStats::analyze ( const edm::Event e,
const edm::EventSetup c 
)
protectedvirtual

Implements edm::EDAnalyzer.

Definition at line 669 of file DQMStoreStats.cc.

References calcstats(), considerAllME, considerOnlyLumiProductME, dumpMemoryProfile(), memoryHistoryVector_, readMemoryEntry(), and runineventloop_.

669  {
670 
671  //now read virtual memory size from proc folder
672  memoryHistoryVector_.push_back( readMemoryEntry() );
673 
674  if (runineventloop_) {
678  }
679 
680 }
std::vector< std::pair< time_t, unsigned int > > memoryHistoryVector_
int calcstats(int)
std::pair< unsigned int, unsigned int > readMemoryEntry(void) const
void dumpMemoryProfile(void)
void DQMStoreStats::beginJob ( void  )
protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 627 of file DQMStoreStats.cc.

References dtNoiseDBValidation_cfg::cerr, dbe_, recoMuon::in, isOpenProcFileSuccessful_, cppFunctionSkipper::operator, and procFileName_.

627  {
628 
631 
632  // access the proc/ folder for memory information
633  procFileName_ << "/proc/" << getpid() << "/status";
634 
635  // open for a test
636  std::ifstream procFile( procFileName_.str().c_str(), ios::in );
637 
638  if( procFile.good() ) {
640  }
641  else {
642  std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
643  std::cerr << " Total memory profile will not be available." << std::endl;
645  }
646 
647  procFile.close();
648 
649 }
DQMStore * dbe_
bool isOpenProcFileSuccessful_
std::stringstream procFileName_
void DQMStoreStats::beginLuminosityBlock ( const edm::LuminosityBlock lumiSeg,
const edm::EventSetup context 
)
protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 661 of file DQMStoreStats.cc.

662  {
663 }
void DQMStoreStats::beginRun ( const edm::Run r,
const edm::EventSetup c 
)
protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 654 of file DQMStoreStats.cc.

654  {
655 }
void DQMStoreStats::calcIgProfDump ( Folder root)
private

Definition at line 81 of file DQMStoreStats.cc.

References Folder::children(), Folder::CreateIterator(), VIterator< Item >::CurrentItem(), Folder::files(), VIterator< Item >::First(), cuy::ii, VIterator< Item >::IsDone(), Folder::mainrows(), Folder::mainrows_cumulative(), VIterator< Item >::Next(), Folder::parents(), edmStreamStallGrapher::stream, AlCaHLTBitMon_QueryRunRegistry::string, Folder::summary(), and Folder::symbols().

Referenced by calcstats().

82 {
83  std::ofstream stream("dqm-bin-stats.sql");
84  stream << ""
85 " PRAGMA journal_mode=OFF;"
86 " PRAGMA count_changes=OFF;"
87 " DROP TABLE IF EXISTS files;"
88 " DROP TABLE IF EXISTS symbols;"
89 " DROP TABLE IF EXISTS mainrows;"
90 " DROP TABLE IF EXISTS children;"
91 " DROP TABLE IF EXISTS parents;"
92 " DROP TABLE IF EXISTS summary;"
93 " CREATE TABLE children ("
94 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
95 " parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
96 " from_parent_count INTEGER,"
97 " from_parent_calls INTEGER,"
98 " from_parent_paths INTEGER,"
99 " pct REAL"
100 " );"
101 " CREATE TABLE files ("
102 " id,"
103 " name TEXT"
104 " );"
105 " CREATE TABLE mainrows ("
106 " id INTEGER PRIMARY KEY,"
107 " symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
108 " self_count INTEGER,"
109 " cumulative_count INTEGER,"
110 " kids INTEGER,"
111 " self_calls INTEGER,"
112 " total_calls INTEGER,"
113 " self_paths INTEGER,"
114 " total_paths INTEGER,"
115 " pct REAL"
116 " );"
117 " CREATE TABLE parents ("
118 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
119 " child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
120 " to_child_count INTEGER,"
121 " to_child_calls INTEGER,"
122 " to_child_paths INTEGER,"
123 " pct REAL"
124 " );"
125 " CREATE TABLE summary ("
126 " counter TEXT,"
127 " total_count INTEGER,"
128 " total_freq INTEGER,"
129 " tick_period REAL"
130 " );"
131 " CREATE TABLE symbols ("
132 " id,"
133 " name TEXT,"
134 " filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
135 " );"
136 " CREATE UNIQUE INDEX fileIndex ON files (id);"
137 " CREATE INDEX selfCountIndex ON mainrows(self_count);"
138 " CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
139 " CREATE INDEX totalCountIndex ON mainrows(cumulative_count);" << std::endl;
140 
141  std::string sql_statement("");
142 
143  root.files(sql_statement);
144  root.symbols(sql_statement);
145  root.mainrows_cumulative(sql_statement);
146  root.summary(sql_statement);
147  VIterator<Folder *> subsystems = root.CreateIterator() ;
148  size_t ii=0;
149  for(subsystems.First() ; !subsystems.IsDone() ; subsystems.Next(), ++ii)
150  {
151  subsystems.CurrentItem()->mainrows(sql_statement);
152  subsystems.CurrentItem()->parents(sql_statement);
153  subsystems.CurrentItem()->children(sql_statement);
154  }
155  stream << sql_statement << std::endl;
156 }
virtual void First()
Definition: DQMStoreStats.h:89
virtual bool IsDone() const
Definition: DQMStoreStats.h:94
void mainrows_cumulative(std::string &sql_statement)
VIterator< Folder * > CreateIterator()
int ii
Definition: cuy.py:588
void summary(std::string &sql_statement)
void mainrows(std::string &sql_statement)
void files(std::string &sql_statement)
virtual void Next()
Definition: DQMStoreStats.h:90
virtual Item CurrentItem() const
void parents(std::string &sql_statement)
void children(std::string &sql_statement)
void symbols(std::string &sql_statement)
int DQMStoreStats::calcstats ( int  mode = DQMStoreStats::considerAllME)
private

do the stats here and produce output;

mode is coded in DQMStoreStats::statMode enum (select subsets of ME, e.g. those with getLumiFlag() == true)

Definition at line 164 of file DQMStoreStats.cc.

References DQMStoreStatsSubfolder::AddBinsD(), DQMStoreStatsSubfolder::AddBinsF(), DQMStoreStatsSubfolder::AddBinsS(), calcIgProfDump(), Folder::cd(), considerAllME, considerOnlyLumiProductME, gather_cfg::cout, dbe_, MonitorElement::DQM_KIND_TH1D, MonitorElement::DQM_KIND_TH1F, MonitorElement::DQM_KIND_TH1S, MonitorElement::DQM_KIND_TH2D, MonitorElement::DQM_KIND_TH2F, MonitorElement::DQM_KIND_TH2S, MonitorElement::DQM_KIND_TH3F, MonitorElement::DQM_KIND_TPROFILE, MonitorElement::DQM_KIND_TPROFILE2D, dumpToFWJR_, getEmptyMetric(), DQMStore::getMatchingContents(), edm::Service< T >::isAvailable(), maxbinsglobal_, maxbinssubsys_, alignBH_cfg::mode, nbinsglobal_, nbinssubsys_, cmsHarvester::path, pathnamematch_, edm::JobReport::reportAnalysisFile(), runineventloop_, runonendjob_, runonendlumi_, runonendrun_, AlCaHLTBitMon_QueryRunRegistry::string, DQMStoreStatsSubfolder::subfolderName_, DQMStoreStatsSubsystem::subsystemName_, and Folder::update().

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

164  {
165 
167  nbinsglobal_ = 0;
168  nbinssubsys_ = 0;
169  maxbinsglobal_ = 0;
170  maxbinssubsys_ = 0;
171  std::string path = "";
172  std::string subsystemname = "";
173  std::string subfoldername = "";
174  size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;
175 
176 
177  std::vector<MonitorElement*> melist;
179 
180  Folder dbeFolder("root");
181  DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
182 
183  // loop all ME
184  typedef std::vector <MonitorElement*>::iterator meIt;
185  for(meIt it = melist.begin(); it != melist.end(); ++it) {
186 
187  // consider only ME with getLumiFlag() == true ?
189  !( (*it)->getLumiFlag() ) ) continue;
190 
191  // figure out subsystem/subfolder names
192  std::string path = (*it)->getPathname();
193 
194  subfolderStringBegin = 0;
195  Folder * curr = &dbeFolder;
196  while(1)
197  {
198  subfolderStringEnd = path.find( '/', subfolderStringBegin );
199  if( std::string::npos == subfolderStringEnd )
200  {
201  curr = curr->cd(path.substr( subfolderStringBegin, path.size()-subfolderStringBegin ));
202  break;
203  }
204  curr = curr->cd(path.substr( subfolderStringBegin, subfolderStringEnd-subfolderStringBegin ));
205  subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
206  }
207 
208  // protection against ghost ME with empty paths
209  if( 0 == path.size() ) continue;
210 
211  subsysStringEnd = path.find( '/', 0 );
212  if( std::string::npos == subsysStringEnd ) subsysStringEnd = path.size(); // no subfolder
213 
214  // new subsystem?
215  if( path.substr( 0, subsysStringEnd ) != subsystemname ) {
216  DQMStoreStatsSubsystem aSubsystem;
217  subsystemname = path.substr( 0, subsysStringEnd );
218  aSubsystem.subsystemName_ = subsystemname;
219  dqmStoreStatsTopLevel.push_back( aSubsystem );
220  subfoldername = "";
221  }
222 
223  // get subfolder name (if there is one..)
224  if( path.size() == subsysStringEnd ) {
225  // no subfolders in subsystem, make dummy
226  DQMStoreStatsSubfolder aSubfolder;
227  aSubfolder.subfolderName_ = subsystemname; // <-- for tagging this case
228  dqmStoreStatsTopLevel.back().push_back( aSubfolder );
229  }
230 
231  else {
232 
233  // there is a subfolder, get its name
234  subfolderStringEnd = path.find( '/', subsysStringEnd + 1 );
235  if( std::string::npos == subfolderStringEnd ) subfolderStringEnd = path.size();
236 
237  // new subfolder?
238  if( path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 ) != subfoldername ) {
239  subfoldername = path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 );
240  DQMStoreStatsSubfolder aSubfolder;
241  aSubfolder.subfolderName_ = subfoldername;
242  dqmStoreStatsTopLevel.back().push_back( aSubfolder );
243  }
244 
245  }
246 
247  // shortcut
248  DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
249 
250  switch( (*it)->kind() ) {
251 
252  // one-dim ME
254  currentSubfolder.AddBinsF( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1F()->GetArray(), (*it)->getTH1F()->fN, 0, 0) );
255  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1F()->GetArray(), (*it)->getTH1F()->fN, 0, 0), (*it)->getNbinsX()*sizeof( float ) );
256  break;
258  currentSubfolder.AddBinsS( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1S()->GetArray(), (*it)->getTH1S()->fN, 0, 0) );
259  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1S()->GetArray(), (*it)->getTH1S()->fN, 0, 0), (*it)->getNbinsX()*sizeof( short ) );
260  break;
262  currentSubfolder.AddBinsD( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1D()->GetArray(), (*it)->getTH1D()->fN, 0, 0) );
263  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1D()->GetArray(), (*it)->getTH1D()->fN, 0, 0), (*it)->getNbinsX()*sizeof( double ) );
264  break;
266  currentSubfolder.AddBinsD( (*it)->getNbinsX(), getEmptyMetric((*it)->getTProfile()->GetArray(), (*it)->getTProfile()->fN, 0, 0) );
267  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTProfile()->GetArray(), (*it)->getTProfile()->fN, 0, 0), (*it)->getNbinsX()*sizeof( double ) );
268  break;
269 
270  // two-dim ME
272  currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
273  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(float) );
274  break;
276  currentSubfolder.AddBinsS( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2S()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
277  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2S()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(short) );
278  break;
280  currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
281  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(double) );
282  break;
284  currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTProfile2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
285  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTProfile2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(double) );
286  break;
287 
288  // three-dim ME
290  currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ(), getEmptyMetric( (*it)->getTH3F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, (*it)->getNbinsZ()+2 ) );
291  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ(),
292  getEmptyMetric( (*it)->getTH3F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, (*it)->getNbinsZ()+2 ),
293  (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ()*sizeof(float));
294  break;
295 
296  default: {}
297  // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
298  // which we don't care much about. Alternatively:
299 
300  // std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: "
301  // << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
302  // << " in path: \"" << path << "\" not considered." << std::endl;
303  }
304  }
305 
307  calcIgProfDump(dbeFolder);
308 
309  // OUTPUT
310 
311  std::cout << endl;
312  std::cout << "===========================================================================================" << std::endl;
313  std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
314  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
315  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
316  std::cout << "===========================================================================================" << std::endl;
317  std::cout << endl;
318 
319  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
320  std::cout << "Configuration:" << std::endl;
321  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
322  std::cout << " > running ";
323  if (runonendrun_) std::cout << "on run end." << std::endl;
324  if (runonendlumi_) std::cout << "on lumi end." << std::endl;
325  if (runonendjob_) std::cout << "on job end." << std::endl;
326  if (runineventloop_) std::cout << "in event loop." << std::endl;
327  std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
328  std::cout << std::endl;
329 
330  // dump folder structure
331  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
332  std::cout << "Top level folder tree:" << std::endl;
333  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
334  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
335  std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
336 
337  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
338  std::cout << " |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
339  }
340 
341  }
342 
343  // dump mem/bin table
344 
345  unsigned int overallNHistograms = 0, overallNBins = 0, overallNEmptyBins = 0, overallNBytes = 0;
346 
347  std::cout << std::endl;
348  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
349  std::cout << "Detailed ressource usage information ";
350  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
351  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
352  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
353  std::cout << "subsystem/folder histograms bins Empty bins Empty/Total bins per MB kB per" << std::endl;
354  std::cout << " (total) (total) (total) histogram (total) histogram " << std::endl;
355  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
356  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
357  std::cout << it0->subsystemName_ << std::endl;
358 
359  unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
360 
361  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
362 
363  // fixed-size working copy
364  std::string thisSubfolderName( it1->subfolderName_ );
365  if( thisSubfolderName.size() > 30 ) {
366  thisSubfolderName.resize( 30 );
367  thisSubfolderName.replace( thisSubfolderName.size() - 3, 3, 3, '.' );
368  }
369 
370  std::cout << " -> " << std::setw( 30 ) << std::left << thisSubfolderName;
371  std::cout << std::setw( 14 ) << std::right << it1->totalHistos_;
372  std::cout << std::setw( 14 ) << std::right << it1->totalBins_;
373  std::cout << std::setw( 14 ) << std::right << it1->totalEmptyBins_;
374  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << (float)it1->totalEmptyBins_/(float)it1->totalBins_;
375 
376  // bins/histogram, need to catch nan if histos=0
377  if( it1->totalHistos_ ) {
378  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalBins_ / float( it1->totalHistos_ );
379  }
380  else std::cout << std::setw( 14 ) << std::right << "-";
381 
382  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / 1024.;
383 
384  // mem/histogram, need to catch nan if histos=0
385  if( it1->totalHistos_ ) {
386  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / it1->totalHistos_;
387  }
388  else std::cout << std::setw( 14 ) << std::right << "-";
389 
390  std::cout << std::endl;
391 
392  // collect totals
393  nHistograms += it1->totalHistos_;
394  nBins += it1->totalBins_;
395  nEmptyBins += it1->totalEmptyBins_;
396  nBytes += it1->totalMemory_;
397 
398  }
399 
400 
401 
402 
403  overallNHistograms += nHistograms;
404  overallNBins += nBins;
405  overallNEmptyBins += nEmptyBins;
406  overallNBytes += nBytes;
407 
408  // display totals
409  std::cout << " " << std::setw( 30 ) << std::left << "SUBSYSTEM TOTAL";
410  std::cout << std::setw( 14 ) << std::right << nHistograms;
411  std::cout << std::setw( 14 ) << std::right << nBins;
412  std::cout << std::setw( 14 ) << std::right << nEmptyBins;
413  std::cout << std::setw( 14 ) << std::right << (float)nEmptyBins/(float)nBins;
414  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBins / float( nHistograms );
415  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / 1000.;
416  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / nHistograms;
417  std::cout << std::endl;
418 
419  std::cout << ".........................................................................................." << std::endl;
420 
421  }
422 
423 
424  // dump total
425  std::cout << std::endl;
426  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
427  std::cout << "Grand total ";
428  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME:" << std::endl;
429  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
430  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
431  std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
432  std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether" << std::endl;
433  std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB" << std::endl;
434 
435 
436 
437  std::cout << endl;
438  std::cout << "===========================================================================================" << std::endl;
439  std::cout << "[DQMStoreStats::calcstats] -- End of output ";
440  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME." << std::endl;
441  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
442  std::cout << "===========================================================================================" << std::endl;
443  std::cout << endl;
444 
445  // Put together a simplified version of the complete dump that is
446  // sent to std::cout. Just dump the very basic information,
447  // i.e. summary for each folder, both for run and LS products.
448  if (dumpToFWJR_)
449  {
451  // Do not even try if the FWJR service is not available.
452  if (!jr.isAvailable())
453  return 0;
454  // Prepare appropriate map to store FWJR output.
455  std::map<std::string, std::string> jrInfo;
456  unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
457 
458  jrInfo["Source"] = "DQMServices/Components";
459  jrInfo["FileClass"] = "DQMStoreStats";
460  if (runonendrun_)
461  jrInfo["DumpType"] = "EndRun";
462  if (runonendlumi_)
463  jrInfo["DumpType"] = "EndLumi";
464  if (runonendjob_)
465  jrInfo["DumpType"] = "EndJob";
466  if (runineventloop_)
467  jrInfo["DumpType"] = "EventLoop";
469  jrInfo["Type"] = "RunProduct";
471  jrInfo["Type"] = "LumiProduct";
472 
473 
474  jrInfo["pathNameMatch"] = pathnamematch_;
475 
476  for (DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 )
477  {
478  unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
479  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
480  // collect totals
481  nHistograms += it1->totalHistos_;
482  nBins += it1->totalBins_;
483  nEmptyBins += it1->totalEmptyBins_;
484  nBytes += it1->totalMemory_;
485  }
486  overallNHistograms += nHistograms;
487  overallNBins += nBins;
488  overallNBytes += nBytes;
489  std::stringstream iss("");
490  iss << nHistograms;
491  jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
492  iss.str("");iss<<nBins;
493  jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
494  iss.str("");iss<<nEmptyBins;
495  jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
496  iss.str("");iss<< ((float)nEmptyBins/(float)nBins);
497  jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
498  iss.str("");iss<< ((float)nBins/(float)nHistograms);
499  jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
500  iss.str("");iss<<nBytes / 1024. / 1024.;
501  jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
502  iss.str("");iss<<nBytes / 1024. / nHistograms;
503  jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
504  }
505  jr->reportAnalysisFile("DQMStatsReport", jrInfo);
506  }
507 
508  return 0;
509 
510 }
std::string subfolderName_
Definition: DQMStoreStats.h:42
DQMStore * dbe_
void AddBinsS(unsigned int nBins, unsigned int nEmptyBins)
Definition: DQMStoreStats.h:48
void update(unsigned int bins, unsigned int empty, unsigned int memory)
Definition: vlib.h:186
void AddBinsD(unsigned int nBins, unsigned int nEmptyBins)
Definition: DQMStoreStats.h:49
std::vector< MonitorElement * > getMatchingContents(const std::string &pattern, lat::Regexp::Syntax syntaxType=lat::Regexp::Wildcard) const
Definition: DQMStore.cc:1944
std::string subsystemName_
Definition: DQMStoreStats.h:59
Definition: ME.h:11
int calcstats(int)
tuple path
else: Piece not in the list, fine.
std::string pathnamematch_
bool isAvailable() const
Definition: Service.h:46
static unsigned int getEmptyMetric(T *array, int lenx, int leny, int lenz)
Folder * cd(const std::string &name)
return(e1-e2)*(e1-e2)+dp *dp
void AddBinsF(unsigned int nBins, unsigned int nEmptyBins)
Definition: DQMStoreStats.h:47
if(dp >Float(M_PI)) dp-
void calcIgProfDump(Folder &)
tuple cout
Definition: gather_cfg.py:121
tuple size
Write out results.
void DQMStoreStats::dumpMemoryProfile ( void  )
private

Definition at line 517 of file DQMStoreStats.cc.

References gather_cfg::cout, dumpMemHistory_, isOpenProcFileSuccessful_, memoryHistoryVector_, download_sqlite_cfg::outputFile, and startingTime_.

Referenced by analyze(), endJob(), endLuminosityBlock(), and endRun().

517  {
518 
519  std::cout << std::endl;
520  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
521  std::cout << "Memory profile:" << std::endl;
522  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
523 
524  // determine virtual memory maximum
525  std::pair<time_t, unsigned int> maxItem( 0, 0 );
526  for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
527  it < memoryHistoryVector_.end(); ++it ) {
528  if( it->second > maxItem.second ) {
529  maxItem = *it;
530  }
531  }
532 
533  std::stringstream rootOutputFileName;
534  rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
535 
536  // dump memory history to root file
538 
539  TFile outputFile( rootOutputFileName.str().c_str(), "RECREATE" );
540 
541  int aTime;
542  float aMb;
543 
544  TTree memHistoryTree( "dqmstorestats_memhistory", "memory history" );
545  memHistoryTree.Branch( "seconds", &aTime, "seconds/I" );
546  memHistoryTree.Branch( "megabytes", &aMb, "megabytes/F" );
547  for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
548  it < memoryHistoryVector_.end(); ++it ) {
549  aTime = it->first - startingTime_;
550  aMb = it->second / 1000.;
551  memHistoryTree.Fill();
552  }
553 
554  outputFile.Write();
555  outputFile.Close();
556 
557  }
558 
559  std::cout << "Approx. maximum total virtual memory size of job: ";
561  std::cout << maxItem.second / 1000.
562  << " MB (reached " << maxItem.first - startingTime_ << " sec. after constructor called)," << std::endl;
563  std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size() << " samples)" << std::endl;
564  } else {
565  std::cout << "(could not be determined)" << std::endl;
566  }
567 
568  std::cout << std::endl << std::endl;
569 
570 }
std::vector< std::pair< time_t, unsigned int > > memoryHistoryVector_
time_t startingTime_
bool isOpenProcFileSuccessful_
tuple cout
Definition: gather_cfg.py:121
void DQMStoreStats::endJob ( void  )
protectedvirtual
void DQMStoreStats::endLuminosityBlock ( const edm::LuminosityBlock lumiSeg,
const edm::EventSetup c 
)
protectedvirtual
void DQMStoreStats::endRun ( const edm::Run r,
const edm::EventSetup c 
)
protectedvirtual
void DQMStoreStats::print ( void  )
private

Definition at line 577 of file DQMStoreStats.cc.

References gather_cfg::cout, maxbinsmesubsys_, maxbinssubsys_, nbinssubsys_, nmesubsys_, subfolder_, and subsystem_.

577  {
578  // subsystem info printout
579  std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
580  std::cout << " " << subfolder_ << ": " ;
581  std::cout << nmesubsys_ << " histograms with "
582  << nbinssubsys_ << " bins. " ;
583  if (nmesubsys_ > 0) std::cout << nbinssubsys_/nmesubsys_ << " bins/histogram " ;
584  std::cout << std::endl;
585  std::cout << " Largest histogram: " << maxbinsmesubsys_ << " with " <<
586  maxbinssubsys_ << " bins." << std::endl;
587 }
std::string subsystem_
std::string maxbinsmesubsys_
std::string subfolder_
tuple cout
Definition: gather_cfg.py:121
std::pair< unsigned int, unsigned int > DQMStoreStats::readMemoryEntry ( void  ) const
private

read virtual memory size from /proc/<pid>/status file

Definition at line 595 of file DQMStoreStats.cc.

References recoMuon::in, isOpenProcFileSuccessful_, procFileName_, AlCaHLTBitMon_QueryRunRegistry::string, and cond::rpcobgas::time.

Referenced by analyze().

595  {
596 
597  // see if initial test reading was successful
599 
600  std::ifstream procFile( procFileName_.str().c_str(), ios::in );
601 
602  std::string readBuffer( "" );
603  unsigned int memSize = 0;
604 
605  // scan procfile
606  while( !procFile.eof() ) {
607  procFile >> readBuffer;
608  if( std::string( "VmSize:" ) == readBuffer ) {
609  procFile >> memSize;
610  break;
611  }
612  }
613 
614  procFile.close();
615  return std::pair<time_t, unsigned int>( time( 0 ), memSize );
616  }
617 
618  return std::pair<time_t, unsigned int>( 0, 0 );
619 
620 }
bool isOpenProcFileSuccessful_
std::stringstream procFileName_

Member Data Documentation

DQMStore* DQMStoreStats::dbe_
private

Definition at line 331 of file DQMStoreStats.h.

Referenced by beginJob(), and calcstats().

bool DQMStoreStats::dumpMemHistory_
private

Definition at line 358 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

bool DQMStoreStats::dumpToFWJR_
private

Definition at line 359 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

bool DQMStoreStats::isOpenProcFileSuccessful_
private

Definition at line 351 of file DQMStoreStats.h.

Referenced by beginJob(), dumpMemoryProfile(), and readMemoryEntry().

int DQMStoreStats::maxbinsglobal_
private

Definition at line 340 of file DQMStoreStats.h.

Referenced by calcstats().

std::string DQMStoreStats::maxbinsmeglobal_
private

Definition at line 342 of file DQMStoreStats.h.

std::string DQMStoreStats::maxbinsmesubsys_
private

Definition at line 343 of file DQMStoreStats.h.

Referenced by print().

int DQMStoreStats::maxbinssubsys_
private

Definition at line 341 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

std::vector<std::pair<time_t, unsigned int> > DQMStoreStats::memoryHistoryVector_
private

Definition at line 349 of file DQMStoreStats.h.

Referenced by analyze(), and dumpMemoryProfile().

int DQMStoreStats::nbinsglobal_
private

Definition at line 336 of file DQMStoreStats.h.

Referenced by calcstats().

int DQMStoreStats::nbinssubsys_
private

Definition at line 337 of file DQMStoreStats.h.

Referenced by calcstats(), and print().

int DQMStoreStats::nmeglobal_
private

Definition at line 338 of file DQMStoreStats.h.

int DQMStoreStats::nmesubsys_
private

Definition at line 339 of file DQMStoreStats.h.

Referenced by print().

edm::ParameterSet DQMStoreStats::parameters_
private
std::string DQMStoreStats::pathnamematch_
private

Definition at line 346 of file DQMStoreStats.h.

Referenced by calcstats(), and DQMStoreStats().

std::stringstream DQMStoreStats::procFileName_
private

Definition at line 352 of file DQMStoreStats.h.

Referenced by beginJob(), and readMemoryEntry().

bool DQMStoreStats::runineventloop_
private

Definition at line 357 of file DQMStoreStats.h.

Referenced by analyze(), calcstats(), and DQMStoreStats().

bool DQMStoreStats::runonendjob_
private

Definition at line 355 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endJob().

bool DQMStoreStats::runonendlumi_
private

Definition at line 356 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endLuminosityBlock().

bool DQMStoreStats::runonendrun_
private

Definition at line 354 of file DQMStoreStats.h.

Referenced by calcstats(), DQMStoreStats(), and endRun().

time_t DQMStoreStats::startingTime_
private

Definition at line 350 of file DQMStoreStats.h.

Referenced by DQMStoreStats(), and dumpMemoryProfile().

int DQMStoreStats::statsdepth_
private

Definition at line 345 of file DQMStoreStats.h.

Referenced by DQMStoreStats().

std::string DQMStoreStats::subfolder_
private

Definition at line 335 of file DQMStoreStats.h.

Referenced by print().

std::string DQMStoreStats::subsystem_
private

Definition at line 334 of file DQMStoreStats.h.

Referenced by print().

int DQMStoreStats::verbose_
private

Definition at line 347 of file DQMStoreStats.h.

Referenced by DQMStoreStats().