CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
DQMStoreStats.cc
Go to the documentation of this file.
1 /*
2  * \file DQMStoreStats.cc
3  * \author Andreas Meyer
4  * Last Update:
5  * $Date: 2009/12/15 08:59:50 $
6  * $Revision: 1.9 $
7  * $Author: dellaric $
8  *
9  * Description: Print out statistics of histograms in DQMStore
10 */
11 
14 
15 using namespace std;
16 using namespace edm;
17 
18 //==================================================================//
19 //================= Constructor and Destructor =====================//
20 //==================================================================//
22  : subsystem_ (""),
23  subfolder_ (""),
24  nbinsglobal_ (0),
25  nbinssubsys_ (0),
26  nmeglobal_ (0),
27  nmesubsys_ (0),
28  maxbinsglobal_ (0),
29  maxbinssubsys_ (0),
30  maxbinsmeglobal_ (""),
31  maxbinsmesubsys_ (""),
32  statsdepth_ (1),
33  pathnamematch_ ("*"),
34  verbose_ (0)
35 {
36  parameters_ = ps;
37  pathnamematch_ = ps.getUntrackedParameter<std::string>( "pathNameMatch", pathnamematch_ );
38  statsdepth_ = ps.getUntrackedParameter<int>( "statsDepth", statsdepth_ );
39  verbose_ = ps.getUntrackedParameter<int>( "verbose", verbose_ );
40  dumpMemHistory_ = ps.getUntrackedParameter<bool>( "dumpMemoryHistory", false );
41  runonendrun_ = ps.getUntrackedParameter<bool>( "runOnEndRun", true );
42  runonendjob_ = ps.getUntrackedParameter<bool>( "runOnEndJob", false );
43  runonendlumi_ = ps.getUntrackedParameter<bool>( "runOnEndLumi", false );
44  runineventloop_ = ps.getUntrackedParameter<bool>( "runInEventLoop", false );
45 
46  startingTime_ = time( 0 );
47 
48 }
49 
51 }
52 
53 
61 
63  nbinsglobal_ = 0;
64  nbinssubsys_ = 0;
65  maxbinsglobal_ = 0;
66  maxbinssubsys_ = 0;
67  std::string path = "";
68  std::string subsystemname = "";
69  std::string subfoldername = "";
70  size_t subsysStringEnd = 0, subfolderStringEnd = 0;
71 
72 
73  std::vector<MonitorElement*> melist;
75 
76  DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
77 
78  // loop all ME
79  typedef std::vector <MonitorElement*>::iterator meIt;
80  for(meIt it = melist.begin(); it != melist.end(); ++it) {
81 
82  // consider only ME with getLumiFlag() == true ?
84  !( (*it)->getLumiFlag() ) ) continue;
85 
86  // figure out subsystem/subfolder names
87  std::string path = (*it)->getPathname();
88 
89  // protection against ghost ME with empty paths
90  if( 0 == path.size() ) continue;
91 
92  subsysStringEnd = path.find( '/', 0 );
93  if( std::string::npos == subsysStringEnd ) subsysStringEnd = path.size(); // no subfolder
94 
95  // new subsystem?
96  if( path.substr( 0, subsysStringEnd ) != subsystemname ) {
97  DQMStoreStatsSubsystem aSubsystem;
98  subsystemname = path.substr( 0, subsysStringEnd );
99  aSubsystem.subsystemName_ = subsystemname;
100  dqmStoreStatsTopLevel.push_back( aSubsystem );
101  }
102 
103  // get subfolder name (if there is one..)
104  if( path.size() == subsysStringEnd ) {
105  // no subfolders in subsystem, make dummy
106  DQMStoreStatsSubfolder aSubfolder;
107  aSubfolder.subfolderName_ = subsystemname; // <-- for tagging this case
108  dqmStoreStatsTopLevel.back().push_back( aSubfolder );
109  }
110 
111  else {
112 
113  // there is a subfolder, get its name
114  subfolderStringEnd = path.find( '/', subsysStringEnd + 1 );
115  if( std::string::npos == subfolderStringEnd ) subfolderStringEnd = path.size();
116 
117  // new subfolder?
118  if( path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 ) != subfoldername ) {
119  subfoldername = path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 );
120  DQMStoreStatsSubfolder aSubfolder;
121  aSubfolder.subfolderName_ = subfoldername;
122  dqmStoreStatsTopLevel.back().push_back( aSubfolder );
123  }
124 
125  }
126 
127  // shortcut
128  DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
129 
130  switch( (*it)->kind() ) {
131 
132  // one-dim ME
133  case MonitorElement::DQM_KIND_TH1F: currentSubfolder.AddBinsF( (*it)->getNbinsX() ); break;
134  case MonitorElement::DQM_KIND_TH1S: currentSubfolder.AddBinsS( (*it)->getNbinsX() ); break;
135  case MonitorElement::DQM_KIND_TH1D: currentSubfolder.AddBinsD( (*it)->getNbinsX() ); break;
136  case MonitorElement::DQM_KIND_TPROFILE: currentSubfolder.AddBinsD( (*it)->getNbinsX() ); break;
137 
138  // two-dim ME
139  case MonitorElement::DQM_KIND_TH2F: currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
140  case MonitorElement::DQM_KIND_TH2S: currentSubfolder.AddBinsS( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
141  case MonitorElement::DQM_KIND_TH2D: currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
142  case MonitorElement::DQM_KIND_TPROFILE2D: currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY() ); break;
143 
144  // three-dim ME
146  currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ() ); break;
147 
148  default: {}
149  // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
150  // which we don't care much about. Alternatively:
151 
152  // std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: "
153  // << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
154  // << " in path: \"" << path << "\" not considered." << std::endl;
155  }
156 
157  }
158 
159 
160 
161  // OUTPUT
162 
163  std::cout << endl;
164  std::cout << "===========================================================================================" << std::endl;
165  std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
166  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
167  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
168  std::cout << "===========================================================================================" << std::endl;
169  std::cout << endl;
170 
171  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
172  std::cout << "Configuration:" << std::endl;
173  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
174  std::cout << " > running ";
175  if (runonendrun_) std::cout << "on run end." << std::endl;
176  if (runonendlumi_) std::cout << "on lumi end." << std::endl;
177  if (runonendjob_) std::cout << "on job end." << std::endl;
178  if (runineventloop_) std::cout << "in event loop." << std::endl;
179  std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
180  std::cout << std::endl;
181 
182  // dump folder structure
183  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
184  std::cout << "Top level folder tree:" << std::endl;
185  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
186  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
187  std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
188 
189  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
190  std::cout << " |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
191  }
192 
193  }
194 
195  // dump mem/bin table
196 
197  unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
198 
199  std::cout << std::endl;
200  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
201  std::cout << "Detailed ressource usage information ";
202  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
203  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
204  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
205  std::cout << "subsystem/folder histograms bins bins per MB kB per" << std::endl;
206  std::cout << " (total) (total) histogram (total) histogram " << std::endl;
207  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
208  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
209  std::cout << it0->subsystemName_ << std::endl;
210 
211  unsigned int nHistograms = 0, nBins = 0, nBytes = 0;
212 
213  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
214 
215  // fixed-size working copy
216  std::string thisSubfolderName( it1->subfolderName_ );
217  if( thisSubfolderName.size() > 30 ) {
218  thisSubfolderName.resize( 30 );
219  thisSubfolderName.replace( thisSubfolderName.size() - 3, 3, 3, '.' );
220  }
221 
222  std::cout << " -> " << std::setw( 30 ) << std::left << thisSubfolderName;
223  std::cout << std::setw( 7 ) << std::right << it1->totalHistos_;
224  std::cout << std::setw( 12 ) << std::right << it1->totalBins_;
225 
226  // bins/histogram, need to catch nan if histos=0
227  if( it1->totalHistos_ ) {
228  std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << it1->totalBins_ / float( it1->totalHistos_ );
229  }
230  else std::cout << std::setw( 12 ) << std::right << "-";
231 
232  std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / 1000.;
233 
234  // mem/histogram, need to catch nan if histos=0
235  if( it1->totalHistos_ ) {
236  std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / it1->totalHistos_;
237  }
238  else std::cout << std::setw( 12 ) << std::right << "-";
239 
240  std::cout << std::endl;
241 
242  // collect totals
243  nHistograms += it1->totalHistos_;
244  nBins += it1->totalBins_;
245  nBytes += it1->totalMemory_;
246 
247  }
248 
249 
250 
251 
252  overallNHistograms += nHistograms;
253  overallNBins += nBins;
254  overallNBytes += nBytes;
255 
256  // display totals
257  std::cout << " " << std::setw( 30 ) << std::left << "SUBSYSTEM TOTAL";
258  std::cout << std::setw( 7 ) << std::right << nHistograms;
259  std::cout << std::setw( 12 ) << std::right << nBins;
260  std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << nBins / float( nHistograms );
261  std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / 1000.;
262  std::cout << std::setw( 12 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / nHistograms;
263  std::cout << std::endl;
264 
265  std::cout << ".........................................................................................." << std::endl;
266 
267  }
268 
269 
270  // dump total
271  std::cout << std::endl;
272  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
273  std::cout << "Grand total ";
274  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME:" << std::endl;
275  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
276  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
277  std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
278  std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether" << std::endl;
279  std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB" << std::endl;
280 
281 
282 
283  std::cout << endl;
284  std::cout << "===========================================================================================" << std::endl;
285  std::cout << "[DQMStoreStats::calcstats] -- End of output ";
286  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME." << std::endl;
287  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
288  std::cout << "===========================================================================================" << std::endl;
289  std::cout << endl;
290 
291  return 0;
292 
293 }
294 
295 
296 
301 
302  std::cout << std::endl;
303  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
304  std::cout << "Memory profile:" << std::endl;
305  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
306 
307  // determine virtual memory maximum
308  std::pair<time_t, unsigned int> maxItem( 0, 0 );
309  for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
310  it < memoryHistoryVector_.end(); ++it ) {
311  if( it->second > maxItem.second ) {
312  maxItem = *it;
313  }
314  }
315 
316  std::stringstream rootOutputFileName;
317  rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
318 
319  // dump memory history to root file
321 
322  TFile outputFile( rootOutputFileName.str().c_str(), "RECREATE" );
323 
324  int aTime;
325  float aMb;
326 
327  TTree memHistoryTree( "dqmstorestats_memhistory", "memory history" );
328  memHistoryTree.Branch( "seconds", &aTime, "seconds/I" );
329  memHistoryTree.Branch( "megabytes", &aMb, "megabytes/F" );
330  for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
331  it < memoryHistoryVector_.end(); ++it ) {
332  aTime = it->first - startingTime_;
333  aMb = it->second / 1000.;
334  memHistoryTree.Fill();
335  }
336 
337  outputFile.Write();
338  outputFile.Close();
339 
340  }
341 
342  std::cout << "Approx. maximum total virtual memory size of job: ";
344  std::cout << maxItem.second / 1000.
345  << " MB (reached " << maxItem.first - startingTime_ << " sec. after constructor called)," << std::endl;
346  std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size() << " samples)" << std::endl;
347  } else {
348  std::cout << "(could not be determined)" << std::endl;
349  }
350 
351  std::cout << std::endl << std::endl;
352 
353 }
354 
355 
356 
361  // subsystem info printout
362  std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
363  std::cout << " " << subfolder_ << ": " ;
364  std::cout << nmesubsys_ << " histograms with "
365  << nbinssubsys_ << " bins. " ;
366  if (nmesubsys_ > 0) std::cout << nbinssubsys_/nmesubsys_ << " bins/histogram " ;
367  std::cout << std::endl;
368  std::cout << " Largest histogram: " << maxbinsmesubsys_ << " with " <<
369  maxbinssubsys_ << " bins." << std::endl;
370 }
371 
372 
373 
374 
378 std::pair<unsigned int, unsigned int> DQMStoreStats::readMemoryEntry( void ) const {
379 
380  // see if initial test reading was successful
382 
383  std::ifstream procFile( procFileName_.str().c_str(), ios::in );
384 
385  std::string readBuffer( "" );
386  unsigned int memSize = 0;
387 
388  // scan procfile
389  while( !procFile.eof() ) {
390  procFile >> readBuffer;
391  if( std::string( "VmSize:" ) == readBuffer ) {
392  procFile >> memSize;
393  break;
394  }
395  }
396 
397  procFile.close();
398  return std::pair<time_t, unsigned int>( time( 0 ), memSize );
399  }
400 
401  return std::pair<time_t, unsigned int>( 0, 0 );
402 
403 }
404 
405 
406 
407 //==================================================================//
408 //========================= beginJob ===============================//
409 //==================================================================//
411 
414 
415  // access the proc/ folder for memory information
416  procFileName_ << "/proc/" << getpid() << "/status";
417 
418  // open for a test
419  std::ifstream procFile( procFileName_.str().c_str(), ios::in );
420 
421  if( procFile.good() ) {
423  }
424  else {
425  std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
426  std::cerr << " Total memory profile will not be available." << std::endl;
428  }
429 
430  procFile.close();
431 
432 }
433 
434 //==================================================================//
435 //========================= beginRun ===============================//
436 //==================================================================//
437 void DQMStoreStats::beginRun(const edm::Run& r, const EventSetup& context) {
438 }
439 
440 
441 //==================================================================//
442 //==================== beginLuminosityBlock ========================//
443 //==================================================================//
445  const EventSetup& context) {
446 }
447 
448 
449 //==================================================================//
450 //==================== analyse (takes each event) ==================//
451 //==================================================================//
452 void DQMStoreStats::analyze(const Event& iEvent, const EventSetup& iSetup) {
453 
454  //now read virtual memory size from proc folder
455  memoryHistoryVector_.push_back( readMemoryEntry() );
456 
457  if (runineventloop_) {
461  }
462 
463 }
464 
465 
466 //==================================================================//
467 //========================= endLuminosityBlock =====================//
468 //==================================================================//
470  const EventSetup& context) {
471  if (runonendlumi_) {
475  }
476 
477 }
478 
479 //==================================================================//
480 //============================= endRun =============================//
481 //==================================================================//
482 void DQMStoreStats::endRun(const Run& r, const EventSetup& context) {
483 
484  if (runonendrun_) {
488  }
489 
490 }
491 
492 //==================================================================//
493 //============================= endJob =============================//
494 //==================================================================//
496 
497  if (runonendjob_) {
501  }
502 
503 }
std::string subfolderName_
Definition: DQMStoreStats.h:43
T getUntrackedParameter(std::string const &, T const &) const
DQMStore * dbe_
void beginRun(const edm::Run &r, const edm::EventSetup &c)
std::vector< std::pair< time_t, unsigned int > > memoryHistoryVector_
std::vector< MonitorElement * > getMatchingContents(const std::string &pattern, lat::Regexp::Syntax syntaxType=lat::Regexp::Wildcard) const
Definition: DQMStore.cc:1493
std::string subsystemName_
Definition: DQMStoreStats.h:59
void endLuminosityBlock(const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c)
void AddBinsD(unsigned int nBins)
Definition: DQMStoreStats.h:49
time_t startingTime_
void endRun(const edm::Run &r, const edm::EventSetup &c)
int path() const
Definition: HLTadd.h:3
std::string subsystem_
int calcstats(int)
int iEvent
Definition: GenABIO.cc:243
bool isOpenProcFileSuccessful_
std::pair< unsigned int, unsigned int > readMemoryEntry(void) const
std::string pathnamematch_
std::string maxbinsmesubsys_
DQMStoreStats(const edm::ParameterSet &)
std::string subfolder_
int mode
Definition: AMPTWrapper.h:139
edm::ParameterSet parameters_
void dumpMemoryProfile(void)
void beginLuminosityBlock(const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &context)
tuple cout
Definition: gather_cfg.py:41
std::stringstream procFileName_
void analyze(const edm::Event &e, const edm::EventSetup &c)
void AddBinsS(unsigned int nBins)
Definition: DQMStoreStats.h:48
void AddBinsF(unsigned int nBins)
Definition: DQMStoreStats.h:47
Definition: Run.h:31