CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
DQMStoreStats.cc
Go to the documentation of this file.
1 /*
2  * \file DQMStoreStats.cc
3  * \author Andreas Meyer
4  * Last Update:
5  *
6  * Description: Print out statistics of histograms in DQMStore
7 */
8 
9 #include "DQMServices/Components/src/DQMStoreStats.h"
10 #include "FWCore/ServiceRegistry/interface/Service.h"
11 #include "FWCore/MessageLogger/interface/JobReport.h"
12 
13 using namespace std;
14 using namespace edm;
15 
16 template <class T>
17 static unsigned int getEmptyMetric(T * array, int lenx, int leny, int lenz)
18 {
19  // len{x,y,z} MUST include under/overflow bins.
20  unsigned int len = lenx+leny+lenz;
21  unsigned int result=0;
22  // start from 1 to exclude underflow bin. The comparison is accurate
23  // since it takes properly into account under/overflow bins, for all
24  // kind of histograms.
25  for (unsigned int i=1; i<len; ++i)
26  {
27  // get rid of under/overflow bins for x,y,z axis, to have a correct statistics.
28  if (i%(lenx-1)==0)
29  continue;
30  if (i%lenx==0)
31  continue;
32  if (i%(lenx+leny-1)==0)
33  continue;
34  if (i%(lenx+leny)==0)
35  continue;
36  if (i%(lenx+leny+lenz-1)==0)
37  continue;
38 
39  if (array[i] == 0)
40  result += 1;
41  }
42 
43  return result;
44 }
45 
46 //==================================================================//
47 //================= Constructor and Destructor =====================//
48 //==================================================================//
49 DQMStoreStats::DQMStoreStats( const edm::ParameterSet& ps )
50  : subsystem_ (""),
51  subfolder_ (""),
52  nbinsglobal_ (0),
53  nbinssubsys_ (0),
54  nmeglobal_ (0),
55  nmesubsys_ (0),
56  maxbinsglobal_ (0),
57  maxbinssubsys_ (0),
58  maxbinsmeglobal_ (""),
59  maxbinsmesubsys_ (""),
60  statsdepth_ (1),
61  pathnamematch_ ("*"),
62  verbose_ (0)
63 {
64  parameters_ = ps;
65  pathnamematch_ = ps.getUntrackedParameter<std::string>( "pathNameMatch", pathnamematch_ );
66  statsdepth_ = ps.getUntrackedParameter<int>( "statsDepth", statsdepth_ );
67  verbose_ = ps.getUntrackedParameter<int>( "verbose", verbose_ );
68  dumpMemHistory_ = ps.getUntrackedParameter<bool>( "dumpMemoryHistory", false );
69  runonendrun_ = ps.getUntrackedParameter<bool>( "runOnEndRun", true );
70  runonendjob_ = ps.getUntrackedParameter<bool>( "runOnEndJob", false );
71  runonendlumi_ = ps.getUntrackedParameter<bool>( "runOnEndLumi", false );
72  runineventloop_ = ps.getUntrackedParameter<bool>( "runInEventLoop", false );
73  dumpToFWJR_ = ps.getUntrackedParameter<bool>( "dumpToFWJR", false );
74 
75  startingTime_ = time( 0 );
76 }
77 
78 DQMStoreStats::~DQMStoreStats(){
79 }
80 
81 void DQMStoreStats::calcIgProfDump(Folder &root)
82 {
83  std::ofstream stream("dqm-bin-stats.sql");
84  stream << ""
85 " PRAGMA journal_mode=OFF;"
86 " PRAGMA count_changes=OFF;"
87 " DROP TABLE IF EXISTS files;"
88 " DROP TABLE IF EXISTS symbols;"
89 " DROP TABLE IF EXISTS mainrows;"
90 " DROP TABLE IF EXISTS children;"
91 " DROP TABLE IF EXISTS parents;"
92 " DROP TABLE IF EXISTS summary;"
93 " CREATE TABLE children ("
94 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
95 " parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
96 " from_parent_count INTEGER,"
97 " from_parent_calls INTEGER,"
98 " from_parent_paths INTEGER,"
99 " pct REAL"
100 " );"
101 " CREATE TABLE files ("
102 " id,"
103 " name TEXT"
104 " );"
105 " CREATE TABLE mainrows ("
106 " id INTEGER PRIMARY KEY,"
107 " symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
108 " self_count INTEGER,"
109 " cumulative_count INTEGER,"
110 " kids INTEGER,"
111 " self_calls INTEGER,"
112 " total_calls INTEGER,"
113 " self_paths INTEGER,"
114 " total_paths INTEGER,"
115 " pct REAL"
116 " );"
117 " CREATE TABLE parents ("
118 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
119 " child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
120 " to_child_count INTEGER,"
121 " to_child_calls INTEGER,"
122 " to_child_paths INTEGER,"
123 " pct REAL"
124 " );"
125 " CREATE TABLE summary ("
126 " counter TEXT,"
127 " total_count INTEGER,"
128 " total_freq INTEGER,"
129 " tick_period REAL"
130 " );"
131 " CREATE TABLE symbols ("
132 " id,"
133 " name TEXT,"
134 " filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
135 " );"
136 " CREATE UNIQUE INDEX fileIndex ON files (id);"
137 " CREATE INDEX selfCountIndex ON mainrows(self_count);"
138 " CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
139 " CREATE INDEX totalCountIndex ON mainrows(cumulative_count);" << std::endl;
140 
141  std::string sql_statement("");
142 
143  root.files(sql_statement);
144  root.symbols(sql_statement);
145  root.mainrows_cumulative(sql_statement);
146  root.summary(sql_statement);
147  VIterator<Folder *> subsystems = root.CreateIterator() ;
148  size_t ii=0;
149  for(subsystems.First() ; !subsystems.IsDone() ; subsystems.Next(), ++ii)
150  {
151  subsystems.CurrentItem()->mainrows(sql_statement);
152  subsystems.CurrentItem()->parents(sql_statement);
153  subsystems.CurrentItem()->children(sql_statement);
154  }
155  stream << sql_statement << std::endl;
156 }
157 
164 int DQMStoreStats::calcstats( int mode = DQMStoreStats::considerAllME ) {
165 
167  nbinsglobal_ = 0;
168  nbinssubsys_ = 0;
169  maxbinsglobal_ = 0;
170  maxbinssubsys_ = 0;
171  std::string path = "";
172  std::string subsystemname = "";
173  std::string subfoldername = "";
174  size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;
175 
176 
177  std::vector<MonitorElement*> melist;
178  melist = dbe_->getMatchingContents( pathnamematch_ );
179 
180  Folder dbeFolder("root");
181  DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
182 
183  // loop all ME
184  typedef std::vector <MonitorElement*>::iterator meIt;
185  for(meIt it = melist.begin(); it != melist.end(); ++it) {
186 
187  // consider only ME with getLumiFlag() == true ?
188  if( mode == DQMStoreStats::considerOnlyLumiProductME &&
189  !( (*it)->getLumiFlag() ) ) continue;
190 
191  // figure out subsystem/subfolder names
192  std::string path = (*it)->getPathname();
193 
194  subfolderStringBegin = 0;
195  Folder * curr = &dbeFolder;
196  while(1)
197  {
198  subfolderStringEnd = path.find( '/', subfolderStringBegin );
199  if( std::string::npos == subfolderStringEnd )
200  {
201  curr = curr->cd(path.substr( subfolderStringBegin, path.size()-subfolderStringBegin ));
202  break;
203  }
204  curr = curr->cd(path.substr( subfolderStringBegin, subfolderStringEnd-subfolderStringBegin ));
205  subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
206  }
207 
208  // protection against ghost ME with empty paths
209  if( 0 == path.size() ) continue;
210 
211  subsysStringEnd = path.find( '/', 0 );
212  if( std::string::npos == subsysStringEnd ) subsysStringEnd = path.size(); // no subfolder
213 
214  // new subsystem?
215  if( path.substr( 0, subsysStringEnd ) != subsystemname ) {
216  DQMStoreStatsSubsystem aSubsystem;
217  subsystemname = path.substr( 0, subsysStringEnd );
218  aSubsystem.subsystemName_ = subsystemname;
219  dqmStoreStatsTopLevel.push_back( aSubsystem );
220  subfoldername = "";
221  }
222 
223  // get subfolder name (if there is one..)
224  if( path.size() == subsysStringEnd ) {
225  // no subfolders in subsystem, make dummy
226  DQMStoreStatsSubfolder aSubfolder;
227  aSubfolder.subfolderName_ = subsystemname; // <-- for tagging this case
228  dqmStoreStatsTopLevel.back().push_back( aSubfolder );
229  }
230 
231  else {
232 
233  // there is a subfolder, get its name
234  subfolderStringEnd = path.find( '/', subsysStringEnd + 1 );
235  if( std::string::npos == subfolderStringEnd ) subfolderStringEnd = path.size();
236 
237  // new subfolder?
238  if( path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 ) != subfoldername ) {
239  subfoldername = path.substr( subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1 );
240  DQMStoreStatsSubfolder aSubfolder;
241  aSubfolder.subfolderName_ = subfoldername;
242  dqmStoreStatsTopLevel.back().push_back( aSubfolder );
243  }
244 
245  }
246 
247  // shortcut
248  DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
249 
250  switch( (*it)->kind() ) {
251 
252  // one-dim ME
253  case MonitorElement::DQM_KIND_TH1F:
254  currentSubfolder.AddBinsF( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1F()->GetArray(), (*it)->getTH1F()->fN, 0, 0) );
255  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1F()->GetArray(), (*it)->getTH1F()->fN, 0, 0), (*it)->getNbinsX()*sizeof( float ) );
256  break;
257  case MonitorElement::DQM_KIND_TH1S:
258  currentSubfolder.AddBinsS( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1S()->GetArray(), (*it)->getTH1S()->fN, 0, 0) );
259  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1S()->GetArray(), (*it)->getTH1S()->fN, 0, 0), (*it)->getNbinsX()*sizeof( short ) );
260  break;
261  case MonitorElement::DQM_KIND_TH1D:
262  currentSubfolder.AddBinsD( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1D()->GetArray(), (*it)->getTH1D()->fN, 0, 0) );
263  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTH1D()->GetArray(), (*it)->getTH1D()->fN, 0, 0), (*it)->getNbinsX()*sizeof( double ) );
264  break;
265  case MonitorElement::DQM_KIND_TPROFILE:
266  currentSubfolder.AddBinsD( (*it)->getNbinsX(), getEmptyMetric((*it)->getTProfile()->GetArray(), (*it)->getTProfile()->fN, 0, 0) );
267  curr->update( (*it)->getNbinsX(), getEmptyMetric((*it)->getTProfile()->GetArray(), (*it)->getTProfile()->fN, 0, 0), (*it)->getNbinsX()*sizeof( double ) );
268  break;
269 
270  // two-dim ME
271  case MonitorElement::DQM_KIND_TH2F:
272  currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
273  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(float) );
274  break;
275  case MonitorElement::DQM_KIND_TH2S:
276  currentSubfolder.AddBinsS( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2S()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
277  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2S()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(short) );
278  break;
279  case MonitorElement::DQM_KIND_TH2D:
280  currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
281  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTH2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(double) );
282  break;
283  case MonitorElement::DQM_KIND_TPROFILE2D:
284  currentSubfolder.AddBinsD( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTProfile2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0) );
285  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY(), getEmptyMetric((*it)->getTProfile2D()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, 0), (*it)->getNbinsX() * (*it)->getNbinsY()*sizeof(double) );
286  break;
287 
288  // three-dim ME
289  case MonitorElement::DQM_KIND_TH3F:
290  currentSubfolder.AddBinsF( (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ(), getEmptyMetric( (*it)->getTH3F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, (*it)->getNbinsZ()+2 ) );
291  curr->update( (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ(),
292  getEmptyMetric( (*it)->getTH3F()->GetArray(), (*it)->getNbinsX()+2, (*it)->getNbinsY()+2, (*it)->getNbinsZ()+2 ),
293  (*it)->getNbinsX() * (*it)->getNbinsY() * (*it)->getNbinsZ()*sizeof(float));
294  break;
295 
296  default: {}
297  // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING
298  // which we don't care much about. Alternatively:
299 
300  // std::cerr << "[DQMStoreStats::calcstats] ** WARNING: monitor element of kind: "
301  // << (*it)->kind() << ", name: \"" << (*it)->getName() << "\"\n"
302  // << " in path: \"" << path << "\" not considered." << std::endl;
303  }
304  }
305 
306  if( mode == DQMStoreStats::considerAllME )
307  calcIgProfDump(dbeFolder);
308 
309  // OUTPUT
310 
311  std::cout << endl;
312  std::cout << "===========================================================================================" << std::endl;
313  std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
314  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
315  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
316  std::cout << "===========================================================================================" << std::endl;
317  std::cout << endl;
318 
319  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
320  std::cout << "Configuration:" << std::endl;
321  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
322  std::cout << " > running ";
323  if (runonendrun_) std::cout << "on run end." << std::endl;
324  if (runonendlumi_) std::cout << "on lumi end." << std::endl;
325  if (runonendjob_) std::cout << "on job end." << std::endl;
326  if (runineventloop_) std::cout << "in event loop." << std::endl;
327  std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
328  std::cout << std::endl;
329 
330  // dump folder structure
331  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
332  std::cout << "Top level folder tree:" << std::endl;
333  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
334  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
335  std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
336 
337  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
338  std::cout << " |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
339  }
340 
341  }
342 
343  // dump mem/bin table
344 
345  unsigned int overallNHistograms = 0, overallNBins = 0, overallNEmptyBins = 0, overallNBytes = 0;
346 
347  std::cout << std::endl;
348  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
349  std::cout << "Detailed ressource usage information ";
350  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME" << std::endl;
351  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
352  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
353  std::cout << "subsystem/folder histograms bins Empty bins Empty/Total bins per MB kB per" << std::endl;
354  std::cout << " (total) (total) (total) histogram (total) histogram " << std::endl;
355  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
356  for( DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 ) {
357  std::cout << it0->subsystemName_ << std::endl;
358 
359  unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
360 
361  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
362 
363  // fixed-size working copy
364  std::string thisSubfolderName( it1->subfolderName_ );
365  if( thisSubfolderName.size() > 30 ) {
366  thisSubfolderName.resize( 30 );
367  thisSubfolderName.replace( thisSubfolderName.size() - 3, 3, 3, '.' );
368  }
369 
370  std::cout << " -> " << std::setw( 30 ) << std::left << thisSubfolderName;
371  std::cout << std::setw( 14 ) << std::right << it1->totalHistos_;
372  std::cout << std::setw( 14 ) << std::right << it1->totalBins_;
373  std::cout << std::setw( 14 ) << std::right << it1->totalEmptyBins_;
374  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << (float)it1->totalEmptyBins_/(float)it1->totalBins_;
375 
376  // bins/histogram, need to catch nan if histos=0
377  if( it1->totalHistos_ ) {
378  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalBins_ / float( it1->totalHistos_ );
379  }
380  else std::cout << std::setw( 14 ) << std::right << "-";
381 
382  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / 1024.;
383 
384  // mem/histogram, need to catch nan if histos=0
385  if( it1->totalHistos_ ) {
386  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << it1->totalMemory_ / 1024. / it1->totalHistos_;
387  }
388  else std::cout << std::setw( 14 ) << std::right << "-";
389 
390  std::cout << std::endl;
391 
392  // collect totals
393  nHistograms += it1->totalHistos_;
394  nBins += it1->totalBins_;
395  nEmptyBins += it1->totalEmptyBins_;
396  nBytes += it1->totalMemory_;
397 
398  }
399 
400 
401 
402 
403  overallNHistograms += nHistograms;
404  overallNBins += nBins;
405  overallNEmptyBins += nEmptyBins;
406  overallNBytes += nBytes;
407 
408  // display totals
409  std::cout << " " << std::setw( 30 ) << std::left << "SUBSYSTEM TOTAL";
410  std::cout << std::setw( 14 ) << std::right << nHistograms;
411  std::cout << std::setw( 14 ) << std::right << nBins;
412  std::cout << std::setw( 14 ) << std::right << nEmptyBins;
413  std::cout << std::setw( 14 ) << std::right << (float)nEmptyBins/(float)nBins;
414  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBins / float( nHistograms );
415  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / 1000.;
416  std::cout << std::setw( 14 ) << std::right << std::setprecision( 3 ) << nBytes / 1024. / nHistograms;
417  std::cout << std::endl;
418 
419  std::cout << ".........................................................................................." << std::endl;
420 
421  }
422 
423 
424  // dump total
425  std::cout << std::endl;
426  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
427  std::cout << "Grand total ";
428  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME:" << std::endl;
429  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
430  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
431  std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
432  std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether" << std::endl;
433  std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB" << std::endl;
434 
435 
436 
437  std::cout << endl;
438  std::cout << "===========================================================================================" << std::endl;
439  std::cout << "[DQMStoreStats::calcstats] -- End of output ";
440  if( mode == DQMStoreStats::considerAllME ) std::cout << "FOR ALL ME." << std::endl;
441  else if( mode == DQMStoreStats::considerOnlyLumiProductME ) std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
442  std::cout << "===========================================================================================" << std::endl;
443  std::cout << endl;
444 
445  // Put together a simplified version of the complete dump that is
446  // sent to std::cout. Just dump the very basic information,
447  // i.e. summary for each folder, both for run and LS products.
448  if (dumpToFWJR_)
449  {
450  edm::Service<edm::JobReport> jr;
451  // Do not even try if the FWJR service is not available.
452  if (!jr.isAvailable())
453  return 0;
454  // Prepare appropriate map to store FWJR output.
455  std::map<std::string, std::string> jrInfo;
456  unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
457 
458  jrInfo["Source"] = "DQMServices/Components";
459  jrInfo["FileClass"] = "DQMStoreStats";
460  if (runonendrun_)
461  jrInfo["DumpType"] = "EndRun";
462  if (runonendlumi_)
463  jrInfo["DumpType"] = "EndLumi";
464  if (runonendjob_)
465  jrInfo["DumpType"] = "EndJob";
466  if (runineventloop_)
467  jrInfo["DumpType"] = "EventLoop";
468  if( mode == DQMStoreStats::considerAllME )
469  jrInfo["Type"] = "RunProduct";
470  else if( mode == DQMStoreStats::considerOnlyLumiProductME )
471  jrInfo["Type"] = "LumiProduct";
472 
473 
474  jrInfo["pathNameMatch"] = pathnamematch_;
475 
476  for (DQMStoreStatsTopLevel::const_iterator it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0 )
477  {
478  unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
479  for( DQMStoreStatsSubsystem::const_iterator it1 = it0->begin(); it1 < it0->end(); ++it1 ) {
480  // collect totals
481  nHistograms += it1->totalHistos_;
482  nBins += it1->totalBins_;
483  nEmptyBins += it1->totalEmptyBins_;
484  nBytes += it1->totalMemory_;
485  }
486  overallNHistograms += nHistograms;
487  overallNBins += nBins;
488  overallNBytes += nBytes;
489  std::stringstream iss("");
490  iss << nHistograms;
491  jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
492  iss.str("");iss<<nBins;
493  jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
494  iss.str("");iss<<nEmptyBins;
495  jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
496  iss.str("");iss<< ((float)nEmptyBins/(float)nBins);
497  jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
498  iss.str("");iss<< ((float)nBins/(float)nHistograms);
499  jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
500  iss.str("");iss<<nBytes / 1024. / 1024.;
501  jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
502  iss.str("");iss<<nBytes / 1024. / nHistograms;
503  jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
504  }
505  jr->reportAnalysisFile("DQMStatsReport", jrInfo);
506  }
507 
508  return 0;
509 
510 }
511 
512 
513 
517 void DQMStoreStats::dumpMemoryProfile( void ) {
518 
519  std::cout << std::endl;
520  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
521  std::cout << "Memory profile:" << std::endl;
522  std::cout << "------------------------------------------------------------------------------------------" << std::endl;
523 
524  // determine virtual memory maximum
525  std::pair<time_t, unsigned int> maxItem( 0, 0 );
526  for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
527  it < memoryHistoryVector_.end(); ++it ) {
528  if( it->second > maxItem.second ) {
529  maxItem = *it;
530  }
531  }
532 
533  std::stringstream rootOutputFileName;
534  rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
535 
536  // dump memory history to root file
537  if( dumpMemHistory_ && isOpenProcFileSuccessful_ ) {
538 
539  TFile outputFile( rootOutputFileName.str().c_str(), "RECREATE" );
540 
541  int aTime;
542  float aMb;
543 
544  TTree memHistoryTree( "dqmstorestats_memhistory", "memory history" );
545  memHistoryTree.Branch( "seconds", &aTime, "seconds/I" );
546  memHistoryTree.Branch( "megabytes", &aMb, "megabytes/F" );
547  for( std::vector<std::pair<time_t, unsigned int> >::const_iterator it = memoryHistoryVector_.begin();
548  it < memoryHistoryVector_.end(); ++it ) {
549  aTime = it->first - startingTime_;
550  aMb = it->second / 1000.;
551  memHistoryTree.Fill();
552  }
553 
554  outputFile.Write();
555  outputFile.Close();
556 
557  }
558 
559  std::cout << "Approx. maximum total virtual memory size of job: ";
560  if( isOpenProcFileSuccessful_ && memoryHistoryVector_.size() ) {
561  std::cout << maxItem.second / 1000.
562  << " MB (reached " << maxItem.first - startingTime_ << " sec. after constructor called)," << std::endl;
563  std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size() << " samples)" << std::endl;
564  } else {
565  std::cout << "(could not be determined)" << std::endl;
566  }
567 
568  std::cout << std::endl << std::endl;
569 
570 }
571 
572 
573 
577 void DQMStoreStats::print(){
578  // subsystem info printout
579  std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
580  std::cout << " " << subfolder_ << ": " ;
581  std::cout << nmesubsys_ << " histograms with "
582  << nbinssubsys_ << " bins. " ;
583  if (nmesubsys_ > 0) std::cout << nbinssubsys_/nmesubsys_ << " bins/histogram " ;
584  std::cout << std::endl;
585  std::cout << " Largest histogram: " << maxbinsmesubsys_ << " with " <<
586  maxbinssubsys_ << " bins." << std::endl;
587 }
588 
589 
590 
591 
595 std::pair<unsigned int, unsigned int> DQMStoreStats::readMemoryEntry( void ) const {
596 
597  // see if initial test reading was successful
598  if( isOpenProcFileSuccessful_ ) {
599 
600  std::ifstream procFile( procFileName_.str().c_str(), ios::in );
601 
602  std::string readBuffer( "" );
603  unsigned int memSize = 0;
604 
605  // scan procfile
606  while( !procFile.eof() ) {
607  procFile >> readBuffer;
608  if( std::string( "VmSize:" ) == readBuffer ) {
609  procFile >> memSize;
610  break;
611  }
612  }
613 
614  procFile.close();
615  return std::pair<time_t, unsigned int>( time( 0 ), memSize );
616  }
617 
618  return std::pair<time_t, unsigned int>( 0, 0 );
619 
620 }
621 
622 
623 
624 //==================================================================//
625 //========================= beginJob ===============================//
626 //==================================================================//
627 void DQMStoreStats::beginJob() {
628 
630  dbe_ = Service<DQMStore>().operator->();
631 
632  // access the proc/ folder for memory information
633  procFileName_ << "/proc/" << getpid() << "/status";
634 
635  // open for a test
636  std::ifstream procFile( procFileName_.str().c_str(), ios::in );
637 
638  if( procFile.good() ) {
639  isOpenProcFileSuccessful_ = true;
640  }
641  else {
642  std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
643  std::cerr << " Total memory profile will not be available." << std::endl;
644  isOpenProcFileSuccessful_ = false;
645  }
646 
647  procFile.close();
648 
649 }
650 
651 //==================================================================//
652 //========================= beginRun ===============================//
653 //==================================================================//
654 void DQMStoreStats::beginRun(const edm::Run& r, const EventSetup& context) {
655 }
656 
657 
658 //==================================================================//
659 //==================== beginLuminosityBlock ========================//
660 //==================================================================//
661 void DQMStoreStats::beginLuminosityBlock(const LuminosityBlock& lumiSeg,
662  const EventSetup& context) {
663 }
664 
665 
666 //==================================================================//
667 //==================== analyse (takes each event) ==================//
668 //==================================================================//
669 void DQMStoreStats::analyze(const Event& iEvent, const EventSetup& iSetup) {
670 
671  //now read virtual memory size from proc folder
672  memoryHistoryVector_.push_back( readMemoryEntry() );
673 
674  if (runineventloop_) {
675  calcstats( DQMStoreStats::considerAllME );
676  calcstats( DQMStoreStats::considerOnlyLumiProductME );
677  dumpMemoryProfile();
678  }
679 
680 }
681 
682 
683 //==================================================================//
684 //========================= endLuminosityBlock =====================//
685 //==================================================================//
686 void DQMStoreStats::endLuminosityBlock(const LuminosityBlock& lumiSeg,
687  const EventSetup& context) {
688  if (runonendlumi_) {
689  calcstats( DQMStoreStats::considerAllME );
690  calcstats( DQMStoreStats::considerOnlyLumiProductME );
691  dumpMemoryProfile();
692  }
693 
694 }
695 
696 //==================================================================//
697 //============================= endRun =============================//
698 //==================================================================//
699 void DQMStoreStats::endRun(const Run& r, const EventSetup& context) {
700 
701  if (runonendrun_) {
702  calcstats( DQMStoreStats::considerAllME );
703  calcstats( DQMStoreStats::considerOnlyLumiProductME );
704  dumpMemoryProfile();
705  }
706 
707 }
708 
709 //==================================================================//
710 //============================= endJob =============================//
711 //==================================================================//
712 void DQMStoreStats::endJob() {
713 
714  if (runonendjob_) {
715  calcstats( DQMStoreStats::considerAllME );
716  calcstats( DQMStoreStats::considerOnlyLumiProductME );
717  dumpMemoryProfile();
718  }
719 
720 }
download_sqlite_cfg.outputFile
tuple outputFile
Definition: download_sqlite_cfg.py:5
DQMStoreStatsSubfolder::subfolderName_
std::string subfolderName_
Definition: DQMStoreStats.h:42
edm::ParameterSet::getUntrackedParameter
T getUntrackedParameter(std::string const &, T const &) const
i
int i
Definition: DBlmapReader.cc:9
DQMStoreStats::dbe_
DQMStore * dbe_
Definition: DQMStoreStats.h:331
DQMStoreStatsSubfolder::AddBinsS
void AddBinsS(unsigned int nBins, unsigned int nEmptyBins)
Definition: DQMStoreStats.h:48
VIterator::First
virtual void First()
Definition: DQMStoreStats.h:89
Folder::update
void update(unsigned int bins, unsigned int empty, unsigned int memory)
Definition: DQMStoreStats.h:179
DQMStoreStats::beginRun
void beginRun(const edm::Run &r, const edm::EventSetup &c)
Definition: DQMStoreStats.cc:654
DQMStoreStatsSubfolder::AddBinsD
void AddBinsD(unsigned int nBins, unsigned int nEmptyBins)
Definition: DQMStoreStats.h:49
VIterator::IsDone
virtual bool IsDone() const
Definition: DQMStoreStats.h:94
DQMStoreStats::memoryHistoryVector_
std::vector< std::pair< time_t, unsigned int > > memoryHistoryVector_
Definition: DQMStoreStats.h:349
DQMStoreStatsSubsystem::subsystemName_
std::string subsystemName_
Definition: DQMStoreStats.h:59
alignBH_cfg.mode
list mode
Definition: alignBH_cfg.py:7
Folder::mainrows_cumulative
void mainrows_cumulative(std::string &sql_statement)
Definition: DQMStoreStats.h:252
Folder::CreateIterator
VIterator< Folder * > CreateIterator()
Definition: DQMStoreStats.h:198
DQMStoreStats::endLuminosityBlock
void endLuminosityBlock(const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c)
Definition: DQMStoreStats.cc:686
cuy.ii
int ii
Definition: cuy.py:588
DQMStoreStats::startingTime_
time_t startingTime_
Definition: DQMStoreStats.h:350
AlCaHLTBitMon_QueryRunRegistry.string
string string
Definition: AlCaHLTBitMon_QueryRunRegistry.py:255
ecal_dqm_sourceclient-live_cfg.cerr
tuple cerr
Definition: ecal_dqm_sourceclient-live_cfg.py:90
DQMStoreStats::endRun
void endRun(const edm::Run &r, const edm::EventSetup &c)
Definition: DQMStoreStats.cc:699
Folder::summary
void summary(std::string &sql_statement)
Definition: DQMStoreStats.h:262
DQMStoreStats::subsystem_
std::string subsystem_
Definition: DQMStoreStats.h:334
Folder::mainrows
void mainrows(std::string &sql_statement)
Definition: DQMStoreStats.h:203
DQMStoreStats::calcstats
int calcstats(int)
Definition: DQMStoreStats.cc:164
iEvent
int iEvent
Definition: GenABIO.cc:230
DQMStoreStats::isOpenProcFileSuccessful_
bool isOpenProcFileSuccessful_
Definition: DQMStoreStats.h:351
DQMStoreStats::readMemoryEntry
std::pair< unsigned int, unsigned int > readMemoryEntry(void) const
Definition: DQMStoreStats.cc:595
query.result
tuple result
Definition: query.py:137
DQMStoreStats::pathnamematch_
std::string pathnamematch_
Definition: DQMStoreStats.h:346
edm::Service::isAvailable
bool isAvailable() const
Definition: Service.h:46
DQMStoreStats::maxbinsmesubsys_
std::string maxbinsmesubsys_
Definition: DQMStoreStats.h:343
DQMStoreStats::DQMStoreStats
DQMStoreStats(const edm::ParameterSet &)
Definition: DQMStoreStats.cc:49
Folder::files
void files(std::string &sql_statement)
Definition: DQMStoreStats.h:270
VIterator::Next
virtual void Next()
Definition: DQMStoreStats.h:90
VIterator::CurrentItem
virtual Item CurrentItem() const
Definition: DQMStoreStats.h:100
Folder::parents
void parents(std::string &sql_statement)
Definition: DQMStoreStats.h:227
fed_dqm_sourceclient-live_cfg.path
string path
Definition: fed_dqm_sourceclient-live_cfg.py:33
DQMStoreStats::subfolder_
std::string subfolder_
Definition: DQMStoreStats.h:335
getEmptyMetric
static unsigned int getEmptyMetric(T *array, int lenx, int leny, int lenz)
Definition: DQMStoreStats.cc:17
Folder::cd
Folder * cd(const std::string &name)
Definition: DQMStoreStats.h:133
DQMStoreStats::parameters_
edm::ParameterSet parameters_
Definition: DQMStoreStats.h:332
DQMStoreStats::dumpMemoryProfile
void dumpMemoryProfile(void)
Definition: DQMStoreStats.cc:517
DQMStoreStatsSubfolder::AddBinsF
void AddBinsF(unsigned int nBins, unsigned int nEmptyBins)
Definition: DQMStoreStats.h:47
DQMStoreStats::beginLuminosityBlock
void beginLuminosityBlock(const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &context)
Definition: DQMStoreStats.cc:661
Folder::children
void children(std::string &sql_statement)
Definition: DQMStoreStats.h:239
edm::JobReport::reportAnalysisFile
void reportAnalysisFile(std::string const &fileName, std::map< std::string, std::string > const &fileData)
Definition: JobReport.cc:559
DQMStoreStats::calcIgProfDump
void calcIgProfDump(Folder &)
Definition: DQMStoreStats.cc:81
gather_cfg.cout
tuple cout
Definition: gather_cfg.py:121
DQMStoreStats::procFileName_
std::stringstream procFileName_
Definition: DQMStoreStats.h:352
edm::false
volatile std::atomic< bool > shutdown_flag false
Definition: UnixSignalHandlers.cc:22
DQMStoreStats::analyze
void analyze(const edm::Event &e, const edm::EventSetup &c)
Definition: DQMStoreStats.cc:669
T
long double T
Definition: Basic3DVectorLD.h:57
Folder::symbols
void symbols(std::string &sql_statement)
Definition: DQMStoreStats.h:216
edm::Run
Definition: Run.h:41
dbtoconf.root
string root
initialization
Definition: dbtoconf.py:70
Generated for CMSSW Reference Manual by doxygen 1.8.5