test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
JobReport.h
Go to the documentation of this file.
1 #ifndef FWCore_MessageLogger_JobReport_h
2 #define FWCore_MessageLogger_JobReport_h
3 // -*- C++ -*-
4 //
5 // Package: MessageLogger
6 // Class : JobReport
7 //
74 //
75 // Original Author: Marc Paterno
76 //
77 
78 /*
79 Changes Log 1: 2009/01/14 10:29:00, Natalia Garcia Nebot
80  Modified and added some methods to report CPU and memory information from /proc/cpuinfo
81  and /proc/meminfo files and Memory statistics
82 */
83 
87 
88 #include <atomic>
89 #include <cstddef>
90 #include <iosfwd>
91 #include <map>
92 #include <memory>
93 #include <mutex>
94 #include <set>
95 #include <string>
96 #include <vector>
97 
98 #include "tbb/concurrent_unordered_map.h"
99 #include "tbb/concurrent_vector.h"
100 
101 namespace edm {
102 
103  class JobReport {
104  public:
105 
106  typedef unsigned int RunNumber;
107  typedef std::size_t Token;
108 
109  struct RunReport {
111  std::set<unsigned int> lumiSections;
112  };
113 
123  struct InputFile {
124  typedef std::vector<std::string> StringVector;
125 
129  std::string inputType; // primaryFiles, secondaryFiles, mixingFiles
130  std::string inputSourceClassName; // class which created the file
131  std::string moduleLabel; // name of class instance
133  std::size_t numEventsRead;
135  std::map<RunNumber, RunReport> runReports;
137  std::set<std::string> fastClonedBranches;
138  };
139 
149  struct OutputFile {
150 
152 
157  std::string moduleLabel; // name of class instance
161  std::size_t numEventsWritten;
163  std::vector<Token> contributingInputs;
164  tbb::concurrent_vector<Token> contributingInputsSecSource;
165  std::map<std::string, bool> fastCopyingInputs;
166  std::map<RunNumber, RunReport> runReports;
168  };
169 
171  public:
174  std::atomic<long long>& value() { return value_; }
175  std::atomic<long long> const& value() const { return value_; }
176  private:
177  std::atomic<long long> value_;
178  };
179 
180  struct JobReportImpl {
181 
182  JobReportImpl& operator=(JobReportImpl const&) = delete;
183  JobReportImpl(JobReportImpl const&) = delete;
184 
187 
188  /*
189  * Add the input file token provided to every output
190  * file currently available.
191  * Used whenever a new input file is opened, it's token
192  * is added to all open output files as a contributor
193  */
194  void insertInputForOutputs(InputType inputType, Token t);
195 
196  /*
197  * Associate a Lumi Section to all open output files
198  *
199  */
200  void associateLumiSection(JobReport::Token token, unsigned int runNumber, unsigned int lumiSection);
201 
202  /*
203  * Associate a Lumi Section to all open input files
204  *
205  */
206  void associateInputLumiSection(unsigned int runNumber, unsigned int lumiSection);
207 
208  /*
209  * Associate a run to all open output files
210  */
211  void associateRun(JobReport::Token token, unsigned int runNumber);
212 
213  /*
214  * Associate a run to all open output files
215  */
216  void associateInputRun(unsigned int runNumber);
217 
218  /*
219  * Write an InputFile object to the Logger
220  * Generate XML string for InputFile instance and dispatch to
221  * job report via MessageLogger
222  */
223  void writeInputFile(InputFile const& f);
224 
225  /*
226  * Write an OutputFile object to the Logger
227  * Generate an XML string for the OutputFile provided and
228  * dispatch it to the logger
229  * Contributing input tokens are resolved to the input LFN and PFN
230  *
231  * TODO: We have not yet addressed the issue where we cleanup not
232  * contributing input files.
233  * Also, it is possible to get fake input to output file mappings
234  * if an input file is open already when a new output file is opened
235  * but the input gets closed without contributing events to the
236  * output file due to filtering etc.
237  *
238  */
239  void writeOutputFile(OutputFile const& f);
240 
241  /*
242  * Flush all open files to logger in event of a problem.
243  */
244  void flushFiles(void);
245 
246  JobReportImpl(std::ostream* iOst): printedReadBranches_(false), ost_(iOst) {}
247 
248  std::ostream const* ost() const {return get_underlying_safe(ost_);}
249  std::ostream*& ost() {return get_underlying_safe(ost_);}
250 
251  std::vector<InputFile> inputFiles_;
252  tbb::concurrent_vector<InputFile> inputFilesSecSource_;
253  std::vector<OutputFile> outputFiles_;
254  std::map<std::string, long long> readBranches_;
255  std::map<std::string, long long> readBranchesSecFile_;
256  tbb::concurrent_unordered_map<std::string, AtomicLongLong> readBranchesSecSource_;
260  };
261 
262  JobReport();
263  //Does not take ownership of pointer
264  JobReport(std::ostream* outputStream);
265 
266  JobReport& operator=(JobReport const&) = delete;
267  JobReport(JobReport const&) = delete;
268 
269  ~JobReport();
270 
272  void childAfterFork(std::string const& jobReportFile, unsigned int childIndex, unsigned int numberOfChildren);
273 
274  void parentBeforeFork(std::string const& jobReportFile, unsigned int numberOfChildren);
275 
276  void parentAfterFork(std::string const& jobReportFile);
277 
281  Token inputFileOpened(std::string const& physicalFileName,
282  std::string const& logicalFileName,
283  std::string const& catalog,
284  std::string const& inputType,
285  std::string const& inputSourceClassName,
286  std::string const& moduleLabel,
287  std::string const& guid,
288  std::vector<std::string> const& branchNames);
289 
292  void eventReadFromFile(InputType inputType, Token fileToken);
293 
298  // CMS-THREADING Current implementation requires an instance of an
299  // OuputModule run on only one thread at a time.
300  void reportDataType(Token fileToken, std::string const& dataType);
301 
302 
306  void inputFileClosed(InputType inputType, Token fileToken);
307 
311  Token outputFileOpened(std::string const& physicalFileName,
312  std::string const& logicalFileName,
313  std::string const& catalog,
314  std::string const& outputModuleClassName,
315  std::string const& moduleLabel,
316  std::string const& guid,
317  std::string const& dataType,
318  std::string const& branchHash,
319  std::vector<std::string> const& branchNames);
320 
323  // CMS-THREADING Current implementation requires an instance of an
324  // OuputModule run on only one thread at a time.
326 
330  void outputFileClosed(Token fileToken);
331 
333 
338  void reportRunNumber(JobReport::Token token, unsigned int run);
339 
344  void reportLumiSection(JobReport::Token token, unsigned int run, unsigned int lumiSectId);
345 
351  void reportInputLumiSection(unsigned int run, unsigned int lumiSectId);
352 
356  void reportInputRunNumber(unsigned int run);
357 
363  void reportError(std::string const& shortDesc,
364  std::string const& longDesc,
365  int const& exitCode);
366 
372  void reportSkippedFile(std::string const& pfn, std::string const& lfn);
373 
374  void reportFallbackAttempt(std::string const& pfn, std::string const& lfn, std::string const& err);
375 
377  std::map<std::string, std::string> const& fileData) ;
378 
385  void reportMemoryInfo(std::vector<std::string> const& memoryData);
386 
393  void reportMessageInfo(std::map<std::string, double> const& messageData);
394 
401  void reportReadBranches();
402 
404  void reportReadBranch(InputType inputType, std::string const& branchName);
405 
407  void reportFastClonedBranches(std::set<std::string> const& fastClonedBranches, long long nEvents);
408 
413 
414  /*
415  * Report information about fast copying. Called for each open output file
416  * whenever an input file is opened.
417  */
418  void reportFastCopyingStatus(Token t, std::string const& inputFileName, bool fastCopying);
419 
427  void reportPerformanceSummary(std::string const& metricClass,
428  std::map<std::string, std::string> const& metrics);
429 
430  void reportPerformanceForModule(std::string const& metricClass,
431  std::string const& moduleName,
432  std::map<std::string, std::string> const& metrics);
433 
435  std::string dumpFiles(void);
436 
437  protected:
439 
440  private:
443  };
444 
445  std::ostream& operator<< (std::ostream& os, JobReport::InputFile const& f);
446  std::ostream& operator<< (std::ostream& os, JobReport::OutputFile const& f);
447 }
448 #endif
InputType
Definition: InputType.h:5
void reportSkippedFile(std::string const &pfn, std::string const &lfn)
Definition: JobReport.cc:597
std::string inputSourceClassName
Definition: JobReport.h:130
std::string outputModuleClassName
Definition: JobReport.h:156
std::string physicalFileName
Definition: JobReport.h:154
void associateRun(JobReport::Token token, unsigned int runNumber)
Definition: JobReport.cc:275
void reportMemoryInfo(std::vector< std::string > const &memoryData)
Definition: JobReport.cc:629
void associateLumiSection(JobReport::Token token, unsigned int runNumber, unsigned int lumiSection)
Definition: JobReport.cc:295
void reportInputRunNumber(unsigned int run)
Definition: JobReport.cc:554
void eventReadFromFile(InputType inputType, Token fileToken)
Definition: JobReport.cc:441
std::map< std::string, long long > readBranches_
Definition: JobReport.h:254
static boost::mutex mutex
Definition: LHEProxy.cc:11
std::mutex write_mutex
Definition: JobReport.h:442
void reportFallbackAttempt(std::string const &pfn, std::string const &lfn, std::string const &err)
Definition: JobReport.cc:613
std::string logicalFileName
Definition: JobReport.h:126
std::ostream const * ost() const
Definition: JobReport.h:248
std::set< std::string > fastClonedBranches
Definition: JobReport.h:137
unsigned long long EventNumber_t
std::atomic< long long > & value()
Definition: JobReport.h:174
std::map< std::string, bool > fastCopyingInputs
Definition: JobReport.h:165
std::string logicalFileName
Definition: JobReport.h:153
void reportFastCopyingStatus(Token t, std::string const &inputFileName, bool fastCopying)
Definition: JobReport.cc:533
JobReportImpl(JobReportImpl const &)=delete
void reportRunNumber(JobReport::Token token, unsigned int run)
Definition: JobReport.cc:549
void reportRandomStateFile(std::string const &name)
Definition: JobReport.cc:719
JobReport & operator=(JobReport const &)=delete
std::vector< std::string > StringVector
Definition: JobReport.h:124
uint16_t size_type
edm::propagate_const< std::unique_ptr< JobReportImpl > > impl_
Definition: JobReport.h:441
Token inputFileOpened(std::string const &physicalFileName, std::string const &logicalFileName, std::string const &catalog, std::string const &inputType, std::string const &inputSourceClassName, std::string const &moduleLabel, std::string const &guid, std::vector< std::string > const &branchNames)
Definition: JobReport.cc:393
std::vector< InputFile > inputFiles_
Definition: JobReport.h:251
void reportPerformanceForModule(std::string const &metricClass, std::string const &moduleName, std::map< std::string, std::string > const &metrics)
Definition: JobReport.cc:754
edm::propagate_const< std::unique_ptr< JobReportImpl > > & impl()
Definition: JobReport.h:438
std::map< RunNumber, RunReport > runReports
Definition: JobReport.h:135
std::vector< Token > contributingInputs
Definition: JobReport.h:163
std::string physicalFileName
Definition: JobReport.h:127
void reportSkippedEvent(RunNumber_t run, EventNumber_t event)
Definition: JobReport.cc:520
tbb::concurrent_unordered_map< std::string, AtomicLongLong > readBranchesSecSource_
Definition: JobReport.h:256
void insertInputForOutputs(InputType inputType, Token t)
Definition: JobReport.cc:176
InputFile::StringVector StringVector
Definition: JobReport.h:151
std::string moduleName(Provenance const &provenance)
Definition: Provenance.cc:27
std::size_t numEventsRead
Definition: JobReport.h:133
void writeInputFile(InputFile const &f)
Definition: JobReport.cc:193
StringVector branchNames
Definition: JobReport.h:162
JobReportImpl & operator=(JobReportImpl const &)=delete
void reportInputLumiSection(unsigned int run, unsigned int lumiSectId)
Definition: JobReport.cc:544
std::map< std::string, long long > readBranchesSecFile_
Definition: JobReport.h:255
def load
Definition: svgfig.py:546
std::map< RunNumber, RunReport > runReports
Definition: JobReport.h:166
void reportPerformanceSummary(std::string const &metricClass, std::map< std::string, std::string > const &metrics)
Definition: JobReport.cc:733
std::ostream *& ost()
Definition: JobReport.h:249
void childAfterFork(std::string const &jobReportFile, unsigned int childIndex, unsigned int numberOfChildren)
New output file for child.
Definition: JobReport.cc:383
std::set< unsigned int > lumiSections
Definition: JobReport.h:111
double f[11][100]
Token outputFileOpened(std::string const &physicalFileName, std::string const &logicalFileName, std::string const &catalog, std::string const &outputModuleClassName, std::string const &moduleLabel, std::string const &guid, std::string const &dataType, std::string const &branchHash, std::vector< std::string > const &branchNames)
Definition: JobReport.cc:467
std::shared_ptr< T > & get_underlying_safe(propagate_const< std::shared_ptr< T >> &iP)
AtomicLongLong(AtomicLongLong const &r)
Definition: JobReport.h:173
edm::propagate_const< std::ostream * > ost_
Definition: JobReport.h:259
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
std::atomic< long long > const & value() const
Definition: JobReport.h:175
std::vector< OutputFile > outputFiles_
Definition: JobReport.h:253
void reportError(std::string const &shortDesc, std::string const &longDesc, int const &exitCode)
Definition: JobReport.cc:580
unsigned int RunNumber
Definition: JobReport.h:106
std::size_t Token
Definition: JobReport.h:107
void associateInputRun(unsigned int runNumber)
Definition: JobReport.cc:283
void reportReadBranch(InputType inputType, std::string const &branchName)
Inform the job report that a branch has been read.
Definition: JobReport.cc:694
std::size_t numEventsWritten
Definition: JobReport.h:161
void reportFastClonedBranches(std::set< std::string > const &fastClonedBranches, long long nEvents)
Inform the job report that branches have been fast Cloned.
Definition: JobReport.cc:709
void eventWrittenToFile(Token fileToken, RunNumber_t run, EventNumber_t event)
Definition: JobReport.cc:507
tbb::concurrent_vector< InputFile > inputFilesSecSource_
Definition: JobReport.h:252
std::atomic< long long > value_
Definition: JobReport.h:177
void parentAfterFork(std::string const &jobReportFile)
Definition: JobReport.cc:379
void reportMessageInfo(std::map< std::string, double > const &messageData)
Definition: JobReport.cc:644
std::string dumpFiles(void)
debug/test util
Definition: JobReport.cc:777
void parentBeforeFork(std::string const &jobReportFile, unsigned int numberOfChildren)
Definition: JobReport.cc:361
void reportAnalysisFile(std::string const &fileName, std::map< std::string, std::string > const &fileData)
Definition: JobReport.cc:559
std::vector< InputFile >::size_type lastOpenedPrimaryInputFile_
Definition: JobReport.h:258
void outputFileClosed(Token fileToken)
Definition: JobReport.cc:513
void writeOutputFile(OutputFile const &f)
Definition: JobReport.cc:220
JobReportImpl(std::ostream *iOst)
Definition: JobReport.h:246
unsigned int RunNumber_t
InputFile & getInputFileForToken(InputType inputType, Token t)
Definition: JobReport.cc:119
void reportLumiSection(JobReport::Token token, unsigned int run, unsigned int lumiSectId)
Definition: JobReport.cc:539
void reportDataType(Token fileToken, std::string const &dataType)
Definition: JobReport.cc:447
volatile std::atomic< bool > shutdown_flag false
UInt_t nEvents
Definition: hcalCalib.cc:42
tbb::concurrent_vector< Token > contributingInputsSecSource
Definition: JobReport.h:164
StringVector branchNames
Definition: JobReport.h:134
std::ostream & operator<<(std::ostream &ost, const HLTGlobalStatus &hlt)
Formatted printout of trigger tbale.
void inputFileClosed(InputType inputType, Token fileToken)
Definition: JobReport.cc:453
void associateInputLumiSection(unsigned int runNumber, unsigned int lumiSection)
Definition: JobReport.cc:305
void reportReadBranches()
Definition: JobReport.cc:660
OutputFile & getOutputFileForToken(Token t)
Definition: JobReport.cc:151