CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Schedule.h
Go to the documentation of this file.
1 #ifndef FWCore_Framework_Schedule_h
2 #define FWCore_Framework_Schedule_h
3 
4 /*
5  Author: Jim Kowalkowski 28-01-06
6 
7  A class for creating a schedule based on paths in the configuration file.
8  The schedule is maintained as a sequence of paths.
9  After construction, events can be fed to the object and passed through
10  all the modules in the schedule. All accounting about processing
11  of events by modules and paths is contained here or in object held
12  by containment.
13 
14  The trigger results producer and product are generated and managed here.
15  This class also manages endpaths and calls to endjob and beginjob.
16  Endpaths are just treated as a simple list of modules that need to
17  do processing of the event and do not participate in trigger path
18  activities.
19 
20  This class requires the high-level process pset. It uses @process_name.
21  If the high-level pset contains an "options" pset, then the
22  following optional parameter can be present:
23  bool wantSummary = true/false # default false
24 
25  wantSummary indicates whether or not the pass/fail/error stats
26  for modules and paths should be printed at the end-of-job.
27 
28  A TriggerResults object will always be inserted into the event
29  for any schedule. The producer of the TriggerResults EDProduct
30  is always the first module in the endpath. The TriggerResultInserter
31  is given a fixed label of "TriggerResults".
32 
33  Processing of an event happens by pushing the event through the Paths.
34  The scheduler performs the reset() on each of the workers independent
35  of the Path objects.
36 
37  ------------------------
38 
39  About Paths:
40  Paths fit into two categories:
41  1) trigger paths that contribute directly to saved trigger bits
42  2) end paths
43  The Schedule holds these paths in two data structures:
44  1) main path list
45  2) end path list
46 
47  Trigger path processing always precedes endpath processing.
48  The order of the paths from the input configuration is
49  preserved in the main paths list.
50 
51  ------------------------
52 
53  The Schedule uses the TriggerNamesService to get the names of the
54  trigger paths and end paths. When a TriggerResults object is created
55  the results are stored in the same order as the trigger names from
56  TriggerNamesService.
57 
58 */
59 
81 
82 #include "boost/shared_ptr.hpp"
83 
84 #include <map>
85 #include <memory>
86 #include <set>
87 #include <string>
88 #include <vector>
89 #include <sstream>
90 
91 namespace edm {
92  namespace service {
93  class TriggerNamesService;
94  }
95  class ActivityRegistry;
96  class BranchIDListHelper;
97  class EventSetup;
98  class ExceptionCollector;
99  class OutputWorker;
100  class RunStopwatch;
101  class UnscheduledCallProducer;
102  class WorkerInPath;
103  class Schedule {
104  public:
105  typedef std::vector<std::string> vstring;
106  typedef std::vector<Path> TrigPaths;
107  typedef std::vector<Path> NonTrigPaths;
108  typedef boost::shared_ptr<HLTGlobalStatus> TrigResPtr;
109  typedef boost::shared_ptr<Worker> WorkerPtr;
110  typedef std::vector<Worker*> AllWorkers;
111  typedef std::vector<OutputWorker*> AllOutputWorkers;
112 
113  typedef std::vector<Worker*> Workers;
114 
115  typedef std::vector<WorkerInPath> PathWorkers;
116 
117  Schedule(ParameterSet& proc_pset,
119  ProductRegistry& pregistry,
120  BranchIDListHelper& branchIDListHelper,
121  ActionTable const& actions,
122  boost::shared_ptr<ActivityRegistry> areg,
123  boost::shared_ptr<ProcessConfiguration> processConfiguration,
124  const ParameterSet* subProcPSet);
125 
126  enum State { Ready = 0, Running, Latched };
127 
128  template <typename T>
129  void processOneOccurrence(typename T::MyPrincipal& principal,
130  EventSetup const& eventSetup,
131  bool cleaningUpAfterException = false);
132 
133  void beginJob(ProductRegistry const&);
134  void endJob(ExceptionCollector & collector);
135 
136  // Write the luminosity block
137  void writeLumi(LuminosityBlockPrincipal const& lbp);
138 
139  // Write the run
140  void writeRun(RunPrincipal const& rp);
141 
142  // Call closeFile() on all OutputModules.
143  void closeOutputFiles();
144 
145  // Call openNewFileIfNeeded() on all OutputModules
147 
148  // Call openFiles() on all OutputModules
150 
151  // Call respondToOpenInputFile() on all Modules
152  void respondToOpenInputFile(FileBlock const& fb);
153 
154  // Call respondToCloseInputFile() on all Modules
155  void respondToCloseInputFile(FileBlock const& fb);
156 
157  // Call respondToOpenOutputFiles() on all Modules
159 
160  // Call respondToCloseOutputFiles() on all Modules
162 
163  // Call shouldWeCloseFile() on all OutputModules.
164  bool shouldWeCloseOutput() const;
165 
167  void postForkReacquireResources(unsigned int iChildIndex, unsigned int iNumberOfChildren);
168 
169  std::pair<double, double> timeCpuReal() const {
170  return std::pair<double, double>(stopwatch_->cpuTime(), stopwatch_->realTime());
171  }
172 
175 
179  std::vector<ModuleDescription const*> getAllModuleDescriptions() const;
180 
182  void availablePaths(std::vector<std::string>& oLabelsToFill) const;
183 
185  void modulesInPath(std::string const& iPathLabel,
186  std::vector<std::string>& oLabelsToFill) const;
187 
191  int totalEvents() const {
192  return total_events_;
193  }
194 
197  int totalEventsPassed() const {
198  return total_passed_;
199  }
200 
203  int totalEventsFailed() const {
204  return totalEvents() - totalEventsPassed();
205  }
206 
209  void enableEndPaths(bool active);
210 
213  bool endPathsEnabled() const;
214 
217  void getTriggerReport(TriggerReport& rep) const;
218 
220  bool terminate() const;
221 
223  void clearCounters();
224 
227  bool changeModule(std::string const& iLabel, ParameterSet const& iPSet);
228 
229  private:
230 
231  AllWorkers::const_iterator workersBegin() const {
232  return all_workers_.begin();
233  }
234 
235  AllWorkers::const_iterator workersEnd() const {
236  return all_workers_.end();
237  }
238 
239  AllWorkers::iterator workersBegin() {
240  return all_workers_.begin();
241  }
242 
243  AllWorkers::iterator workersEnd() {
244  return all_workers_.end();
245  }
246 
247  void resetAll();
248 
249  template <typename T>
250  bool runTriggerPaths(typename T::MyPrincipal&, EventSetup const&);
251 
252  template <typename T>
253  void runEndPaths(typename T::MyPrincipal&, EventSetup const&);
254 
256 
257  void reportSkipped(EventPrincipal const& ep) const;
259  void reportSkipped(RunPrincipal const&) const {}
260 
261  void reduceParameterSet(ParameterSet& proc_pset,
262  vstring& modulesInConfig,
263  std::set<std::string> const& modulesInConfigSet,
264  vstring& labelsOnTriggerPaths,
265  vstring& shouldBeUsedLabels,
266  std::map<std::string, std::vector<std::pair<std::string, int> > >& outputModulePathPositions);
267 
268  void fillWorkers(ParameterSet& proc_pset,
269  ProductRegistry& preg,
270  boost::shared_ptr<ProcessConfiguration const> processConfiguration,
271  std::string const& name, bool ignoreFilters, PathWorkers& out,
272  vstring* labelsOnPaths);
273  void fillTrigPath(ParameterSet& proc_pset,
274  ProductRegistry& preg,
275  boost::shared_ptr<ProcessConfiguration const> processConfiguration,
276  int bitpos, std::string const& name, TrigResPtr,
277  vstring* labelsOnTriggerPaths);
278  void fillEndPath(ParameterSet& proc_pset,
279  ProductRegistry& preg,
280  boost::shared_ptr<ProcessConfiguration const> processConfiguration,
281  int bitpos, std::string const& name);
282 
283  void limitOutput(ParameterSet const& proc_pset, BranchIDLists const& branchIDLists);
284 
285  void addToAllWorkers(Worker* w);
286 
287  void resetEarlyDelete();
289  edm::ProductRegistry const& preg,
290  edm::ParameterSet const* subProcPSet);
291 
294  boost::shared_ptr<ActivityRegistry> actReg_;
295 
299 
302 
308  std::vector<int> empty_trig_paths_;
310 
311  //For each branch that has been marked for early deletion
312  // keep track of how many modules are left that read this data but have
313  // not yet been run in this event
314  std::vector<std::pair<BranchID,unsigned int>> earlyDeleteBranchToCount_;
315  //NOTE the following is effectively internal data for each EarlyDeleteHelper
316  // but putting it into one vector makes for better allocation as well as
317  // faster iteration when used to reset the earlyDeleteBranchToCount_
318  // Each EarlyDeleteHelper hold a begin and end range into this vector. The values
319  // of this vector correspond to indexes into earlyDeleteBranchToCount_ so
320  // tell which EarlyDeleteHelper is associated with which BranchIDs.
321  std::vector<unsigned int> earlyDeleteHelperToBranchIndicies_;
322  //There is one EarlyDeleteHelper per Module which are reading data that
323  // has been marked for early deletion
324  std::vector<EarlyDeleteHelper> earlyDeleteHelpers_;
325 
330 
331  boost::shared_ptr<UnscheduledCallProducer> unscheduled_;
332 
333  volatile bool endpathsAreActive_;
334  };
335 
336  // -----------------------------
337  // ProcessOneOccurrence is a functor that has bound a specific
338  // Principal and Event Setup, and can be called with a Path, to
339  // execute Path::processOneOccurrence for that event
340 
341  template <typename T>
343  public:
344  typedef void result_type;
345  ProcessOneOccurrence(typename T::MyPrincipal& principal, EventSetup const& setup) :
346  ep(principal), es(setup) {};
347 
349 
350  private:
351  typename T::MyPrincipal& ep;
352  EventSetup const& es;
353  };
354 
355  void
356  inline
358  Service<JobReport> reportSvc;
359  reportSvc->reportSkippedEvent(ep.id().run(), ep.id().event());
360  }
361 
362  template <typename T>
363  void
364  Schedule::processOneOccurrence(typename T::MyPrincipal& ep,
365  EventSetup const& es,
366  bool cleaningUpAfterException) {
367  this->resetAll();
368  for (int empty_trig_path : empty_trig_paths_) {
369  results_->at(empty_trig_path) = HLTPathStatus(hlt::Pass, 0);
370  }
371  state_ = Running;
372 
373  // A RunStopwatch, but only if we are processing an event.
374  RunStopwatch stopwatch(T::isEvent_ ? stopwatch_ : RunStopwatch::StopwatchPointer());
375 
376  if (T::isEvent_) {
377  ++total_events_;
378  setupOnDemandSystem(dynamic_cast<EventPrincipal&>(ep), es);
379  }
380  try {
381  try {
382  try {
383  //make sure the unscheduled items see this transition [Event will be a no-op]
384  unscheduled_->runNow<T>(ep, es);
385  if (runTriggerPaths<T>(ep, es)) {
386  if (T::isEvent_) ++total_passed_;
387  }
388  state_ = Latched;
389  }
390  catch(cms::Exception& e) {
392  assert (action != actions::IgnoreCompletely);
393  assert (action != actions::FailPath);
394  if (action == actions::SkipEvent) {
395  edm::printCmsExceptionWarning("SkipEvent", e);
396  } else {
397  throw;
398  }
399  }
400 
401  try {
402  CPUTimer timer;
403  if (results_inserter_.get()) results_inserter_->doWork<T>(ep, es, nullptr, &timer);
404  }
405  catch (cms::Exception & ex) {
406  if (T::isEvent_) {
407  ex.addContext("Calling produce method for module TriggerResultInserter");
408  }
409  std::ostringstream ost;
410  ost << "Processing " << ep.id();
411  ex.addContext(ost.str());
412  throw;
413  }
414 
415  if (endpathsAreActive_) runEndPaths<T>(ep, es);
416  if(T::isEvent_) resetEarlyDelete();
417  }
418  catch (cms::Exception& e) { throw; }
419  catch(std::bad_alloc& bda) { convertException::badAllocToEDM(); }
420  catch (std::exception& e) { convertException::stdToEDM(e); }
422  catch(char const* c) { convertException::charPtrToEDM(c); }
423  catch (...) { convertException::unknownToEDM(); }
424  }
425  catch(cms::Exception& ex) {
426  if (ex.context().empty()) {
427  addContextAndPrintException("Calling function Schedule::processOneOccurrence", ex, cleaningUpAfterException);
428  } else {
429  addContextAndPrintException("", ex, cleaningUpAfterException);
430  }
431  state_ = Ready;
432  throw;
433  }
434  // next thing probably is not needed, the product insertion code clears it
435  state_ = Ready;
436  }
437 
438  template <typename T>
439  bool
440  Schedule::runTriggerPaths(typename T::MyPrincipal& ep, EventSetup const& es) {
442  return results_->accept();
443  }
444 
445  template <typename T>
446  void
447  Schedule::runEndPaths(typename T::MyPrincipal& ep, EventSetup const& es) {
448  // Note there is no state-checking safety controlling the
449  // activation/deactivation of endpaths.
451 
452  // We could get rid of the functor ProcessOneOccurrence if we used
453  // boost::lambda, but the use of lambda with member functions
454  // which take multiple arguments, by both non-const and const
455  // reference, seems much more obscure...
456  //
457  // using namespace boost::lambda;
458  // for_all(end_paths_,
459  // bind(&Path::processOneOccurrence,
460  // boost::lambda::_1, // qualification to avoid ambiguity
461  // var(ep), // pass by reference (not copy)
462  // constant_ref(es))); // pass by const-reference (not copy)
463  }
464 }
465 
466 #endif
RunNumber_t run() const
Definition: EventID.h:42
State state_
Definition: Schedule.h:296
std::vector< Path > NonTrigPaths
Definition: Schedule.h:107
EventNumber_t event() const
Definition: EventID.h:44
void initializeEarlyDelete(edm::ParameterSet const &opts, edm::ProductRegistry const &preg, edm::ParameterSet const *subProcPSet)
Definition: Schedule.cc:466
string rep
Definition: cuy.py:1188
std::pair< double, double > timeCpuReal() const
Definition: Schedule.h:169
std::vector< BranchIDList > BranchIDLists
Definition: BranchIDList.h:19
roAction_t actions[nactions]
Definition: GenABIO.cc:200
void availablePaths(std::vector< std::string > &oLabelsToFill) const
adds to oLabelsToFill the labels for all paths in the process
Definition: Schedule.cc:1373
AllOutputWorkers all_output_workers_
Definition: Schedule.h:305
WorkerPtr results_inserter_
Definition: Schedule.h:303
bool endPathsEnabled() const
Definition: Schedule.cc:1404
void respondToCloseInputFile(FileBlock const &fb)
Definition: Schedule.cc:1304
std::vector< OutputWorker * > AllOutputWorkers
Definition: Schedule.h:111
void resetEarlyDelete()
Definition: Schedule.cc:1499
std::vector< std::string > vstring
Definition: Schedule.cc:277
bool changeModule(std::string const &iLabel, ParameterSet const &iPSet)
Definition: Schedule.cc:1337
void writeLumi(LuminosityBlockPrincipal const &lbp)
Definition: Schedule.cc:1289
std::vector< Worker * > AllWorkers
Definition: Schedule.h:110
std::vector< EarlyDeleteHelper > earlyDeleteHelpers_
Definition: Schedule.h:324
WorkerRegistry worker_reg_
Definition: Schedule.h:292
std::vector< WorkerInPath > PathWorkers
Definition: Schedule.h:115
void reportSkipped(LuminosityBlockPrincipal const &) const
Definition: Schedule.h:258
EventID const & id() const
TrigPaths trig_paths_
Definition: Schedule.h:306
void enableEndPaths(bool active)
Definition: Schedule.cc:1399
void addContextAndPrintException(char const *context, cms::Exception &ex, bool disablePrint)
std::string const & category() const
Definition: Exception.cc:183
TrigResPtr endpath_results_
Definition: Schedule.h:301
Schedule(ParameterSet &proc_pset, service::TriggerNamesService &tns, ProductRegistry &pregistry, BranchIDListHelper &branchIDListHelper, ActionTable const &actions, boost::shared_ptr< ActivityRegistry > areg, boost::shared_ptr< ProcessConfiguration > processConfiguration, const ParameterSet *subProcPSet)
Definition: Schedule.cc:281
int total_passed_
Definition: Schedule.h:328
boost::shared_ptr< HLTGlobalStatus > TrigResPtr
Definition: Schedule.h:108
int totalEventsFailed() const
Definition: Schedule.h:203
void resetAll()
Definition: Schedule.cc:1475
dictionary map
Definition: Association.py:205
void fillEndPath(ParameterSet &proc_pset, ProductRegistry &preg, boost::shared_ptr< ProcessConfiguration const > processConfiguration, int bitpos, std::string const &name)
Definition: Schedule.cc:878
RunStopwatch::StopwatchPointer stopwatch_
Definition: Schedule.h:329
Func for_all(ForwardSequence &s, Func f)
wrapper for std::for_each
Definition: Algorithms.h:16
void runEndPaths(typename T::MyPrincipal &, EventSetup const &)
Definition: Schedule.h:447
int totalEventsPassed() const
Definition: Schedule.h:197
void operator()(Path &p)
Definition: Schedule.h:348
void processOneOccurrence(typename T::MyPrincipal &, EventSetup const &)
Definition: Path.h:157
boost::shared_ptr< CPUTimer > StopwatchPointer
Definition: RunStopwatch.h:23
accept
Definition: HLTenums.h:22
void limitOutput(ParameterSet const &proc_pset, BranchIDLists const &branchIDLists)
Definition: Schedule.cc:734
void reduceParameterSet(ParameterSet &proc_pset, vstring &modulesInConfig, std::set< std::string > const &modulesInConfigSet, vstring &labelsOnTriggerPaths, vstring &shouldBeUsedLabels, std::map< std::string, std::vector< std::pair< std::string, int > > > &outputModulePathPositions)
Definition: Schedule.cc:616
int totalEvents() const
Definition: Schedule.h:191
void stdToEDM(std::exception const &e)
std::vector< int > empty_trig_paths_
Definition: Schedule.h:308
AllWorkers::iterator workersBegin()
Definition: Schedule.h:239
Definition: Path.h:39
void clearCounters()
Clear all the counters in the trigger report.
Definition: Schedule.cc:1467
T::MyPrincipal & ep
Definition: Schedule.h:351
std::list< std::string > const & context() const
Definition: Exception.cc:191
void printCmsExceptionWarning(char const *behavior, cms::Exception const &e, edm::JobReport *jobRep=0, int rc=-1)
bool runTriggerPaths(typename T::MyPrincipal &, EventSetup const &)
Definition: Schedule.h:440
void respondToCloseOutputFiles(FileBlock const &fb)
Definition: Schedule.cc:1312
std::vector< std::string > vstring
Definition: Schedule.h:105
bool terminate() const
Return whether each output module has reached its maximum count.
Definition: Schedule.cc:774
void respondToOpenInputFile(FileBlock const &fb)
Definition: Schedule.cc:1300
vstring trig_name_list_
Definition: Schedule.h:297
EventSetup const & es
Definition: Schedule.h:352
ActionTable const * act_table_
Definition: Schedule.h:293
std::vector< Worker * > Workers
Definition: Schedule.h:113
AllWorkers::const_iterator workersBegin() const
Definition: Schedule.h:231
void getTriggerReport(TriggerReport &rep) const
Definition: Schedule.cc:1456
tuple out
Definition: dbtoconf.py:99
boost::shared_ptr< Worker > WorkerPtr
Definition: Schedule.h:109
void respondToOpenOutputFiles(FileBlock const &fb)
Definition: Schedule.cc:1308
void charPtrToEDM(char const *c)
void stringToEDM(std::string &s)
volatile bool endpathsAreActive_
Definition: Schedule.h:333
TrigResPtr results_
Definition: Schedule.h:300
void reportSkipped(RunPrincipal const &) const
Definition: Schedule.h:259
void addContext(std::string const &context)
Definition: Exception.cc:227
int total_events_
Definition: Schedule.h:327
vstring empty_trig_path_names_
Definition: Schedule.h:309
void modulesInPath(std::string const &iPathLabel, std::vector< std::string > &oLabelsToFill) const
adds to oLabelsToFill in execution order the labels of all modules in path iPathLabel ...
Definition: Schedule.cc:1382
TrigPaths end_paths_
Definition: Schedule.h:307
void reportSkipped(EventPrincipal const &ep) const
Definition: Schedule.h:357
bool wantSummary_
Definition: Schedule.h:326
void postForkReacquireResources(unsigned int iChildIndex, unsigned int iNumberOfChildren)
Definition: Schedule.cc:1333
std::vector< unsigned int > earlyDeleteHelperToBranchIndicies_
Definition: Schedule.h:321
std::vector< ModuleDescription const * > getAllModuleDescriptions() const
Definition: Schedule.cc:1358
void fillTrigPath(ParameterSet &proc_pset, ProductRegistry &preg, boost::shared_ptr< ProcessConfiguration const > processConfiguration, int bitpos, std::string const &name, TrigResPtr, vstring *labelsOnTriggerPaths)
Definition: Schedule.cc:850
void processOneOccurrence(typename T::MyPrincipal &principal, EventSetup const &eventSetup, bool cleaningUpAfterException=false)
Definition: Schedule.h:364
void beginJob(ProductRegistry const &)
Definition: Schedule.cc:1316
actions::ActionCodes find(const std::string &category) const
Definition: Actions.cc:93
std::vector< Path > TrigPaths
Definition: Schedule.h:106
T w() const
void openNewOutputFilesIfNeeded()
Definition: Schedule.cc:1277
void preForkReleaseResources()
Definition: Schedule.cc:1330
void openOutputFiles(FileBlock &fb)
Definition: Schedule.cc:1281
void writeRun(RunPrincipal const &rp)
Definition: Schedule.cc:1285
ProcessOneOccurrence(typename T::MyPrincipal &principal, EventSetup const &setup)
Definition: Schedule.h:345
long double T
std::vector< std::pair< BranchID, unsigned int > > earlyDeleteBranchToCount_
Definition: Schedule.h:314
void endJob(ExceptionCollector &collector)
Definition: Schedule.cc:900
AllWorkers::iterator workersEnd()
Definition: Schedule.h:243
bool shouldWeCloseOutput() const
Definition: Schedule.cc:1293
void setupOnDemandSystem(EventPrincipal &principal, EventSetup const &es)
Definition: Schedule.cc:1492
void setup(std::vector< TH2F > &depth, std::string name, std::string units="")
void fillWorkers(ParameterSet &proc_pset, ProductRegistry &preg, boost::shared_ptr< ProcessConfiguration const > processConfiguration, std::string const &name, bool ignoreFilters, PathWorkers &out, vstring *labelsOnPaths)
Definition: Schedule.cc:792
vstring end_path_name_list_
Definition: Schedule.h:298
void closeOutputFiles()
Definition: Schedule.cc:1273
AllWorkers::const_iterator workersEnd() const
Definition: Schedule.h:235
void addToAllWorkers(Worker *w)
Definition: Schedule.cc:1482
boost::shared_ptr< ActivityRegistry > actReg_
Definition: Schedule.h:294
AllWorkers all_workers_
Definition: Schedule.h:304
boost::shared_ptr< UnscheduledCallProducer > unscheduled_
Definition: Schedule.h:331