StreamSchedule.h

Go to the documentation of this file.

#ifndef FWCore_Framework_StreamSchedule_h
#define FWCore_Framework_StreamSchedule_h

/*
  Author: Jim Kowalkowski  28-01-06

  A class for creating a schedule based on paths in the configuration file.
  The schedule is maintained as a sequence of paths.
  After construction, events can be fed to the object and passed through
  all the modules in the schedule.  All accounting about processing
  of events by modules and paths is contained here or in object held
  by containment.

  The trigger results producer and product are generated and managed here.
  This class also manages endpaths and calls to endjob and beginjob.
  Endpaths are just treated as a simple list of modules that need to
  do processing of the event and do not participate in trigger path
  activities.

  This class requires the high-level process pset.  It uses @process_name.
  If the high-level pset contains an "options" pset, then the
  following optional parameter can be present:
  bool wantSummary = true/false   # default false

  wantSummary indicates whether or not the pass/fail/error stats
  for modules and paths should be printed at the end-of-job.

  A TriggerResults object will always be inserted into the event
  for any schedule.  The producer of the TriggerResults EDProduct
  is always the first module in the endpath.  The TriggerResultInserter
  is given a fixed label of "TriggerResults".

  Processing of an event happens by pushing the event through the Paths.
  The scheduler performs the reset() on each of the workers independent
  of the Path objects.

  ------------------------

  About Paths:
  Paths fit into two categories:
  1) trigger paths that contribute directly to saved trigger bits
  2) end paths
  The StreamSchedule holds these paths in two data structures:
  1) main path list
  2) end path list

  Trigger path processing always precedes endpath processing.
  The order of the paths from the input configuration is
  preserved in the main paths list.

  ------------------------

  The StreamSchedule uses the TriggerNamesService to get the names of the
  trigger paths and end paths. When a TriggerResults object is created
  the results are stored in the same order as the trigger names from
  TriggerNamesService.

*/

#include "DataFormats/Common/interface/HLTGlobalStatus.h"
#include "DataFormats/Provenance/interface/ModuleDescription.h"
#include "FWCore/Framework/interface/ExceptionActions.h"
#include "FWCore/Framework/interface/EventPrincipal.h"
#include "FWCore/Framework/interface/ExceptionHelpers.h"
#include "FWCore/Framework/interface/Frameworkfwd.h"
#include "FWCore/Framework/interface/OccurrenceTraits.h"
#include "FWCore/Framework/interface/UnscheduledCallProducer.h"
#include "FWCore/Framework/interface/WorkerManager.h"
#include "FWCore/Framework/interface/Path.h"
#include "FWCore/Framework/interface/TransitionInfoTypes.h"
#include "FWCore/Framework/interface/maker/Worker.h"
#include "FWCore/Framework/interface/WorkerRegistry.h"
#include "FWCore/Framework/interface/EarlyDeleteHelper.h"
#include "FWCore/MessageLogger/interface/ExceptionMessages.h"
#include "FWCore/MessageLogger/interface/JobReport.h"
#include "FWCore/MessageLogger/interface/MessageLogger.h"
#include "FWCore/ServiceRegistry/interface/Service.h"
#include "FWCore/ServiceRegistry/interface/StreamContext.h"
#include "FWCore/Concurrency/interface/FunctorTask.h"
#include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
#include "FWCore/Utilities/interface/Algorithms.h"
#include "FWCore/Utilities/interface/BranchType.h"
#include "FWCore/Utilities/interface/ConvertException.h"
#include "FWCore/Utilities/interface/Exception.h"
#include "FWCore/Utilities/interface/StreamID.h"
#include "FWCore/Utilities/interface/get_underlying_safe.h"
#include "FWCore/Utilities/interface/propagate_const.h"
#include "FWCore/Utilities/interface/thread_safety_macros.h"

#include <map>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include <sstream>
#include <atomic>
#include <unordered_set>

namespace edm {

  class ActivityRegistry;
  class BranchIDListHelper;
  class ExceptionCollector;
  class ExceptionToActionTable;
  class OutputModuleCommunicator;
  class ProcessContext;
  class UnscheduledCallProducer;
  class WorkerInPath;
  class ModuleRegistry;
  class TriggerResultInserter;
  class PathStatusInserter;
  class EndPathStatusInserter;
  class PreallocationConfiguration;
  class WaitingTaskHolder;

  class ConditionalTaskHelper;

  namespace service {
    class TriggerNamesService;
  }

  namespace {
    template <typename T>
    class StreamScheduleSignalSentry {
    public:
      StreamScheduleSignalSentry(ActivityRegistry* a, typename T::Context const* context)
          : a_(a), context_(context), allowThrow_(false) {
        if (a_)
          T::preScheduleSignal(a_, context_);
      }
      ~StreamScheduleSignalSentry() noexcept(false) {
        // Caught exception is rethrown (when allowed)
        CMS_SA_ALLOW try {
          if (a_) {
            T::postScheduleSignal(a_, context_);
          }
        } catch (...) {
          if (allowThrow_) {
            throw;
          }
        }
      }

      void allowThrow() { allowThrow_ = true; }

    private:
      // We own none of these resources.
      ActivityRegistry* a_;  // We do not use propagate_const because the registry itself is mutable.
      typename T::Context const* context_;
      bool allowThrow_;
    };
  }  // namespace

  class StreamSchedule {
  public:
    typedef std::vector<std::string> vstring;
    typedef std::vector<Path> TrigPaths;
    typedef std::vector<Path> NonTrigPaths;
    typedef std::shared_ptr<HLTGlobalStatus> TrigResPtr;
    typedef std::shared_ptr<HLTGlobalStatus const> TrigResConstPtr;
    typedef std::shared_ptr<Worker> WorkerPtr;
    typedef std::vector<Worker*> AllWorkers;

    typedef std::vector<Worker*> Workers;

    typedef std::vector<WorkerInPath> PathWorkers;

    StreamSchedule(std::shared_ptr<TriggerResultInserter> inserter,
                   std::vector<edm::propagate_const<std::shared_ptr<PathStatusInserter>>>& pathStatusInserters,
                   std::vector<edm::propagate_const<std::shared_ptr<EndPathStatusInserter>>>& endPathStatusInserters,
                   std::shared_ptr<ModuleRegistry>,
                   ParameterSet& proc_pset,
                   service::TriggerNamesService const& tns,
                   PreallocationConfiguration const& prealloc,
                   ProductRegistry& pregistry,
                   ExceptionToActionTable const& actions,
                   std::shared_ptr<ActivityRegistry> areg,
                   std::shared_ptr<ProcessConfiguration const> processConfiguration,
                   StreamID streamID,
                   ProcessContext const* processContext);

    StreamSchedule(StreamSchedule const&) = delete;

    void processOneEventAsync(
        WaitingTaskHolder iTask,
        EventTransitionInfo&,
        ServiceToken const& token,
        std::vector<edm::propagate_const<std::shared_ptr<PathStatusInserter>>>& pathStatusInserters);

    template <typename T>
    void processOneStreamAsync(WaitingTaskHolder iTask,
                               typename T::TransitionInfoType& transitionInfo,
                               ServiceToken const& token,
                               bool cleaningUpAfterException = false);

    void beginStream();
    void endStream();

    StreamID streamID() const { return streamID_; }


    std::vector<ModuleDescription const*> getAllModuleDescriptions() const;

    void availablePaths(std::vector<std::string>& oLabelsToFill) const;

    void modulesInPath(std::string const& iPathLabel, std::vector<std::string>& oLabelsToFill) const;

    void moduleDescriptionsInPath(std::string const& iPathLabel,
                                  std::vector<ModuleDescription const*>& descriptions,
                                  unsigned int hint) const;

    void moduleDescriptionsInEndPath(std::string const& iEndPathLabel,
                                     std::vector<ModuleDescription const*>& descriptions,
                                     unsigned int hint) const;

    int totalEvents() const { return total_events_; }

    int totalEventsPassed() const { return total_passed_; }

    int totalEventsFailed() const { return totalEvents() - totalEventsPassed(); }

    void getTriggerReport(TriggerReport& rep) const;

    void clearCounters();

    void replaceModule(maker::ModuleHolder* iMod, std::string const& iLabel);

    void deleteModule(std::string const& iLabel);

    void initializeEarlyDelete(ModuleRegistry& modReg,
                               std::vector<std::string> const& branchesToDeleteEarly,
                               std::multimap<std::string, std::string> const& referencesToBranches,
                               std::vector<std::string> const& modulesToSkip,
                               edm::ProductRegistry const& preg);

    AllWorkers const& allWorkers() const { return workerManager_.allWorkers(); }

    AllWorkers const& unscheduledWorkers() const { return workerManager_.unscheduledWorkers(); }
    unsigned int numberOfUnscheduledModules() const { return number_of_unscheduled_modules_; }

    StreamContext const& context() const { return streamContext_; }

    struct AliasInfo {
      std::string friendlyClassName;
      std::string instanceLabel;
      std::string originalInstanceLabel;
      std::string originalModuleLabel;
    };

  private:
    //Sentry class to only send a signal if an
    // exception occurs. An exception is identified
    // by the destructor being called without first
    // calling completedSuccessfully().
    class SendTerminationSignalIfException {
    public:
      SendTerminationSignalIfException(edm::ActivityRegistry* iReg, edm::StreamContext const* iContext)
          : reg_(iReg), context_(iContext) {}
      ~SendTerminationSignalIfException() {
        if (reg_) {
          reg_->preStreamEarlyTerminationSignal_(*context_, TerminationOrigin::ExceptionFromThisContext);
        }
      }
      void completedSuccessfully() { reg_ = nullptr; }

    private:
      edm::ActivityRegistry* reg_;  // We do not use propagate_const because the registry itself is mutable.
      StreamContext const* context_;
    };

    ExceptionToActionTable const& actionTable() const { return workerManager_.actionTable(); }

    void resetAll();

    void finishedPaths(std::atomic<std::exception_ptr*>&, WaitingTaskHolder, EventTransitionInfo&);

    std::exception_ptr finishProcessOneEvent(std::exception_ptr);

    void reportSkipped(EventPrincipal const& ep) const;

    std::vector<Worker*> tryToPlaceConditionalModules(
        Worker*,
        std::unordered_set<std::string>& conditionalModules,
        std::unordered_multimap<std::string, edm::BranchDescription const*> const& conditionalModuleBranches,
        std::unordered_multimap<std::string, AliasInfo> const& aliasMap,
        ParameterSet& proc_pset,
        ProductRegistry& preg,
        PreallocationConfiguration const* prealloc,
        std::shared_ptr<ProcessConfiguration const> processConfiguration);
    void fillWorkers(ParameterSet& proc_pset,
                     ProductRegistry& preg,
                     PreallocationConfiguration const* prealloc,
                     std::shared_ptr<ProcessConfiguration const> processConfiguration,
                     std::string const& name,
                     bool ignoreFilters,
                     PathWorkers& out,
                     std::vector<std::string> const& endPathNames,
                     ConditionalTaskHelper const& conditionalTaskHelper,
                     std::unordered_set<std::string>& allConditionalModules);
    void fillTrigPath(ParameterSet& proc_pset,
                      ProductRegistry& preg,
                      PreallocationConfiguration const* prealloc,
                      std::shared_ptr<ProcessConfiguration const> processConfiguration,
                      int bitpos,
                      std::string const& name,
                      TrigResPtr,
                      std::vector<std::string> const& endPathNames,
                      ConditionalTaskHelper const& conditionalTaskHelper,
                      std::unordered_set<std::string>& allConditionalModules);
    void fillEndPath(ParameterSet& proc_pset,
                     ProductRegistry& preg,
                     PreallocationConfiguration const* prealloc,
                     std::shared_ptr<ProcessConfiguration const> processConfiguration,
                     int bitpos,
                     std::string const& name,
                     std::vector<std::string> const& endPathNames,
                     ConditionalTaskHelper const& conditionalTaskHelper,
                     std::unordered_set<std::string>& allConditionalModules);

    void addToAllWorkers(Worker* w);

    void resetEarlyDelete();

    TrigResConstPtr results() const { return get_underlying_safe(results_); }
    TrigResPtr& results() { return get_underlying_safe(results_); }

    void makePathStatusInserters(
        std::vector<edm::propagate_const<std::shared_ptr<PathStatusInserter>>>& pathStatusInserters,
        std::vector<edm::propagate_const<std::shared_ptr<EndPathStatusInserter>>>& endPathStatusInserters,
        ExceptionToActionTable const& actions);

    WorkerManager workerManager_;
    std::shared_ptr<ActivityRegistry> actReg_;  // We do not use propagate_const because the registry itself is mutable.

    edm::propagate_const<TrigResPtr> results_;

    edm::propagate_const<WorkerPtr> results_inserter_;
    std::vector<edm::propagate_const<WorkerPtr>> pathStatusInserterWorkers_;
    std::vector<edm::propagate_const<WorkerPtr>> endPathStatusInserterWorkers_;

    TrigPaths trig_paths_;
    TrigPaths end_paths_;
    std::vector<int> empty_trig_paths_;
    std::vector<int> empty_end_paths_;

    //For each branch that has been marked for early deletion
    // keep track of how many modules are left that read this data but have
    // not yet been run in this event
    std::vector<BranchToCount> earlyDeleteBranchToCount_;
    //NOTE the following is effectively internal data for each EarlyDeleteHelper
    // but putting it into one vector makes for better allocation as well as
    // faster iteration when used to reset the earlyDeleteBranchToCount_
    // Each EarlyDeleteHelper hold a begin and end range into this vector. The values
    // of this vector correspond to indexes into earlyDeleteBranchToCount_ so
    // tell which EarlyDeleteHelper is associated with which BranchIDs.
    std::vector<unsigned int> earlyDeleteHelperToBranchIndicies_;
    //There is one EarlyDeleteHelper per Module which are reading data that
    // has been marked for early deletion
    std::vector<EarlyDeleteHelper> earlyDeleteHelpers_;

    int total_events_;
    int total_passed_;
    unsigned int number_of_unscheduled_modules_;

    StreamID streamID_;
    StreamContext streamContext_;
  };

  void inline StreamSchedule::reportSkipped(EventPrincipal const& ep) const {
    Service<JobReport> reportSvc;
    reportSvc->reportSkippedEvent(ep.id().run(), ep.id().event());
  }

  template <typename T>
  void StreamSchedule::processOneStreamAsync(WaitingTaskHolder iHolder,
                                             typename T::TransitionInfoType& transitionInfo,
                                             ServiceToken const& token,
                                             bool cleaningUpAfterException) {
    auto const& principal = transitionInfo.principal();
    T::setStreamContext(streamContext_, principal);

    auto id = principal.id();
    ServiceWeakToken weakToken = token;
    auto doneTask = make_waiting_task(
        [this, iHolder, id, cleaningUpAfterException, weakToken](std::exception_ptr const* iPtr) mutable {
          std::exception_ptr excpt;
          if (iPtr) {
            excpt = *iPtr;
            //add context information to the exception and print message
            try {
              convertException::wrap([&]() { std::rethrow_exception(excpt); });
            } catch (cms::Exception& ex) {
              //TODO: should add the transition type info
              std::ostringstream ost;
              if (ex.context().empty()) {
                ost << "Processing " << T::transitionName() << " " << id;
              }
              ServiceRegistry::Operate op(weakToken.lock());
              addContextAndPrintException(ost.str().c_str(), ex, cleaningUpAfterException);
              excpt = std::current_exception();
            }

            ServiceRegistry::Operate op(weakToken.lock());
            actReg_->preStreamEarlyTerminationSignal_(streamContext_, TerminationOrigin::ExceptionFromThisContext);
          }
          // Caught exception is propagated via WaitingTaskHolder
          CMS_SA_ALLOW try {
            ServiceRegistry::Operate op(weakToken.lock());
            T::postScheduleSignal(actReg_.get(), &streamContext_);
          } catch (...) {
            if (not excpt) {
              excpt = std::current_exception();
            }
          }
          iHolder.doneWaiting(excpt);
        });

    auto task = make_functor_task(
        [this, h = WaitingTaskHolder(*iHolder.group(), doneTask), info = transitionInfo, weakToken]() mutable {
          auto token = weakToken.lock();
          ServiceRegistry::Operate op(token);
          // Caught exception is propagated via WaitingTaskHolder
          CMS_SA_ALLOW try {
            T::preScheduleSignal(actReg_.get(), &streamContext_);

            workerManager_.resetAll();
          } catch (...) {
            h.doneWaiting(std::current_exception());
            return;
          }

          for (auto& p : end_paths_) {
            p.runAllModulesAsync<T>(h, info, token, streamID_, &streamContext_);
          }

          for (auto& p : trig_paths_) {
            p.runAllModulesAsync<T>(h, info, token, streamID_, &streamContext_);
          }

          workerManager_.processOneOccurrenceAsync<T>(h, info, token, streamID_, &streamContext_, &streamContext_);
        });

    if (streamID_.value() == 0) {
      //Enqueueing will start another thread if there is only
      // one thread in the job. Having stream == 0 use spawn
      // avoids starting up another thread when there is only one stream.
      iHolder.group()->run([task]() {
        TaskSentry s{task};
        task->execute();
      });
    } else {
      oneapi::tbb::task_arena arena{oneapi::tbb::task_arena::attach()};
      arena.enqueue([task]() {
        TaskSentry s{task};
        task->execute();
      });
    }
  }
}  // namespace edm

#endif