IndexIntoFile.h

Go to the documentation of this file.

#ifndef DataFormats_Provenance_IndexIntoFile_h
#define DataFormats_Provenance_IndexIntoFile_h

#include "DataFormats/Provenance/interface/EventID.h"
#include "DataFormats/Provenance/interface/ProcessHistoryID.h"
#include "DataFormats/Provenance/interface/RunID.h"
#include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
#include "FWCore/Utilities/interface/propagate_const.h"
#include "FWCore/Utilities/interface/value_ptr.h"
#include "FWCore/Utilities/interface/thread_safety_macros.h"

#include <memory>

#include <cassert>
#include <iosfwd>
#include <map>
#include <set>
#include <vector>

class TestIndexIntoFile;
class TestIndexIntoFile1;
class TestIndexIntoFile2;
class TestIndexIntoFile3;
class TestIndexIntoFile4;
class TestIndexIntoFile5;

namespace edm {

  class ProcessHistoryRegistry;
  class RootFile;

  class IndexIntoFile {
  public:
    class IndexIntoFileItr;
    class SortedRunOrLumiItr;
    class IndexRunLumiEventKey;

    using EntryNumber_t = long long;
    static constexpr int invalidIndex = -1;
    static constexpr RunNumber_t invalidRun = 0U;
    static constexpr LuminosityBlockNumber_t invalidLumi = 0U;
    static constexpr EventNumber_t invalidEvent = 0U;
    static constexpr EntryNumber_t invalidEntry = -1LL;

    enum EntryType { kRun, kLumi, kEvent, kEnd };

    IndexIntoFile();
    ~IndexIntoFile();

    ProcessHistoryID const& processHistoryID(int i) const;
    std::vector<ProcessHistoryID> const& processHistoryIDs() const;


    enum SortOrder { numericalOrder, firstAppearanceOrder, entryOrder };

    IndexIntoFileItr begin(SortOrder sortOrder) const;

    IndexIntoFileItr end(SortOrder sortOrder) const;

    bool iterationWillBeInEntryOrder(SortOrder sortOrder) const;

    bool empty() const;

    IndexIntoFileItr findPosition(RunNumber_t run, LuminosityBlockNumber_t lumi = 0U, EventNumber_t event = 0U) const;

    IndexIntoFileItr findPosition(SortOrder sortOrder,
                                  RunNumber_t run,
                                  LuminosityBlockNumber_t lumi = 0U,
                                  EventNumber_t event = 0U) const;

    IndexIntoFileItr findEventPosition(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const;

    IndexIntoFileItr findLumiPosition(RunNumber_t run, LuminosityBlockNumber_t lumi) const;

    IndexIntoFileItr findRunPosition(RunNumber_t run) const;

    bool containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const;
    bool containsEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const;
    bool containsLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) const;
    bool containsRun(RunNumber_t run) const;

    SortedRunOrLumiItr beginRunOrLumi() const;
    SortedRunOrLumiItr endRunOrLumi() const;

    void set_intersection(IndexIntoFile const& indexIntoFile, std::set<IndexRunLumiEventKey>& intersection) const;

    bool containsDuplicateEvents() const;

    //*****************************************************************************
    //*****************************************************************************

    class RunOrLumiEntry {
    public:
      RunOrLumiEntry();

      RunOrLumiEntry(EntryNumber_t orderPHIDRun,
                     EntryNumber_t orderPHIDRunLumi,
                     EntryNumber_t entry,
                     int processHistoryIDIndex,
                     RunNumber_t run,
                     LuminosityBlockNumber_t lumi,
                     EntryNumber_t beginEvents,
                     EntryNumber_t Event);

      EntryNumber_t orderPHIDRun() const { return orderPHIDRun_; }
      EntryNumber_t orderPHIDRunLumi() const { return orderPHIDRunLumi_; }
      EntryNumber_t entry() const { return entry_; }
      int processHistoryIDIndex() const { return processHistoryIDIndex_; }
      RunNumber_t run() const { return run_; }
      LuminosityBlockNumber_t lumi() const { return lumi_; }
      EntryNumber_t beginEvents() const { return beginEvents_; }
      EntryNumber_t endEvents() const { return endEvents_; }

      bool isRun() const { return lumi() == invalidLumi; }

      void setOrderPHIDRun(EntryNumber_t v) { orderPHIDRun_ = v; }
      void setOrderPHIDRunLumi(EntryNumber_t v) { orderPHIDRunLumi_ = v; }
      void setProcessHistoryIDIndex(int v) { processHistoryIDIndex_ = v; }

      bool operator<(RunOrLumiEntry const& right) const {
        if (orderPHIDRun_ == right.orderPHIDRun()) {
          if (orderPHIDRunLumi_ == right.orderPHIDRunLumi()) {
            return entry_ < right.entry();
          }
          return orderPHIDRunLumi_ < right.orderPHIDRunLumi();
        }
        return orderPHIDRun_ < right.orderPHIDRun();
      }

    private:
      // All Runs, Lumis, and Events associated with the same
      // ProcessHistory and Run in the same input file are processed
      // contiguously.  This parameter establishes the default order
      // of processing of these contiguous subsets of data.
      EntryNumber_t orderPHIDRun_;

      // All Lumis and Events associated with the same
      // ProcessHistory, Run, and Lumi in the same input file are
      // processed contiguously.  This parameter establishes the
      // default order of processing of these contiguous subsets
      // of data.
      EntryNumber_t orderPHIDRunLumi_;  // -1 if a run

      // TTree entry number of Run or Lumi
      // Always will be valid except when the IndexIntoFile was
      // created while processing more than 1 luminosity block
      // at a time (multiple concurrent lumis). In that case
      // there can be multiple contiguous event ranges associated
      // with the same lumi TTree entry. Exactly one of those will
      // have a valid entry_ number and the rest will be set
      // to the invalid value (-1). For a particular lumi, the
      // invalid ones sort before the valid ones.
      EntryNumber_t entry_;

      int processHistoryIDIndex_;
      RunNumber_t run_;
      LuminosityBlockNumber_t lumi_;  // 0 indicates this is a run entry

      // These are entry numbers in the Events TTree
      // Each RunOrLumiEntry is associated with one contiguous range of events.
      // This is disjoint from the ranges associated with all other RunOrLumiEntry's
      EntryNumber_t beginEvents_;  // -1 if a run or a lumi with no events
      EntryNumber_t endEvents_;    // -1 if a run or a lumi with no events
    };

    //*****************************************************************************
    //*****************************************************************************

    class RunOrLumiIndexes {
    public:
      RunOrLumiIndexes(int processHistoryIDIndex, RunNumber_t run, LuminosityBlockNumber_t lumi, int indexToGetEntry);

      int processHistoryIDIndex() const { return processHistoryIDIndex_; }
      RunNumber_t run() const { return run_; }
      LuminosityBlockNumber_t lumi() const { return lumi_; }
      int indexToGetEntry() const { return indexToGetEntry_; }
      long long beginEventNumbers() const { return beginEventNumbers_; }
      long long endEventNumbers() const { return endEventNumbers_; }

      bool isRun() const { return lumi() == invalidLumi; }

      void setBeginEventNumbers(long long v) { beginEventNumbers_ = v; }
      void setEndEventNumbers(long long v) { endEventNumbers_ = v; }

      bool operator<(RunOrLumiIndexes const& right) const {
        if (processHistoryIDIndex_ == right.processHistoryIDIndex()) {
          if (run_ == right.run()) {
            return lumi_ < right.lumi();
          }
          return run_ < right.run();
        }
        return processHistoryIDIndex_ < right.processHistoryIDIndex();
      }

    private:
      int processHistoryIDIndex_;
      RunNumber_t run_;
      LuminosityBlockNumber_t lumi_;  // 0 indicates this is a run entry
      int indexToGetEntry_;

      // The next two data members are indexes into the vectors eventNumbers_ and
      // eventEntries_ (which both have the same number of entries in the same order,
      // the only difference being that one contains only events numbers and is
      // smaller in memory).

      // If there are no events, then the next two are equal (and the value is the
      // index where the first event would have gone if there had been one)

      // Note that there can be many RunOrLumiIndexes objects where these two values are
      // the same if there are many noncontiguous ranges of events associated with the same
      // PHID-Run-Lumi (this one range in eventNumbers_ corresponds to the union of
      // all the noncontiguous ranges in the Events TTree).
      long long beginEventNumbers_;  // first event this PHID-Run-Lumi (-1 if a run or not set)
      long long endEventNumbers_;    // one past last event this PHID-Run-Lumi (-1 if a run or not set)
    };

    //*****************************************************************************
    //*****************************************************************************

    class EventEntry {
    public:
      EventEntry() : event_(invalidEvent), entry_(invalidEntry) {}
      EventEntry(EventNumber_t event, EntryNumber_t entry) : event_(event), entry_(entry) {}

      EventNumber_t event() const { return event_; }
      EntryNumber_t entry() const { return entry_; }

      bool operator<(EventEntry const& right) const { return event() < right.event(); }

      bool operator==(EventEntry const& right) const { return event() == right.event(); }

    private:
      EventNumber_t event_;
      EntryNumber_t entry_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class SortedRunOrLumiItr {
    public:
      SortedRunOrLumiItr(IndexIntoFile const* indexIntoFile, unsigned runOrLumi);

      IndexIntoFile const* indexIntoFile() const { return indexIntoFile_; }
      unsigned runOrLumi() const { return runOrLumi_; }

      bool operator==(SortedRunOrLumiItr const& right) const;
      bool operator!=(SortedRunOrLumiItr const& right) const;
      SortedRunOrLumiItr& operator++();

      bool isRun();

      void getRange(long long& beginEventNumbers,
                    long long& endEventNumbers,
                    EntryNumber_t& beginEventEntry,
                    EntryNumber_t& endEventEntry);

      RunOrLumiIndexes const& runOrLumiIndexes() const;

    private:
      IndexIntoFile const* indexIntoFile_;

      // This is an index into runOrLumiIndexes_
      // which gives the current position of the iteration
      unsigned runOrLumi_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexIntoFileItrImpl {
    public:
      IndexIntoFileItrImpl(IndexIntoFile const* indexIntoFile,
                           EntryType entryType,
                           int indexToRun,
                           int indexToLumi,
                           int indexToEventRange,
                           long long indexToEvent,
                           long long nEvents);
      virtual ~IndexIntoFileItrImpl();

      virtual IndexIntoFileItrImpl* clone() const = 0;

      EntryType getEntryType() const { return type_; }

      void next();

      void skipEventForward(int& phIndexOfSkippedEvent,
                            RunNumber_t& runOfSkippedEvent,
                            LuminosityBlockNumber_t& lumiOfSkippedEvent,
                            EntryNumber_t& skippedEventEntry);

      void skipEventBackward(int& phIndexOfEvent,
                             RunNumber_t& runOfEvent,
                             LuminosityBlockNumber_t& lumiOfEvent,
                             EntryNumber_t& eventEntry);

      virtual int processHistoryIDIndex() const = 0;
      virtual RunNumber_t run() const = 0;
      virtual LuminosityBlockNumber_t lumi() const = 0;
      virtual EntryNumber_t entry() const = 0;
      virtual bool shouldProcessLumi() const = 0;
      virtual bool shouldProcessRun() const = 0;
      virtual LuminosityBlockNumber_t peekAheadAtLumi() const = 0;
      virtual EntryNumber_t peekAheadAtEventEntry() const = 0;
      EntryNumber_t firstEventEntryThisRun();
      EntryNumber_t firstEventEntryThisLumi();
      virtual bool skipLumiInRun() = 0;
      virtual bool lumiIterationStartingIndex(int index) const = 0;

      void advanceToNextRun();
      void advanceToNextLumiOrRun();
      bool skipToNextEventInLumi();
      void initializeRun();

      void initializeLumi();

      bool operator==(IndexIntoFileItrImpl const& right) const;

      IndexIntoFile const* indexIntoFile() const { return indexIntoFile_; }

      // runOrLumiEntries_ and runOrLumiIndexes_ have the same size. It
      // is returned by the function size(). There are iterator data members
      // that are indexes into a container. The iterators also need the size of that
      // container and the function indexedSize() returns it. For the NoSort and
      // Sorted derived iterator classes, those indexes point directly into
      // runOrLumiEntries_ or runOrLumiIndexes_ and therefore return the same
      // value as size(). The indexes in the EntryOrder iterator class point into
      // a larger container and indexedSize() is overridden to give the size of
      // that container.
      int size() const { return size_; }
      virtual int indexedSize() const;

      EntryType type() const { return type_; }
      int indexToRun() const { return indexToRun_; }

      int indexToLumi() const { return indexToLumi_; }
      int indexToEventRange() const { return indexToEventRange_; }
      long long indexToEvent() const { return indexToEvent_; }
      long long nEvents() const { return nEvents_; }

      void copyPosition(IndexIntoFileItrImpl const& position);

      void getLumisInRun(std::vector<LuminosityBlockNumber_t>& lumis) const;

    protected:
      void setInvalid();

      void setIndexToLumi(int value) { indexToLumi_ = value; }
      void setIndexToEventRange(int value) { indexToEventRange_ = value; }
      void setIndexToEvent(long long value) { indexToEvent_ = value; }
      void setNEvents(long long value) { nEvents_ = value; }

    private:
      virtual void initializeLumi_() = 0;
      virtual bool nextEventRange() = 0;
      virtual bool previousEventRange() = 0;
      bool previousLumiWithEvents();
      virtual bool setToLastEventInRange(int index) = 0;
      virtual EntryType getRunOrLumiEntryType(int index) const = 0;
      virtual bool isSameLumi(int index1, int index2) const = 0;
      virtual bool isSameRun(int index1, int index2) const = 0;
      virtual LuminosityBlockNumber_t lumi(int index) const = 0;

      IndexIntoFile const* indexIntoFile_;
      int size_;

      EntryType type_;
      int indexToRun_;
      int indexToLumi_;
      int indexToEventRange_;
      long long indexToEvent_;
      long long nEvents_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexIntoFileItrNoSort : public IndexIntoFileItrImpl {
    public:
      IndexIntoFileItrNoSort(IndexIntoFile const* indexIntoFile,
                             EntryType entryType,
                             int indexToRun,
                             int indexToLumi,
                             int indexToEventRange,
                             long long indexToEvent,
                             long long nEvents);

      IndexIntoFileItrImpl* clone() const override;

      int processHistoryIDIndex() const override;
      RunNumber_t run() const override;
      LuminosityBlockNumber_t lumi() const override;
      EntryNumber_t entry() const override;
      bool shouldProcessLumi() const final { return true; }
      bool shouldProcessRun() const final { return true; }
      LuminosityBlockNumber_t peekAheadAtLumi() const override;
      EntryNumber_t peekAheadAtEventEntry() const override;
      bool skipLumiInRun() override;
      bool lumiIterationStartingIndex(int index) const override;

    private:
      void initializeLumi_() override;
      bool nextEventRange() override;
      bool previousEventRange() override;
      bool setToLastEventInRange(int index) override;
      EntryType getRunOrLumiEntryType(int index) const override;
      bool isSameLumi(int index1, int index2) const override;
      bool isSameRun(int index1, int index2) const override;
      LuminosityBlockNumber_t lumi(int index) const override;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexIntoFileItrSorted : public IndexIntoFileItrImpl {
    public:
      IndexIntoFileItrSorted(IndexIntoFile const* indexIntoFile,
                             EntryType entryType,
                             int indexToRun,
                             int indexToLumi,
                             int indexToEventRange,
                             long long indexToEvent,
                             long long nEvents);

      IndexIntoFileItrImpl* clone() const override;
      int processHistoryIDIndex() const override;
      RunNumber_t run() const override;
      LuminosityBlockNumber_t lumi() const override;
      EntryNumber_t entry() const override;
      bool shouldProcessLumi() const final { return true; }
      bool shouldProcessRun() const final { return true; }
      LuminosityBlockNumber_t peekAheadAtLumi() const override;
      EntryNumber_t peekAheadAtEventEntry() const override;
      bool skipLumiInRun() override;
      bool lumiIterationStartingIndex(int index) const override;

    private:
      void initializeLumi_() override;
      bool nextEventRange() override;
      bool previousEventRange() override;
      bool setToLastEventInRange(int index) override;
      EntryType getRunOrLumiEntryType(int index) const override;
      bool isSameLumi(int index1, int index2) const override;
      bool isSameRun(int index1, int index2) const override;
      LuminosityBlockNumber_t lumi(int index) const override;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexIntoFileItrEntryOrder : public IndexIntoFileItrImpl {
    public:
      IndexIntoFileItrEntryOrder(IndexIntoFile const* indexIntoFile,
                                 EntryType entryType,
                                 int indexToRun,
                                 int indexToLumi,
                                 int indexToEventRange,
                                 long long indexToEvent,
                                 long long nEvents);

      IndexIntoFileItrImpl* clone() const override;
      int processHistoryIDIndex() const override;
      RunNumber_t run() const override;
      LuminosityBlockNumber_t lumi() const override;
      EntryNumber_t entry() const override;
      bool shouldProcessLumi() const final;
      bool shouldProcessRun() const final;
      LuminosityBlockNumber_t peekAheadAtLumi() const override;
      EntryNumber_t peekAheadAtEventEntry() const override;
      bool skipLumiInRun() override;
      bool lumiIterationStartingIndex(int index) const override;
      int indexedSize() const override { return indexedSize_; }

    private:
      // Note the argument to the function runOrLumisEntry is NOT a
      // direct index into the vector in IndexIntoFile! It returns a
      // reference to an object from that vector. However, the argument
      // to runOrLumisEntry is an index into fileOrderRunOrLumiEntry_
      // which reorders the elements so the iteration is in event TTree
      // entry order. It also adds in dummy entries to insert
      // the extra run and lumi transitions required for that ordering.

      RunOrLumiEntry const& runOrLumisEntry(EntryNumber_t iEntry) const {
        return indexIntoFile()->runOrLumiEntries()[fileOrderRunOrLumiEntry_[iEntry]];
      }
      bool shouldProcessRunOrLumi(EntryNumber_t iEntry) const { return shouldProcessRunOrLumi_[iEntry]; }
      bool shouldProcessEvents(EntryNumber_t iEntry) const { return shouldProcessEvents_[iEntry]; }
      void initializeLumi_() override;
      bool nextEventRange() override;
      bool previousEventRange() override;
      bool setToLastEventInRange(int index) override;
      EntryType getRunOrLumiEntryType(int index) const override;
      bool isSameLumi(int index1, int index2) const override;
      bool isSameRun(int index1, int index2) const override;
      LuminosityBlockNumber_t lumi(int index) const override;

      struct TTreeEntryAndIndex {
        IndexIntoFile::EntryNumber_t ttreeEntry_;
        int runOrLumiIndex_;
      };

      class EntryOrderInitializationInfo {
      public:
        void resizeVectors(std::vector<RunOrLumiEntry> const&);
        void gatherNeededInfo(std::vector<RunOrLumiEntry> const&);
        void fillIndexesSortedByEventEntry(std::vector<RunOrLumiEntry> const&);
        void fillIndexesToLastContiguousEvents(std::vector<RunOrLumiEntry> const&);

        // Contains information only needed in the constructor of IndexIntoFileItrEntryOrder
        std::vector<int> firstIndexOfRun_;
        std::vector<int> firstIndexOfLumi_;
        std::vector<int> startOfLastContiguousEventsInRun_;
        std::vector<int> startOfLastContiguousEventsInLumi_;

        // Will contain indexes of lumi entries with events and will be
        // sorted by first Event TTree entry number.
        std::vector<TTreeEntryAndIndex> indexesSortedByEventEntry_;
        std::vector<TTreeEntryAndIndex>::const_iterator iEventSequence_;
        std::vector<TTreeEntryAndIndex>::const_iterator iEventSequenceEnd_;
        int eventSequenceIndex_ = 0;
        RunOrLumiEntry const* eventSequenceRunOrLumiEntry_ = nullptr;

        void nextEventSequence(std::vector<RunOrLumiEntry> const& runOrLumiEntries) {
          ++iEventSequence_;
          if (iEventSequence_ < iEventSequenceEnd_) {
            eventSequenceIndex_ = iEventSequence_->runOrLumiIndex_;
            eventSequenceRunOrLumiEntry_ = &runOrLumiEntries[eventSequenceIndex_];
          }
        }

        // Holds the index to the first entry associated with each run with no events
        // Sorted by the first run TTree entry number
        std::vector<TTreeEntryAndIndex> runsWithNoEvents_;
        std::vector<TTreeEntryAndIndex>::const_iterator nextRunWithNoEvents_;
        std::vector<TTreeEntryAndIndex>::const_iterator endRunsWithNoEvents_;
      };

      void addRunsWithNoEvents(EntryOrderInitializationInfo&, EntryNumber_t maxRunTTreeEntry = invalidEntry);
      void fillLumisWithNoRemainingEvents(std::vector<TTreeEntryAndIndex>& lumisWithNoRemainingEvents,
                                          int startingIndex,
                                          EntryNumber_t currentRun,
                                          RunOrLumiEntry const* eventSequenceRunOrLumiEntry) const;
      void reserveSpaceInVectors(std::vector<EntryNumber_t>::size_type);
      void addToFileOrder(int index, bool processRunOrLumi, bool processEvents);
      void handleToEndOfContiguousEventsInRun(EntryOrderInitializationInfo& info, EntryNumber_t currentRun);
      void handleToEndOfContiguousEventsInLumis(EntryOrderInitializationInfo& info,
                                                EntryNumber_t currentRun,
                                                int endOfRunEntries);
      EntryNumber_t lowestInLumi(EntryOrderInitializationInfo& info, int currentLumi) const;
      void handleLumisWithNoEvents(std::vector<TTreeEntryAndIndex>::const_iterator& nextLumiWithNoEvents,
                                   std::vector<TTreeEntryAndIndex>::const_iterator& endLumisWithNoEvents,
                                   EntryNumber_t lumiTTreeEntryNumber,
                                   bool completeAll = false);
      void handleLumiWithEvents(EntryOrderInitializationInfo& info,
                                int currentLumi,
                                EntryNumber_t firstBeginEventsContiguousLumi);
      void handleLumiEntriesNoRemainingEvents(EntryOrderInitializationInfo& info,
                                              int& iLumiIndex,
                                              int currentLumi,
                                              EntryNumber_t firstBeginEventsContiguousLumi,
                                              bool completeAll = false);

      int indexedSize_ = 0;
      std::vector<EntryNumber_t> fileOrderRunOrLumiEntry_;
      std::vector<bool> shouldProcessRunOrLumi_;
      std::vector<bool> shouldProcessEvents_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexIntoFileItr {
    public:
      IndexIntoFileItr(IndexIntoFile const* indexIntoFile,
                       SortOrder sortOrder,
                       EntryType entryType,
                       int indexToRun,
                       int indexToLumi,
                       int indexToEventRange,
                       long long indexToEvent,
                       long long nEvents);

      EntryType getEntryType() const { return impl_->getEntryType(); }
      int processHistoryIDIndex() const { return impl_->processHistoryIDIndex(); }
      RunNumber_t run() const { return impl_->run(); }
      LuminosityBlockNumber_t lumi() const { return impl_->lumi(); }
      EntryNumber_t entry() const { return impl_->entry(); }
      bool shouldProcessLumi() const { return impl_->shouldProcessLumi(); }
      bool shouldProcessRun() const { return impl_->shouldProcessRun(); }
      bool lumiIterationStartingIndex(int index) const { return impl_->lumiIterationStartingIndex(index); }

      LuminosityBlockNumber_t peekAheadAtLumi() const { return impl_->peekAheadAtLumi(); }

      EntryNumber_t peekAheadAtEventEntry() const { return impl_->peekAheadAtEventEntry(); }

      EntryNumber_t firstEventEntryThisRun() { return impl_->firstEventEntryThisRun(); }
      EntryNumber_t firstEventEntryThisLumi() { return impl_->firstEventEntryThisLumi(); }

      // This is intentionally not implemented.
      // It would be difficult to implement for the no sort mode,
      // either slow or using extra memory.
      // It would be easy to implement for the sorted iteration,
      // but I did not implement it so both cases would offer a
      // consistent interface.
      // It looks like in all cases where this would be needed
      // it would not be difficult to get the event number
      // directly from the event auxiliary.
      // We may need to revisit this decision in the future.
      // EventNumber_t event() const;

      IndexIntoFileItr& operator++() {
        impl_->next();
        return *this;
      }

      void skipEventForward(int& phIndexOfSkippedEvent,
                            RunNumber_t& runOfSkippedEvent,
                            LuminosityBlockNumber_t& lumiOfSkippedEvent,
                            EntryNumber_t& skippedEventEntry) {
        impl_->skipEventForward(phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, skippedEventEntry);
      }

      void skipEventBackward(int& phIndexOfEvent,
                             RunNumber_t& runOfEvent,
                             LuminosityBlockNumber_t& lumiOfEvent,
                             EntryNumber_t& eventEntry) {
        impl_->skipEventBackward(phIndexOfEvent, runOfEvent, lumiOfEvent, eventEntry);
      }

      bool skipLumiInRun() { return impl_->skipLumiInRun(); }

      bool skipToNextEventInLumi() { return impl_->skipToNextEventInLumi(); }

      void advanceToNextRun() { impl_->advanceToNextRun(); }
      void advanceToNextLumiOrRun() { impl_->advanceToNextLumiOrRun(); }

      void advanceToEvent();
      void advanceToLumi();

      bool operator==(IndexIntoFileItr const& right) const { return *impl_ == *right.impl_; }

      bool operator!=(IndexIntoFileItr const& right) const { return !(*this == right); }

      void initializeRun() { impl_->initializeRun(); }

      void initializeLumi() { impl_->initializeLumi(); }

      void copyPosition(IndexIntoFileItr const& position);

      void getLumisInRun(std::vector<LuminosityBlockNumber_t>& lumis) const { impl_->getLumisInRun(lumis); }

    private:
      //for testing
      friend class ::TestIndexIntoFile;
      friend class ::TestIndexIntoFile2;
      friend class ::TestIndexIntoFile3;
      friend class ::TestIndexIntoFile4;
      friend class ::TestIndexIntoFile5;

      // The rest of these are intended to be used only by code which tests
      // this class.
      IndexIntoFile const* indexIntoFile() const { return impl_->indexIntoFile(); }
      int size() const { return impl_->size(); }
      int indexedSize() const { return impl_->indexedSize(); }
      EntryType type() const { return impl_->type(); }
      int indexToRun() const { return impl_->indexToRun(); }
      int indexToLumi() const { return impl_->indexToLumi(); }
      int indexToEventRange() const { return impl_->indexToEventRange(); }
      long long indexToEvent() const { return impl_->indexToEvent(); }
      long long nEvents() const { return impl_->nEvents(); }

      value_ptr<IndexIntoFileItrImpl> impl_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexRunKey {
    public:
      IndexRunKey(int index, RunNumber_t run) : processHistoryIDIndex_(index), run_(run) {}

      int processHistoryIDIndex() const { return processHistoryIDIndex_; }
      RunNumber_t run() const { return run_; }

      bool operator<(IndexRunKey const& right) const {
        if (processHistoryIDIndex_ == right.processHistoryIDIndex()) {
          return run_ < right.run();
        }
        return processHistoryIDIndex_ < right.processHistoryIDIndex();
      }

    private:
      int processHistoryIDIndex_;
      RunNumber_t run_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexRunLumiKey {
    public:
      IndexRunLumiKey(int index, RunNumber_t run, LuminosityBlockNumber_t lumi)
          : processHistoryIDIndex_(index), run_(run), lumi_(lumi) {}

      int processHistoryIDIndex() const { return processHistoryIDIndex_; }
      RunNumber_t run() const { return run_; }
      LuminosityBlockNumber_t lumi() const { return lumi_; }

      bool operator<(IndexRunLumiKey const& right) const {
        if (processHistoryIDIndex_ == right.processHistoryIDIndex()) {
          if (run_ == right.run()) {
            return lumi_ < right.lumi();
          }
          return run_ < right.run();
        }
        return processHistoryIDIndex_ < right.processHistoryIDIndex();
      }

    private:
      int processHistoryIDIndex_;
      RunNumber_t run_;
      LuminosityBlockNumber_t lumi_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class IndexRunLumiEventKey {
    public:
      IndexRunLumiEventKey(int index, RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event)
          : processHistoryIDIndex_(index), run_(run), lumi_(lumi), event_(event) {}

      int processHistoryIDIndex() const { return processHistoryIDIndex_; }
      RunNumber_t run() const { return run_; }
      LuminosityBlockNumber_t lumi() const { return lumi_; }
      EventNumber_t event() const { return event_; }

      bool operator<(IndexRunLumiEventKey const& right) const {
        if (processHistoryIDIndex_ == right.processHistoryIDIndex()) {
          if (run_ == right.run()) {
            if (lumi_ == right.lumi()) {
              return event_ < right.event();
            }
            return lumi_ < right.lumi();
          }
          return run_ < right.run();
        }
        return processHistoryIDIndex_ < right.processHistoryIDIndex();
      }

    private:
      int processHistoryIDIndex_;
      RunNumber_t run_;
      LuminosityBlockNumber_t lumi_;
      EventNumber_t event_;
    };

    //*****************************************************************************
    //*****************************************************************************

    class EventFinder {
    public:
      virtual ~EventFinder() {}
      virtual EventNumber_t getEventNumberOfEntry(EntryNumber_t entry) const = 0;
    };

    //*****************************************************************************
    //*****************************************************************************

    // The next two functions are used by the output module to fill the
    // persistent data members.

    void addEntry(ProcessHistoryID const& processHistoryID,
                  RunNumber_t run,
                  LuminosityBlockNumber_t lumi,
                  EventNumber_t event,
                  EntryNumber_t entry);

    void sortVector_Run_Or_Lumi_Entries();

    void checkForMissingRunOrLumiEntry() const;

    //used internally by addEntry
    void addLumi(int index, RunNumber_t run, LuminosityBlockNumber_t lumi, EntryNumber_t entry);
    //*****************************************************************************
    //*****************************************************************************

    // The next group of functions is used by the PoolSource (or other
    // input related code) to fill the IndexIntoFile.

    void fixIndexes(std::vector<ProcessHistoryID>& processHistoryIDs);

    void setNumberOfEvents(EntryNumber_t nevents) { transient_.numberOfEvents_ = nevents; }

    void setEventFinder(std::shared_ptr<EventFinder> ptr) { transient_.eventFinder_ = ptr; }

    void fillEventNumbers() const;

    void fillEventEntries() const;

    void fillEventNumbersOrEntries(bool needEventNumbers, bool needEventEntries) const;

    std::vector<EventNumber_t>& unsortedEventNumbers() { return transient_.unsortedEventNumbers_; }
    std::vector<EventNumber_t> const& unsortedEventNumbers() const { return transient_.unsortedEventNumbers_; }

    void inputFileClosed();

    void doneFileInitialization();

    std::vector<RunOrLumiEntry>& setRunOrLumiEntries() { return runOrLumiEntries_; }

    std::vector<ProcessHistoryID>& setProcessHistoryIDs() { return processHistoryIDs_; }

    void reduceProcessHistoryIDs(ProcessHistoryRegistry const& processHistoryRegistry);

    //*****************************************************************************
    //*****************************************************************************

    std::vector<RunOrLumiEntry> const& runOrLumiEntries() const { return runOrLumiEntries_; }

    //*****************************************************************************
    //*****************************************************************************

    void initializeTransients() { transient_.reset(); }

    struct Transients {
      Transients();
      void reset();
      int previousAddedIndex_;
      std::map<IndexRunKey, EntryNumber_t> runToOrder_;
      std::map<IndexRunLumiKey, EntryNumber_t> lumiToOrder_;
      EntryNumber_t beginEvents_;
      EntryNumber_t endEvents_;
      int currentIndex_;
      RunNumber_t currentRun_;
      LuminosityBlockNumber_t currentLumi_;
      EntryNumber_t numberOfEvents_;
      edm::propagate_const<std::shared_ptr<EventFinder>> eventFinder_;
      std::vector<RunOrLumiIndexes> runOrLumiIndexes_;
      std::vector<EventNumber_t> eventNumbers_;
      std::vector<EventEntry> eventEntries_;
      std::vector<EventNumber_t> unsortedEventNumbers_;
    };

  private:
    //for testing
    friend class ::TestIndexIntoFile;
    friend class ::TestIndexIntoFile1;
    friend class ::TestIndexIntoFile2;
    friend class ::TestIndexIntoFile3;
    friend class ::TestIndexIntoFile4;
    friend class ::TestIndexIntoFile5;

    void fillRunOrLumiIndexes() const;

    std::vector<EventNumber_t>& unsortedEventNumbersMutable() const { return transient_.unsortedEventNumbers_; }
    void fillUnsortedEventNumbers() const;
    void resetEventFinder() const { transient_.eventFinder_ = nullptr; }  // propagate_const<T> has no reset() function
    std::vector<EventEntry>& eventEntries() const { return transient_.eventEntries_; }
    std::vector<EventNumber_t>& eventNumbers() const { return transient_.eventNumbers_; }
    void sortEvents() const;
    void sortEventEntries() const;
    int& previousAddedIndex() const { return transient_.previousAddedIndex_; }
    std::map<IndexRunKey, EntryNumber_t>& runToOrder() const { return transient_.runToOrder_; }
    std::map<IndexRunLumiKey, EntryNumber_t>& lumiToOrder() const { return transient_.lumiToOrder_; }
    EntryNumber_t& beginEvents() const { return transient_.beginEvents_; }
    EntryNumber_t& endEvents() const { return transient_.endEvents_; }
    int& currentIndex() const { return transient_.currentIndex_; }
    RunNumber_t& currentRun() const { return transient_.currentRun_; }
    LuminosityBlockNumber_t& currentLumi() const { return transient_.currentLumi_; }
    std::vector<RunOrLumiIndexes>& runOrLumiIndexes() const { return transient_.runOrLumiIndexes_; }
    size_t numberOfEvents() const { return transient_.numberOfEvents_; }
    EventNumber_t getEventNumberOfEntry(EntryNumber_t entry) const {
      return transient_.eventFinder_->getEventNumberOfEntry(entry);
    }

    //This class is used only by one thread at a time within the source serialized code
    CMS_SA_ALLOW mutable Transients transient_;

    std::vector<ProcessHistoryID> processHistoryIDs_;  // of reduced process histories
    std::vector<RunOrLumiEntry> runOrLumiEntries_;
  };

  template <>
  struct value_ptr_traits<IndexIntoFile::IndexIntoFileItrImpl> {
    static IndexIntoFile::IndexIntoFileItrImpl* clone(IndexIntoFile::IndexIntoFileItrImpl const* p) {
      return p->clone();
    }
    static void destroy(IndexIntoFile::IndexIntoFileItrImpl* p) { delete p; }
  };

  class Compare_Index_Run {
  public:
    bool operator()(IndexIntoFile::RunOrLumiIndexes const& lh, IndexIntoFile::RunOrLumiIndexes const& rh);
  };

  class Compare_Index {
  public:
    bool operator()(IndexIntoFile::RunOrLumiIndexes const& lh, IndexIntoFile::RunOrLumiIndexes const& rh);
  };

  // This class exists only to allow forward declarations of IndexIntoFile::IndexIntoFileItr
  class IndexIntoFileItrHolder {
  public:
    IndexIntoFileItrHolder(IndexIntoFile::IndexIntoFileItr const& iIter) : iter_(iIter) {}
    void getLumisInRun(std::vector<LuminosityBlockNumber_t>& lumis) const { iter_.getLumisInRun(lumis); }

  private:
    IndexIntoFile::IndexIntoFileItr const& iter_;
  };
}  // namespace edm

#endif