RootFile.h

Go to the documentation of this file.

#ifndef IOPool_Input_RootFile_h
#define IOPool_Input_RootFile_h

/*----------------------------------------------------------------------

RootFile.h // used by ROOT input sources

----------------------------------------------------------------------*/

#include "RootTree.h"
#include "DataFormats/Provenance/interface/BranchChildren.h"
#include "DataFormats/Provenance/interface/BranchIDList.h"
#include "DataFormats/Provenance/interface/BranchListIndex.h"
#include "DataFormats/Provenance/interface/EntryDescriptionID.h"  // backward compatibility
#include "DataFormats/Provenance/interface/EventAuxiliary.h"
#include "DataFormats/Provenance/interface/EventEntryDescription.h"  // backward compatibility
#include "DataFormats/Provenance/interface/EventProcessHistoryID.h"  // backward compatibility
#include "DataFormats/Provenance/interface/EventSelectionID.h"
#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h"
#include "DataFormats/Provenance/interface/FileFormatVersion.h"
#include "DataFormats/Provenance/interface/FileID.h"
#include "DataFormats/Provenance/interface/History.h"
#include "DataFormats/Provenance/interface/IndexIntoFile.h"
#include "DataFormats/Provenance/interface/ProvenanceFwd.h"
#include "FWCore/Common/interface/FWCoreCommonFwd.h"
#include "FWCore/Framework/interface/Frameworkfwd.h"
#include "FWCore/Framework/interface/InputSource.h"
#include "FWCore/Utilities/interface/InputType.h"
#include "FWCore/Utilities/interface/get_underlying_safe.h"
#include "FWCore/Utilities/interface/propagate_const.h"

#include "TBranch.h"

#include <array>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <tuple>
#include <vector>

namespace edm {

  //------------------------------------------------------------
  // Class RootFile: supports file reading.

  class BranchID;
  class BranchIDListHelper;
  class ProductProvenanceRetriever;
  struct DaqProvenanceHelper;
  class DuplicateChecker;
  class EventSkipperByID;
  class ProcessHistoryRegistry;
  class ProductSelectorRules;
  class InputFile;
  class ProvenanceReaderBase;
  class ProvenanceAdaptor;
  class StoredMergeableRunProductMetadata;
  class RunHelperBase;
  class ThinnedAssociationsHelper;

  using EntryDescriptionMap = std::map<EntryDescriptionID, EventEntryDescription>;

  class MakeProvenanceReader {
  public:
    virtual std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
                                                             DaqProvenanceHelper const* daqProvenanceHelper) const = 0;
    virtual ~MakeProvenanceReader() = default;
  };

  class RootFile {
  public:
    // Constructor used by RootPrimaryFileSequence
    RootFile(std::string const& fileName,
             ProcessConfiguration const& processConfiguration,
             std::string const& logicalFileName,
             std::shared_ptr<InputFile> filePtr,
             std::shared_ptr<EventSkipperByID> eventSkipperByID,
             bool skipAnyEvents,
             int remainingEvents,
             int remainingLumis,
             unsigned int nStreams,
             unsigned int treeCacheSize,
             int treeMaxVirtualSize,
             InputSource::ProcessingMode processingMode,
             RunHelperBase* runHelper,
             bool noRunLumiSort,
             bool noEventSort,
             ProductSelectorRules const& productSelectorRules,
             InputType inputType,
             std::shared_ptr<BranchIDListHelper> branchIDListHelper,
             ProcessBlockHelper*,
             std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper,
             std::vector<BranchID> const* associationsFromSecondary,
             std::shared_ptr<DuplicateChecker> duplicateChecker,
             bool dropDescendantsOfDroppedProducts,
             ProcessHistoryRegistry& processHistoryRegistry,
             std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
             std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
             std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
             bool bypassVersionCheck,
             bool labelRawDataLikeMC,
             bool usingGoToEvent,
             bool enablePrefetching,
             bool enforceGUIDInFileName);

    // Constructor used by RootSecondaryFileSequence
    RootFile(std::string const& fileName,
             ProcessConfiguration const& processConfiguration,
             std::string const& logicalFileName,
             std::shared_ptr<InputFile> filePtr,
             unsigned int nStreams,
             int treeMaxVirtualSize,
             InputSource::ProcessingMode processingMode,
             RunHelperBase* runHelper,
             ProductSelectorRules const& productSelectorRules,
             InputType inputType,
             std::shared_ptr<BranchIDListHelper> branchIDListHelper,
             std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper,
             std::vector<BranchID> const* associationsFromSecondary,
             bool dropDescendantsOfDroppedProducts,
             ProcessHistoryRegistry& processHistoryRegistry,
             std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
             std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
             std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
             bool bypassVersionCheck,
             bool labelRawDataLikeMC,
             bool enablePrefetching,
             bool enforceGUIDInFileName)
        : RootFile(fileName,
                   processConfiguration,
                   logicalFileName,
                   filePtr,
                   nullptr,
                   false,
                   -1,
                   -1,
                   nStreams,
                   0U,
                   treeMaxVirtualSize,
                   processingMode,
                   runHelper,
                   false,
                   false,
                   productSelectorRules,
                   inputType,
                   branchIDListHelper,
                   nullptr,
                   thinnedAssociationsHelper,
                   associationsFromSecondary,
                   nullptr,
                   dropDescendantsOfDroppedProducts,
                   processHistoryRegistry,
                   indexesIntoFiles,
                   currentIndexIntoFile,
                   orderedProcessHistoryIDs,
                   bypassVersionCheck,
                   labelRawDataLikeMC,
                   false,
                   enablePrefetching,
                   enforceGUIDInFileName) {}

    // Constructor used by RootEmbeddedFileSequence
    RootFile(std::string const& fileName,
             ProcessConfiguration const& processConfiguration,
             std::string const& logicalFileName,
             std::shared_ptr<InputFile> filePtr,
             unsigned int nStreams,
             unsigned int treeCacheSize,
             int treeMaxVirtualSize,
             RunHelperBase* runHelper,
             ProductSelectorRules const& productSelectorRules,
             InputType inputType,
             ProcessHistoryRegistry& processHistoryRegistry,
             std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
             std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
             std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
             bool bypassVersionCheck,
             bool enablePrefetching,
             bool enforceGUIDInFileName)
        : RootFile(fileName,
                   processConfiguration,
                   logicalFileName,
                   filePtr,
                   nullptr,
                   false,
                   -1,
                   -1,
                   nStreams,
                   treeCacheSize,
                   treeMaxVirtualSize,
                   InputSource::RunsLumisAndEvents,
                   runHelper,
                   false,
                   false,
                   productSelectorRules,
                   inputType,
                   nullptr,
                   nullptr,
                   nullptr,
                   nullptr,
                   nullptr,
                   false,
                   processHistoryRegistry,
                   indexesIntoFiles,
                   currentIndexIntoFile,
                   orderedProcessHistoryIDs,
                   bypassVersionCheck,
                   false,
                   false,
                   enablePrefetching,
                   enforceGUIDInFileName) {}

    ~RootFile();

    RootFile(RootFile const&) = delete;             // Disallow copying and moving
    RootFile& operator=(RootFile const&) = delete;  // Disallow copying and moving

    void reportOpened(std::string const& inputType);
    void close();
    std::tuple<bool, bool> readCurrentEvent(EventPrincipal& cache, bool assertOnFailure = true);
    bool readEvent(EventPrincipal& cache);

    std::shared_ptr<LuminosityBlockAuxiliary> readLuminosityBlockAuxiliary_();
    std::shared_ptr<RunAuxiliary> readRunAuxiliary_();
    std::shared_ptr<RunAuxiliary> readFakeRunAuxiliary_();

    void fillProcessBlockHelper_();
    bool initializeFirstProcessBlockEntry();
    bool endOfProcessBlocksReached() const;
    bool nextProcessBlock_(ProcessBlockPrincipal&);
    void readProcessBlock_(ProcessBlockPrincipal&);

    void readRun_(RunPrincipal& runPrincipal);
    void readFakeRun_(RunPrincipal& runPrincipal);
    void readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal);
    std::string const& file() const { return file_; }
    std::shared_ptr<ProductRegistry const> productRegistry() const { return productRegistry_; }
    // IndexIntoFile::EntryNumber_t const& entryNumber() const {return indexIntoFileIter().entry();}
    // LuminosityBlockNumber_t const& luminosityBlockNumber() const {return indexIntoFileIter().lumi();}
    // RunNumber_t const& runNumber() const {return indexIntoFileIter().run();}
    RootTree const& eventTree() const { return eventTree_; }
    RootTree const& lumiTree() const { return lumiTree_; }
    RootTree const& runTree() const { return runTree_; }
    FileFormatVersion fileFormatVersion() const { return fileFormatVersion_; }
    int whyNotFastClonable() const { return whyNotFastClonable_; }
    std::array<bool, NumBranchTypes> const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; }
    bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; }
    bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; }
    std::shared_ptr<FileBlock> createFileBlock();
    void updateFileBlock(FileBlock&);

    bool setEntryAtItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) {
      return (event != 0) ? setEntryAtEvent(run, lumi, event) : (lumi ? setEntryAtLumi(run, lumi) : setEntryAtRun(run));
    }
    bool containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const;
    bool setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event);
    bool setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi);
    bool setEntryAtRun(RunNumber_t run);
    bool setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi);
    void setAtEventEntry(IndexIntoFile::EntryNumber_t entry);

    void rewind() {
      indexIntoFileIter_ = indexIntoFileBegin_;
      eventTree_.rewind();
      lumiTree_.rewind();
      runTree_.rewind();
      currentProcessBlockTree_ = 0;
      for (auto& processBlockTree : processBlockTrees_) {
        processBlockTree->rewindToInvalid();
      }
    }
    void setToLastEntry() { indexIntoFileIter_ = indexIntoFileEnd_; }

    bool skipEntries(unsigned int& offset) { return eventTree_.skipEntries(offset); }
    bool skipEvents(int& offset);
    bool goToEvent(EventID const& eventID);
    bool nextEventEntry() { return eventTree_.nextWithCache(); }
    IndexIntoFile::EntryType getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event);
    std::shared_ptr<BranchIDListHelper const> branchIDListHelper() const {
      return get_underlying_safe(branchIDListHelper_);
    }
    std::shared_ptr<BranchIDListHelper>& branchIDListHelper() { return get_underlying_safe(branchIDListHelper_); }
    std::shared_ptr<IndexIntoFile const> indexIntoFileSharedPtr() const {
      return get_underlying_safe(indexIntoFileSharedPtr_);
    }
    std::shared_ptr<IndexIntoFile>& indexIntoFileSharedPtr() { return get_underlying_safe(indexIntoFileSharedPtr_); }
    bool wasLastEventJustRead() const;
    bool wasFirstEventJustRead() const;
    IndexIntoFile::IndexIntoFileItr indexIntoFileIter() const;
    void setPosition(IndexIntoFile::IndexIntoFileItr const& position);
    void initAssociationsFromSecondary(std::vector<BranchID> const&);

    void setSignals(
        signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* preEventReadSource,
        signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* postEventReadSource);

  private:
    void makeProcessBlockRootTrees(std::shared_ptr<InputFile> filePtr,
                                   int treeMaxVirtualSize,
                                   bool enablePrefetching,
                                   InputType inputType,
                                   StoredProcessBlockHelper const& storedProcessBlockHelper);
    bool skipThisEntry();
    void setIfFastClonable(int remainingEvents, int remainingLumis);
    void validateFile(InputType inputType, bool usingGoToEvent);
    void fillIndexIntoFile();
    EventAuxiliary fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry);
    EventAuxiliary const& fillThisEventAuxiliary();
    void fillEventToProcessBlockIndexes();
    bool fillEventHistory(EventAuxiliary& evtAux,
                          EventSelectionIDVector& eventSelectionIDs,
                          BranchListIndexes& branchListIndexes,
                          bool assertOnFailure = true);
    std::shared_ptr<LuminosityBlockAuxiliary> fillLumiAuxiliary();
    std::shared_ptr<RunAuxiliary> fillRunAuxiliary();
    std::string const& newBranchToOldBranch(std::string const& newBranch) const;
    void setPresenceInProductRegistry(ProductRegistry&, StoredProcessBlockHelper const&);
    void markBranchToBeDropped(bool dropDescendants,
                               BranchDescription const& branch,
                               std::set<BranchID>& branchesToDrop,
                               std::map<BranchID, BranchID> const& droppedToKeptAlias) const;
    void dropOnInputAndReorder(ProductRegistry&,
                               ProductSelectorRules const&,
                               bool dropDescendants,
                               InputType,
                               StoredProcessBlockHelper&,
                               ProcessBlockHelper const*);
    void dropProcessesAndReorder(StoredProcessBlockHelper&,
                                 std::set<std::string> const& processesWithKeptProcessBlockProducts,
                                 ProcessBlockHelper const*);

    void readParentageTree(InputType inputType);
    void readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap,
                                  InputType inputType);  // backward compatibility
    void readEventHistoryTree();
    bool isDuplicateEvent();

    void initializeDuplicateChecker(std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
                                    std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile);

    std::unique_ptr<MakeProvenanceReader> makeProvenanceReaderMaker(InputType inputType);
    std::shared_ptr<ProductProvenanceRetriever> makeProductProvenanceRetriever(unsigned int iStreamIndex);

    std::shared_ptr<RunAuxiliary const> savedRunAuxiliary() const { return get_underlying_safe(savedRunAuxiliary_); }
    std::shared_ptr<RunAuxiliary>& savedRunAuxiliary() { return get_underlying_safe(savedRunAuxiliary_); }

    std::shared_ptr<BranchChildren const> branchChildren() const { return get_underlying_safe(branchChildren_); }
    std::shared_ptr<BranchChildren>& branchChildren() { return get_underlying_safe(branchChildren_); }

    std::shared_ptr<ProductProvenanceRetriever const> eventProductProvenanceRetriever(size_t index) const {
      return get_underlying_safe(eventProductProvenanceRetrievers_[index]);
    }
    std::shared_ptr<ProductProvenanceRetriever>& eventProductProvenanceRetriever(size_t index) {
      return get_underlying_safe(eventProductProvenanceRetrievers_[index]);
    }

    std::string const file_;
    std::string const logicalFile_;
    ProcessConfiguration const& processConfiguration_;
    edm::propagate_const<ProcessHistoryRegistry*> processHistoryRegistry_;  // We don't own this
    edm::propagate_const<std::shared_ptr<InputFile>> filePtr_;
    edm::propagate_const<std::shared_ptr<EventSkipperByID>> eventSkipperByID_;
    FileFormatVersion fileFormatVersion_;
    FileID fid_;
    edm::propagate_const<std::shared_ptr<IndexIntoFile>> indexIntoFileSharedPtr_;
    IndexIntoFile& indexIntoFile_;
    std::vector<ProcessHistoryID>& orderedProcessHistoryIDs_;
    IndexIntoFile::IndexIntoFileItr indexIntoFileBegin_;
    IndexIntoFile::IndexIntoFileItr indexIntoFileEnd_;
    IndexIntoFile::IndexIntoFileItr indexIntoFileIter_;
    edm::propagate_const<std::unique_ptr<StoredMergeableRunProductMetadata>> storedMergeableRunProductMetadata_;
    std::vector<EventProcessHistoryID> eventProcessHistoryIDs_;                   // backward compatibility
    std::vector<EventProcessHistoryID>::const_iterator eventProcessHistoryIter_;  // backward compatibility
    edm::propagate_const<std::shared_ptr<RunAuxiliary>> savedRunAuxiliary_;
    bool skipAnyEvents_;
    bool noRunLumiSort_;
    bool noEventSort_;
    bool enforceGUIDInFileName_;
    int whyNotFastClonable_;
    std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
    bool branchListIndexesUnchanged_;
    EventAuxiliary eventAuxCache_;  //Should only be used by fillThisEventAuxiliary()
    RootTree eventTree_;
    RootTree lumiTree_;
    RootTree runTree_;
    std::vector<edm::propagate_const<std::unique_ptr<RootTree>>> processBlockTrees_;
    unsigned int currentProcessBlockTree_ = 0;
    std::vector<edm::propagate_const<RootTree*>> treePointers_;
    //Should only be used by fillThisEventAuxiliary()
    IndexIntoFile::EntryNumber_t lastEventEntryNumberRead_;
    std::shared_ptr<ProductRegistry const> productRegistry_;
    std::shared_ptr<BranchIDLists const> branchIDLists_;
    edm::propagate_const<std::shared_ptr<BranchIDListHelper>> branchIDListHelper_;
    edm::propagate_const<ProcessBlockHelper*> processBlockHelper_;
    edm::propagate_const<std::unique_ptr<StoredProcessBlockHelper>> storedProcessBlockHelper_;
    edm::propagate_const<std::unique_ptr<ThinnedAssociationsHelper>> fileThinnedAssociationsHelper_;
    edm::propagate_const<std::shared_ptr<ThinnedAssociationsHelper>> thinnedAssociationsHelper_;
    InputSource::ProcessingMode processingMode_;
    edm::propagate_const<RunHelperBase*> runHelper_;
    std::map<std::string, std::string> newBranchToOldBranch_;
    edm::propagate_const<TTree*> eventHistoryTree_;  // backward compatibility
    EventToProcessBlockIndexes eventToProcessBlockIndexes_;
    edm::propagate_const<TBranch*> eventToProcessBlockIndexesBranch_;
    edm::propagate_const<std::unique_ptr<History>> history_;  // backward compatibility
    edm::propagate_const<std::shared_ptr<BranchChildren>> branchChildren_;
    edm::propagate_const<std::shared_ptr<DuplicateChecker>> duplicateChecker_;
    edm::propagate_const<std::unique_ptr<ProvenanceAdaptor>> provenanceAdaptor_;  // backward comatibility
    edm::propagate_const<std::unique_ptr<MakeProvenanceReader>> provenanceReaderMaker_;
    std::vector<edm::propagate_const<std::shared_ptr<ProductProvenanceRetriever>>> eventProductProvenanceRetrievers_;
    std::vector<ParentageID> parentageIDLookup_;
    edm::propagate_const<std::unique_ptr<DaqProvenanceHelper>> daqProvenanceHelper_;
    edm::propagate_const<TClass*> edProductClass_;
    InputType inputType_;
  };  // class RootFile

}  // namespace edm
#endif