CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
RootFile.cc
Go to the documentation of this file.
1 /*----------------------------------------------------------------------
2 ----------------------------------------------------------------------*/
3 
4 #include "RootFile.h"
5 #include "DuplicateChecker.h"
6 #include "InputFile.h"
7 #include "ProvenanceAdaptor.h"
8 #include "RunHelper.h"
9 
49 
52 
53 //used for backward compatibility
58 
59 #include "Rtypes.h"
60 #include "TClass.h"
61 #include "TString.h"
62 #include "TTree.h"
63 #include "TTreeCache.h"
64 
65 #include <algorithm>
66 #include <list>
67 
68 namespace edm {
69 
70  // Algorithm classes for making ProvenanceReader:
72  public:
73  std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
74  DaqProvenanceHelper const* daqProvenanceHelper) const override;
75  };
77  public:
78  MakeOldProvenanceReader(std::unique_ptr<EntryDescriptionMap>&& entryDescriptionMap)
79  : MakeProvenanceReader(), entryDescriptionMap_(std::move(entryDescriptionMap)) {}
80  std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
81  DaqProvenanceHelper const* daqProvenanceHelper) const override;
82 
83  private:
85  };
87  public:
88  std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
89  DaqProvenanceHelper const* daqProvenanceHelper) const override;
90  };
92  public:
93  MakeReducedProvenanceReader(std::vector<ParentageID> const& parentageIDLookup)
94  : parentageIDLookup_(parentageIDLookup) {}
95  std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
96  DaqProvenanceHelper const* daqProvenanceHelper) const override;
97 
98  private:
99  std::vector<ParentageID> const& parentageIDLookup_;
100  };
101 
102  namespace {
103  void checkReleaseVersion(std::vector<ProcessHistory> processHistoryVector, std::string const& fileName) {
104  std::string releaseVersion = getReleaseVersion();
105  releaseversion::DecomposedReleaseVersion currentRelease(releaseVersion);
106  for (auto const& ph : processHistoryVector) {
107  for (auto const& pc : ph) {
108  if (releaseversion::isEarlierRelease(currentRelease, pc.releaseVersion())) {
110  << "The release you are using, " << getReleaseVersion() << " , predates\n"
111  << "a release (" << pc.releaseVersion() << ") used in writing the input file, " << fileName << ".\n"
112  << "Forward compatibility cannot be supported.\n";
113  }
114  }
115  }
116  }
117  } // namespace
118 
119  // This is a helper class for IndexIntoFile.
121  public:
122  explicit RootFileEventFinder(RootTree& eventTree) : eventTree_(eventTree) {}
123  ~RootFileEventFinder() override {}
124 
126  roottree::EntryNumber saveEntry = eventTree_.entryNumber();
127  eventTree_.setEntryNumber(entry);
128  EventAuxiliary eventAux;
129  EventAuxiliary* pEvAux = &eventAux;
130  eventTree_.fillAux<EventAuxiliary>(pEvAux);
131  eventTree_.setEntryNumber(saveEntry);
132  return eventAux.event();
133  }
134 
135  private:
137  };
138 
139  //---------------------------------------------------------------------
141  ProcessConfiguration const& processConfiguration,
142  std::string const& logicalFileName,
143  std::shared_ptr<InputFile> filePtr,
144  std::shared_ptr<EventSkipperByID> eventSkipperByID,
145  bool skipAnyEvents,
146  int remainingEvents,
147  int remainingLumis,
148  unsigned int nStreams,
149  unsigned int treeCacheSize,
150  int treeMaxVirtualSize,
151  InputSource::ProcessingMode processingMode,
152  RunHelperBase* runHelper,
153  bool noEventSort,
154  ProductSelectorRules const& productSelectorRules,
155  InputType inputType,
156  std::shared_ptr<BranchIDListHelper> branchIDListHelper,
157  std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper,
158  std::vector<BranchID> const* associationsFromSecondary,
159  std::shared_ptr<DuplicateChecker> duplicateChecker,
160  bool dropDescendants,
161  ProcessHistoryRegistry& processHistoryRegistry,
162  std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
163  std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
164  std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
165  bool bypassVersionCheck,
166  bool labelRawDataLikeMC,
167  bool usingGoToEvent,
168  bool enablePrefetching,
169  bool enforceGUIDInFileName)
170  : file_(fileName),
171  logicalFile_(logicalFileName),
172  processConfiguration_(processConfiguration),
173  processHistoryRegistry_(&processHistoryRegistry),
174  filePtr_(filePtr),
175  eventSkipperByID_(eventSkipperByID),
176  fileFormatVersion_(),
177  fid_(),
178  indexIntoFileSharedPtr_(new IndexIntoFile),
179  indexIntoFile_(*indexIntoFileSharedPtr_),
180  orderedProcessHistoryIDs_(orderedProcessHistoryIDs),
181  indexIntoFileBegin_(
182  indexIntoFile_.begin(noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder)),
183  indexIntoFileEnd_(indexIntoFileBegin_),
184  indexIntoFileIter_(indexIntoFileBegin_),
185  storedMergeableRunProductMetadata_((inputType == InputType::Primary) ? new StoredMergeableRunProductMetadata
186  : nullptr),
187  eventProcessHistoryIDs_(),
188  eventProcessHistoryIter_(eventProcessHistoryIDs_.begin()),
189  savedRunAuxiliary_(),
190  skipAnyEvents_(skipAnyEvents),
191  noEventSort_(noEventSort),
192  enforceGUIDInFileName_(enforceGUIDInFileName),
193  whyNotFastClonable_(0),
194  hasNewlyDroppedBranch_(),
195  branchListIndexesUnchanged_(false),
196  eventAux_(),
197  eventTree_(filePtr,
198  InEvent,
199  nStreams,
200  treeMaxVirtualSize,
201  treeCacheSize,
202  roottree::defaultLearningEntries,
203  enablePrefetching,
204  inputType),
205  lumiTree_(filePtr,
206  InLumi,
207  1,
208  treeMaxVirtualSize,
209  roottree::defaultNonEventCacheSize,
211  enablePrefetching,
212  inputType),
213  runTree_(filePtr,
214  InRun,
215  1,
216  treeMaxVirtualSize,
217  roottree::defaultNonEventCacheSize,
219  enablePrefetching,
220  inputType),
221  treePointers_(),
222  lastEventEntryNumberRead_(IndexIntoFile::invalidEntry),
223  productRegistry_(),
224  branchIDLists_(),
225  branchIDListHelper_(branchIDListHelper),
226  fileThinnedAssociationsHelper_(),
227  thinnedAssociationsHelper_(thinnedAssociationsHelper),
228  processingMode_(processingMode),
229  runHelper_(runHelper),
230  newBranchToOldBranch_(),
231  eventHistoryTree_(nullptr),
232  eventSelectionIDs_(),
233  branchListIndexes_(),
234  history_(),
235  branchChildren_(new BranchChildren),
236  duplicateChecker_(duplicateChecker),
237  provenanceAdaptor_(),
238  provenanceReaderMaker_(),
239  eventProductProvenanceRetrievers_(),
240  parentageIDLookup_(),
241  daqProvenanceHelper_(),
242  edProductClass_(TypeWithDict::byName("edm::WrapperBase").getClass()),
243  inputType_(inputType) {
244  hasNewlyDroppedBranch_.fill(false);
245 
249 
250  // Read the metadata tree.
251  // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file.
252  std::unique_ptr<TTree> metaDataTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::metaDataTreeName().c_str())));
253  if (nullptr == metaDataTree.get()) {
255  << "Could not find tree " << poolNames::metaDataTreeName() << " in the input file.\n";
256  }
257 
258  // To keep things simple, we just read in every possible branch that exists.
259  // We don't pay attention to which branches exist in which file format versions
260 
262  if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) {
263  TBranch* fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str());
264  fft->SetAddress(&fftPtr);
265  roottree::getEntry(fft, 0);
266  metaDataTree->SetBranchAddress(poolNames::fileFormatVersionBranchName().c_str(), &fftPtr);
267  }
268 
269  FileID* fidPtr = &fid_;
270  if (metaDataTree->FindBranch(poolNames::fileIdentifierBranchName().c_str()) != nullptr) {
271  metaDataTree->SetBranchAddress(poolNames::fileIdentifierBranchName().c_str(), &fidPtr);
272  }
273 
274  IndexIntoFile* iifPtr = &indexIntoFile_;
275  if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) {
276  metaDataTree->SetBranchAddress(poolNames::indexIntoFileBranchName().c_str(), &iifPtr);
277  }
278 
279  StoredMergeableRunProductMetadata* smrc = nullptr;
280  if (inputType == InputType::Primary) {
282  if (metaDataTree->FindBranch(poolNames::mergeableRunProductMetadataBranchName().c_str()) != nullptr) {
283  metaDataTree->SetBranchAddress(poolNames::mergeableRunProductMetadataBranchName().c_str(), &smrc);
284  }
285  }
286 
287  // Need to read to a temporary registry so we can do a translation of the BranchKeys.
288  // This preserves backward compatibility against friendly class name algorithm changes.
289  ProductRegistry inputProdDescReg;
290  ProductRegistry* ppReg = &inputProdDescReg;
291  metaDataTree->SetBranchAddress(poolNames::productDescriptionBranchName().c_str(), (&ppReg));
292 
293  typedef std::map<ParameterSetID, ParameterSetBlob> PsetMap;
294  PsetMap psetMap;
295  PsetMap* psetMapPtr = &psetMap;
296  if (metaDataTree->FindBranch(poolNames::parameterSetMapBranchName().c_str()) != nullptr) {
297  //backward compatibility
298  assert(!fileFormatVersion().parameterSetsTree());
299  metaDataTree->SetBranchAddress(poolNames::parameterSetMapBranchName().c_str(), &psetMapPtr);
300  } else {
301  assert(fileFormatVersion().parameterSetsTree());
302  // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file.
303  std::unique_ptr<TTree> psetTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::parameterSetsTreeName().c_str())));
304  if (nullptr == psetTree.get()) {
306  << "Could not find tree " << poolNames::parameterSetsTreeName() << " in the input file.\n";
307  }
308 
309  typedef std::pair<ParameterSetID, ParameterSetBlob> IdToBlobs;
310  IdToBlobs idToBlob;
311  IdToBlobs* pIdToBlob = &idToBlob;
312  psetTree->SetBranchAddress(poolNames::idToParameterSetBlobsBranchName().c_str(), &pIdToBlob);
313 
314  std::unique_ptr<TTreeCache> psetTreeCache =
316  psetTreeCache->SetEnablePrefetching(false);
317  filePtr_->SetCacheRead(psetTreeCache.get());
318  for (Long64_t i = 0; i != psetTree->GetEntries(); ++i) {
319  psetTree->GetEntry(i);
320  psetMap.insert(idToBlob);
321  }
322  filePtr_->SetCacheRead(nullptr);
323  }
324 
325  // backward compatibility
327  ProcessHistoryRegistry::collection_type* pHistMapPtr = &pHistMap;
328  if (metaDataTree->FindBranch(poolNames::processHistoryMapBranchName().c_str()) != nullptr) {
329  metaDataTree->SetBranchAddress(poolNames::processHistoryMapBranchName().c_str(), &pHistMapPtr);
330  }
331 
333  ProcessHistoryRegistry::vector_type* pHistVectorPtr = &pHistVector;
334  if (metaDataTree->FindBranch(poolNames::processHistoryBranchName().c_str()) != nullptr) {
335  metaDataTree->SetBranchAddress(poolNames::processHistoryBranchName().c_str(), &pHistVectorPtr);
336  }
337 
338  // backward compatibility
339  ProcessConfigurationVector processConfigurations;
340  ProcessConfigurationVector* procConfigVectorPtr = &processConfigurations;
341  if (metaDataTree->FindBranch(poolNames::processConfigurationBranchName().c_str()) != nullptr) {
342  metaDataTree->SetBranchAddress(poolNames::processConfigurationBranchName().c_str(), &procConfigVectorPtr);
343  }
344 
345  auto branchIDListsAPtr = std::make_unique<BranchIDLists>();
346  BranchIDLists* branchIDListsPtr = branchIDListsAPtr.get();
347  if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) != nullptr) {
348  metaDataTree->SetBranchAddress(poolNames::branchIDListBranchName().c_str(), &branchIDListsPtr);
349  }
350 
351  if (inputType != InputType::SecondarySource) {
353  std::make_unique<ThinnedAssociationsHelper>(); // propagate_const<T> has no reset() function
354  ThinnedAssociationsHelper* thinnedAssociationsHelperPtr = fileThinnedAssociationsHelper_.get();
355  if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) != nullptr) {
356  metaDataTree->SetBranchAddress(poolNames::thinnedAssociationsHelperBranchName().c_str(),
357  &thinnedAssociationsHelperPtr);
358  }
359  }
360 
361  BranchChildren* branchChildrenBuffer = branchChildren_.get();
362  if (metaDataTree->FindBranch(poolNames::productDependenciesBranchName().c_str()) != nullptr) {
363  metaDataTree->SetBranchAddress(poolNames::productDependenciesBranchName().c_str(), &branchChildrenBuffer);
364  }
365 
366  // backward compatibility
367  std::vector<EventProcessHistoryID>* eventHistoryIDsPtr = &eventProcessHistoryIDs_;
368  if (metaDataTree->FindBranch(poolNames::eventHistoryBranchName().c_str()) != nullptr) {
369  metaDataTree->SetBranchAddress(poolNames::eventHistoryBranchName().c_str(), &eventHistoryIDsPtr);
370  }
371 
372  if (metaDataTree->FindBranch(poolNames::moduleDescriptionMapBranchName().c_str()) != nullptr) {
373  if (metaDataTree->GetBranch(poolNames::moduleDescriptionMapBranchName().c_str())->GetSplitLevel() != 0) {
374  metaDataTree->SetBranchStatus((poolNames::moduleDescriptionMapBranchName() + ".*").c_str(), false);
375  } else {
376  metaDataTree->SetBranchStatus(poolNames::moduleDescriptionMapBranchName().c_str(), false);
377  }
378  }
379 
380  // Here we read the metadata tree
381  roottree::getEntry(metaDataTree.get(), 0);
382 
384 
385  // Here we read the event history tree, if we have one.
387 
389  if (!fileFormatVersion().triggerPathsTracked()) {
390  ParameterSetConverter converter(psetMap, psetIdConverter, fileFormatVersion().parameterSetsByReference());
391  } else {
392  // Merge into the parameter set registry.
393  pset::Registry& psetRegistry = *pset::Registry::instance();
394  try {
395  for (auto const& psetEntry : psetMap) {
396  ParameterSet pset(psetEntry.second.pset());
397  pset.setID(psetEntry.first);
398  // For thread safety, don't update global registries when a secondary source opens a file.
399  if (inputType != InputType::SecondarySource) {
400  psetRegistry.insertMapped(pset);
401  }
402  }
403  } catch (edm::Exception const& iExcept) {
404  if (iExcept.categoryCode() == edm::errors::Configuration) {
406  exception << iExcept.message();
407  exception.addContext("Creating ParameterSets from file");
408  throw exception;
409  } else {
410  throw;
411  }
412  }
413  }
414  if (!fileFormatVersion().splitProductIDs()) {
415  // Old provenance format input file. Create a provenance adaptor.
416  // propagate_const<T> has no reset() function
417  provenanceAdaptor_ = std::make_unique<ProvenanceAdaptor>(
418  inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, true);
419  // Fill in the branchIDLists branch from the provenance adaptor
420  branchIDLists_ = provenanceAdaptor_->branchIDLists();
421  } else {
422  if (!fileFormatVersion().triggerPathsTracked()) {
423  // New provenance format, but change in ParameterSet Format. Create a provenance adaptor.
424  // propagate_const<T> has no reset() function
425  provenanceAdaptor_ = std::make_unique<ProvenanceAdaptor>(
426  inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, false);
427  }
428  // New provenance format input file. The branchIDLists branch was read directly from the input file.
429  if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) == nullptr) {
430  throw Exception(errors::EventCorruption) << "Failed to find branchIDLists branch in metaData tree.\n";
431  }
432  branchIDLists_.reset(branchIDListsAPtr.release());
433  }
434 
436  if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) == nullptr) {
438  << "Failed to find thinnedAssociationsHelper branch in metaData tree.\n";
439  }
440  }
441 
442  if (!bypassVersionCheck) {
443  checkReleaseVersion(pHistVector, file());
444  }
445 
446  if (labelRawDataLikeMC) {
447  std::string const rawData("FEDRawDataCollection");
448  std::string const source("source");
449  ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator();
450  BranchKey finder(rawData, source, "", "");
451  ProductRegistry::ProductList::iterator it = pList.lower_bound(finder);
452  if (it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source) {
453  // We found raw data with a module label of source.
454  // We need to change the module label and process name.
455  // Create helper.
456  it->second.init();
457  // propagate_const<T> has no reset() function
458  daqProvenanceHelper_ = std::make_unique<DaqProvenanceHelper>(it->second.unwrappedTypeID());
459  // Create the new branch description
460  BranchDescription const& newBD = daqProvenanceHelper_->branchDescription();
461  // Save info from the old and new branch descriptions
462  daqProvenanceHelper_->saveInfo(it->second, newBD);
463  // Map the new branch name to the old branch name.
464  newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), it->second.branchName()));
465  // Remove the old branch description from the product Registry.
466  pList.erase(it);
467  // Check that there was only one.
468  it = pList.lower_bound(finder);
469  assert(!(it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source));
470  // Insert the new branch description into the product registry.
471  inputProdDescReg.copyProduct(newBD);
472  // Fix up other per file metadata.
473  daqProvenanceHelper_->fixMetaData(processConfigurations, pHistVector);
474  daqProvenanceHelper_->fixMetaData(*branchIDLists_);
475  daqProvenanceHelper_->fixMetaData(*branchChildren_);
476  }
477  }
478 
479  for (auto const& history : pHistVector) {
480  processHistoryRegistry.registerProcessHistory(history);
481  }
482 
484 
485  // Update the branch id info. This has to be done before validateFile since
486  // depending on the file format, the branchIDListHelper_ may have its fixBranchListIndexes call made
487  if (inputType == InputType::Primary) {
489  }
490 
491  validateFile(inputType, usingGoToEvent);
492 
493  // Here, we make the class that will make the ProvenanceReader
494  // It reads whatever trees it needs.
495  // propagate_const<T> has no reset() function
496  provenanceReaderMaker_ = std::unique_ptr<MakeProvenanceReader>(makeProvenanceReaderMaker(inputType).release());
497 
498  // Merge into the hashed registries.
499  if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
501  }
502 
503  initializeDuplicateChecker(indexesIntoFiles, currentIndexIntoFile);
507  indexIntoFile_.end(noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder);
509  eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin();
510 
511  // Set product presence information in the product registry.
512  ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator();
513  for (auto& product : pList) {
514  BranchDescription& prod = product.second;
515  prod.init();
516  treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName()));
517  }
518 
519  auto newReg = std::make_unique<ProductRegistry>();
520 
521  // Do the translation from the old registry to the new one
522  {
523  ProductRegistry::ProductList const& prodList = inputProdDescReg.productList();
524  for (auto const& product : prodList) {
525  BranchDescription const& prod = product.second;
526  std::string newFriendlyName = friendlyname::friendlyName(prod.className());
527  if (newFriendlyName == prod.friendlyClassName()) {
528  newReg->copyProduct(prod);
529  } else {
530  if (fileFormatVersion().splitProductIDs()) {
532  << "Cannot change friendly class name algorithm without more development work\n"
533  << "to update BranchIDLists and ThinnedAssociationsHelper. Contact the framework group.\n";
534  }
535  BranchDescription newBD(prod);
536  newBD.updateFriendlyClassName();
537  newReg->copyProduct(newBD);
538  newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), prod.branchName()));
539  }
540  }
541  dropOnInput(*newReg, productSelectorRules, dropDescendants, inputType);
542  if (inputType == InputType::SecondaryFile) {
543  thinnedAssociationsHelper->updateFromSecondaryInput(*fileThinnedAssociationsHelper_,
544  *associationsFromSecondary);
545  } else if (inputType == InputType::Primary) {
546  thinnedAssociationsHelper->updateFromPrimaryInput(*fileThinnedAssociationsHelper_);
547  }
548 
549  if (inputType == InputType::Primary) {
550  for (auto& product : newReg->productListUpdator()) {
551  setIsMergeable(product.second);
552  }
553  }
554 
555  // freeze the product registry
556  newReg->setFrozen(inputType != InputType::Primary);
557  productRegistry_.reset(newReg.release());
558  }
559 
560  // Set up information from the product registry.
561  ProductRegistry::ProductList const& prodList = productRegistry()->productList();
562 
563  {
564  std::array<size_t, NumBranchTypes> nBranches;
565  nBranches.fill(0);
566  for (auto const& product : prodList) {
567  ++nBranches[product.second.branchType()];
568  }
569 
570  int i = 0;
571  for (auto t : treePointers_) {
572  t->numberOfBranchesToAdd(nBranches[i]);
573  ++i;
574  }
575  }
576  for (auto const& product : prodList) {
577  BranchDescription const& prod = product.second;
578  treePointers_[prod.branchType()]->addBranch(prod, newBranchToOldBranch(prod.branchName()));
579  }
580 
581  // Determine if this file is fast clonable.
582  setIfFastClonable(remainingEvents, remainingLumis);
583 
584  // We are done with our initial reading of EventAuxiliary.
586 
587  // Tell the event tree to begin training at the next read.
589 
590  // Train the run and lumi trees.
591  runTree_.trainCache("*");
592  lumiTree_.trainCache("*");
593  }
594 
596 
597  void RootFile::readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType) {
598  // Called only for old format files.
599  // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file.
600  std::unique_ptr<TTree> entryDescriptionTree(
601  dynamic_cast<TTree*>(filePtr_->Get(poolNames::entryDescriptionTreeName().c_str())));
602  if (nullptr == entryDescriptionTree.get()) {
604  << "Could not find tree " << poolNames::entryDescriptionTreeName() << " in the input file.\n";
605  }
606 
607  EntryDescriptionID idBuffer;
608  EntryDescriptionID* pidBuffer = &idBuffer;
609  entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), &pidBuffer);
610 
611  EventEntryDescription entryDescriptionBuffer;
612  EventEntryDescription* pEntryDescriptionBuffer = &entryDescriptionBuffer;
613  entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), &pEntryDescriptionBuffer);
614 
615  // Fill in the parentage registry.
617 
618  for (Long64_t i = 0, numEntries = entryDescriptionTree->GetEntries(); i < numEntries; ++i) {
619  roottree::getEntry(entryDescriptionTree.get(), i);
620  if (idBuffer != entryDescriptionBuffer.id()) {
621  throw Exception(errors::EventCorruption) << "Corruption of EntryDescription tree detected.\n";
622  }
623  entryDescriptionMap.insert(std::make_pair(entryDescriptionBuffer.id(), entryDescriptionBuffer));
625  parents.setParents(entryDescriptionBuffer.parents());
626  if (daqProvenanceHelper_) {
627  ParentageID const oldID = parents.id();
628  daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate());
629  ParentageID newID = parents.id();
630  if (newID != oldID) {
631  daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID);
632  }
633  }
634  // For thread safety, don't update global registries when a secondary source opens a file.
635  if (inputType != InputType::SecondarySource) {
636  registry.insertMapped(parents);
637  }
638  }
639  entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), nullptr);
640  entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), nullptr);
641  }
642 
644  // New format file
645  // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file.
646  std::unique_ptr<TTree> parentageTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::parentageTreeName().c_str())));
647  if (nullptr == parentageTree.get()) {
649  << "Could not find tree " << poolNames::parentageTreeName() << " in the input file.\n";
650  }
651 
653  Parentage* pParentageBuffer = &parents;
654  parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), &pParentageBuffer);
655 
657 
658  parentageIDLookup_.reserve(parentageTree->GetEntries());
659  for (Long64_t i = 0, numEntries = parentageTree->GetEntries(); i < numEntries; ++i) {
660  roottree::getEntry(parentageTree.get(), i);
661  if (daqProvenanceHelper_) {
662  ParentageID const oldID = parents.id();
663  daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate());
664  ParentageID newID = parents.id();
665  if (newID != oldID) {
666  daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID);
667  }
668  }
669  // For thread safety, don't update global registries when a secondary source opens a file.
670  if (inputType != InputType::SecondarySource) {
671  registry.insertMapped(parents);
672  }
673  parentageIDLookup_.push_back(parents.id());
674  }
675  parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), nullptr);
676  }
677 
678  void RootFile::setIfFastClonable(int remainingEvents, int remainingLumis) {
679  if (fileFormatVersion().noMetaDataTrees() and !fileFormatVersion().storedProductProvenanceUsed()) {
680  //we must avoid copying the old branch which stored the per product per event provenance
682  return;
683  }
684  if (!fileFormatVersion().splitProductIDs()) {
686  return;
687  }
690  return;
691  }
692  // Find entry for first event in file
694  while (it != indexIntoFileEnd_ && it.getEntryType() != IndexIntoFile::kEvent) {
695  ++it;
696  }
697  if (it == indexIntoFileEnd_) {
699  return;
700  }
701 
702  // From here on, record all reasons we can't fast clone.
703  IndexIntoFile::SortOrder sortOrder =
707  }
708  if (skipAnyEvents_) {
710  }
711  if (remainingEvents >= 0 && eventTree_.entries() > remainingEvents) {
713  }
714  if (remainingLumis >= 0 && lumiTree_.entries() > remainingLumis) {
716  }
717  if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) {
719  }
720  }
721 
722  std::unique_ptr<FileBlock> RootFile::createFileBlock() const {
723  return std::make_unique<FileBlock>(fileFormatVersion(),
724  eventTree_.tree(),
726  lumiTree_.tree(),
728  runTree_.tree(),
729  runTree_.metaTree(),
732  file_,
734  modifiedIDs(),
735  branchChildren());
736  }
737 
738  std::string const& RootFile::newBranchToOldBranch(std::string const& newBranch) const {
739  std::map<std::string, std::string>::const_iterator it = newBranchToOldBranch_.find(newBranch);
740  if (it != newBranchToOldBranch_.end()) {
741  return it->second;
742  }
743  return newBranch;
744  }
745 
747 
750  }
751 
752  void RootFile::initAssociationsFromSecondary(std::vector<BranchID> const& associationsFromSecondary) {
753  thinnedAssociationsHelper_->initAssociationsFromSecondary(associationsFromSecondary,
755  }
756 
759  return false;
760  }
761  if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
762  // See first if the entire lumi or run is skipped, so we won't have to read the event Auxiliary in that case.
764  return true;
765  }
766 
767  // The Lumi is not skipped. If this is an event, see if the event is skipped.
771  return true;
772  }
773  }
774 
775  // Skip runs with no lumis if either lumisToSkip or lumisToProcess have been set to select lumis
778 
779  // There are no lumis in this run, not even ones we will skip
780  if (iterLumi.peekAheadAtLumi() == IndexIntoFile::invalidLumi) {
781  return true;
782  }
783  // If we get here there are lumis in the run, check to see if we are skipping all of them
784  do {
785  if (!eventSkipperByID_->skipIt(iterLumi.run(), iterLumi.peekAheadAtLumi(), 0U)) {
786  return false;
787  }
788  } while (iterLumi.skipLumiInRun());
789  return true;
790  }
791  }
792  return false;
793  }
794 
797  if (duplicateChecker_.get() == nullptr) {
798  return false;
799  }
801  return duplicateChecker_->isDuplicateAndCheckActive(indexIntoFileIter_.processHistoryIDIndex(),
804  eventAux_.id().event(),
805  file_);
806  }
807 
809  return indexIntoFile_.containsItem(run, lumi, event);
810  }
811 
814  EventNumber_t& event) {
815  // First, account for consecutive skipped entries.
816  while (skipThisEntry()) {
821  } else {
823  }
824  }
825  // OK, we have an entry that is not skipped.
827  if (entryType == IndexIntoFile::kEnd) {
828  return IndexIntoFile::kEnd;
829  }
830  if (entryType == IndexIntoFile::kRun) {
831  run = indexIntoFileIter_.run();
832  runHelper_->checkForNewRun(run);
833  return IndexIntoFile::kRun;
834  } else if (processingMode_ == InputSource::Runs) {
836  return getNextItemType(run, lumi, event);
837  }
838  if (entryType == IndexIntoFile::kLumi) {
839  run = indexIntoFileIter_.run();
840  lumi = indexIntoFileIter_.lumi();
841  return IndexIntoFile::kLumi;
844  return getNextItemType(run, lumi, event);
845  }
846  if (isDuplicateEvent()) {
848  return getNextItemType(run, lumi, event);
849  }
850  run = indexIntoFileIter_.run();
851  lumi = indexIntoFileIter_.lumi();
853  event = eventAux_.event();
854  return IndexIntoFile::kEvent;
855  }
856 
859  itr.advanceToEvent();
860  return itr.getEntryType() == IndexIntoFile::kEnd;
861  }
862 
865  int phIndex;
868  IndexIntoFile::EntryNumber_t eventEntry;
869  itr.skipEventBackward(phIndex, run, lumi, eventEntry);
870  itr.skipEventBackward(phIndex, run, lumi, eventEntry);
871  return eventEntry == IndexIntoFile::invalidEntry;
872  }
873 
874  namespace {
875  struct RunItem {
876  RunItem(ProcessHistoryID const& phid, RunNumber_t const& run) : phid_(phid), run_(run) {}
877  ProcessHistoryID phid_;
878  RunNumber_t run_;
879  };
880  struct RunItemSortByRun {
881  bool operator()(RunItem const& a, RunItem const& b) const { return a.run_ < b.run_; }
882  };
883  struct RunItemSortByRunPhid {
884  bool operator()(RunItem const& a, RunItem const& b) const {
885  return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.phid_ < b.phid_);
886  }
887  };
888  struct LumiItem {
889  LumiItem(ProcessHistoryID const& phid,
890  RunNumber_t const& run,
893  : phid_(phid),
894  run_(run),
895  lumi_(lumi),
896  firstEventEntry_(entry),
897  lastEventEntry_(entry == IndexIntoFile::invalidEntry ? IndexIntoFile::invalidEntry : entry + 1) {}
898  ProcessHistoryID phid_;
899  RunNumber_t run_;
901  IndexIntoFile::EntryNumber_t firstEventEntry_;
902  IndexIntoFile::EntryNumber_t lastEventEntry_;
903  };
904  struct LumiItemSortByRunLumi {
905  bool operator()(LumiItem const& a, LumiItem const& b) const {
906  return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.lumi_ < b.lumi_);
907  }
908  };
909  struct LumiItemSortByRunLumiPhid {
910  bool operator()(LumiItem const& a, LumiItem const& b) const {
911  if (a.run_ < b.run_)
912  return true;
913  if (b.run_ < a.run_)
914  return false;
915  if (a.lumi_ < b.lumi_)
916  return true;
917  if (b.lumi_ < a.lumi_)
918  return false;
919  return a.phid_ < b.phid_;
920  }
921  };
922  } // namespace
923 
925  // This function is for backward compatibility.
926  // If reading a current format file, indexIntoFile_ is read from the input
927  // file and should always be there. Note that the algorithm below will work
928  // sometimes but often fail with the new format introduced in release 3_8_0.
929  // If it ever becomes necessary to rebuild IndexIntoFile from the new format,
930  // probably a separate function should be written to deal with the task.
931  // This is possible just not implemented yet.
932  assert(!fileFormatVersion().hasIndexIntoFile());
933 
934  typedef std::list<LumiItem> LumiList;
935  LumiList lumis; // (declare 1)
936 
937  typedef std::set<LuminosityBlockID> RunLumiSet;
938  RunLumiSet runLumiSet; // (declare 2)
939 
940  typedef std::list<RunItem> RunList;
941  RunList runs; // (declare 5)
942 
943  typedef std::set<RunNumber_t> RunSet;
944  RunSet runSet; // (declare 4)
945 
946  typedef std::set<RunItem, RunItemSortByRunPhid> RunItemSet;
947  RunItemSet runItemSet; // (declare 3)
948 
949  typedef std::map<RunNumber_t, ProcessHistoryID> PHIDMap;
950  PHIDMap phidMap;
951 
952  RunNumber_t prevRun = 0;
953  LuminosityBlockNumber_t prevLumi = 0;
954  ProcessHistoryID prevPhid;
955  bool iFirst = true;
956 
957  indexIntoFile_.unsortedEventNumbers().clear(); // should already be empty, just being careful
959 
960  // First, loop through the event tree.
961  while (eventTree_.next()) {
962  bool newRun = false;
963  bool newLumi = false;
966 
967  // Save the event numbers as we loop through the event auxiliary to avoid
968  // having to read through the event auxiliary again later. These event numbers
969  // are not actually used in this function, but could be needed elsewhere.
971 
972  ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(eventAux().processHistoryID());
973 
974  if (iFirst || prevPhid != reducedPHID || prevRun != eventAux().run()) {
975  iFirst = false;
976  newRun = newLumi = true;
977  } else if (prevLumi != eventAux().luminosityBlock()) {
978  newLumi = true;
979  }
980  prevPhid = reducedPHID;
981  prevRun = eventAux().run();
982  prevLumi = eventAux().luminosityBlock();
983  if (newLumi) {
984  lumis.emplace_back(
985  reducedPHID, eventAux().run(), eventAux().luminosityBlock(), eventTree_.entryNumber()); // (insert 1)
986  runLumiSet.insert(LuminosityBlockID(eventAux().run(), eventAux().luminosityBlock())); // (insert 2)
987  } else {
988  LumiItem& currentLumi = lumis.back();
989  assert(currentLumi.lastEventEntry_ == eventTree_.entryNumber());
990  ++currentLumi.lastEventEntry_;
991  }
992  if (newRun) {
993  // Insert run in list if it is not already there.
994  RunItem item(reducedPHID, eventAux().run());
995  if (runItemSet.insert(item).second) { // (check 3, insert 3)
996  runs.push_back(std::move(item)); // (insert 5)
997  runSet.insert(eventAux().run()); // (insert 4)
998  phidMap.insert(std::make_pair(eventAux().run(), reducedPHID));
999  }
1000  }
1001  }
1002  // now clean up.
1006 
1007  // Loop over run entries and fill information.
1008 
1009  typedef std::map<RunNumber_t, IndexIntoFile::EntryNumber_t> RunMap;
1010  RunMap runMap; // (declare 11)
1011 
1012  typedef std::vector<RunItem> RunVector;
1013  RunVector emptyRuns; // (declare 12)
1014 
1015  if (runTree_.isValid()) {
1016  while (runTree_.next()) {
1017  // Note: adjacent duplicates will be skipped without an explicit check.
1018 
1019  std::shared_ptr<RunAuxiliary> runAux = fillRunAuxiliary();
1020  ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(runAux->processHistoryID());
1021 
1022  if (runSet.insert(runAux->run()).second) { // (check 4, insert 4)
1023  // This run was not associated with any events.
1024  emptyRuns.emplace_back(reducedPHID, runAux->run()); // (insert 12)
1025  }
1026  runMap.insert(std::make_pair(runAux->run(), runTree_.entryNumber())); // (insert 11)
1027  phidMap.insert(std::make_pair(runAux->run(), reducedPHID));
1028  }
1029  // now clean up.
1031  }
1032 
1033  // Insert the ordered empty runs into the run list.
1034  RunItemSortByRun runItemSortByRun;
1035  stable_sort_all(emptyRuns, runItemSortByRun);
1036 
1037  RunList::iterator itRuns = runs.begin(), endRuns = runs.end();
1038  for (auto const& emptyRun : emptyRuns) {
1039  for (; itRuns != endRuns; ++itRuns) {
1040  if (runItemSortByRun(emptyRun, *itRuns)) {
1041  break;
1042  }
1043  }
1044  runs.insert(itRuns, emptyRun);
1045  }
1046 
1047  // Loop over luminosity block entries and fill information.
1048 
1049  typedef std::vector<LumiItem> LumiVector;
1050  LumiVector emptyLumis; // (declare 7)
1051 
1052  typedef std::map<LuminosityBlockID, IndexIntoFile::EntryNumber_t> RunLumiMap;
1053  RunLumiMap runLumiMap; // (declare 6)
1054 
1055  if (lumiTree_.isValid()) {
1056  while (lumiTree_.next()) {
1057  // Note: adjacent duplicates will be skipped without an explicit check.
1058  std::shared_ptr<LuminosityBlockAuxiliary> lumiAux = fillLumiAuxiliary();
1059  LuminosityBlockID lumiID = LuminosityBlockID(lumiAux->run(), lumiAux->luminosityBlock());
1060  if (runLumiSet.insert(lumiID).second) { // (check 2, insert 2)
1061  // This lumi was not associated with any events.
1062  // Use the process history ID from the corresponding run. In cases of practical
1063  // importance, this should be the correct process history ID, but it is possible
1064  // to construct files where this is not the correct process history ID ...
1065  PHIDMap::const_iterator iPhidMap = phidMap.find(lumiAux->run());
1066  assert(iPhidMap != phidMap.end());
1067  emptyLumis.emplace_back(
1068  iPhidMap->second, lumiAux->run(), lumiAux->luminosityBlock(), IndexIntoFile::invalidEntry); // (insert 7)
1069  }
1070  runLumiMap.insert(std::make_pair(lumiID, lumiTree_.entryNumber()));
1071  }
1072  // now clean up.
1074  }
1075 
1076  // Insert the ordered empty lumis into the lumi list.
1077  LumiItemSortByRunLumi lumiItemSortByRunLumi;
1078  stable_sort_all(emptyLumis, lumiItemSortByRunLumi);
1079 
1080  LumiList::iterator itLumis = lumis.begin(), endLumis = lumis.end();
1081  for (auto const& emptyLumi : emptyLumis) {
1082  for (; itLumis != endLumis; ++itLumis) {
1083  if (lumiItemSortByRunLumi(emptyLumi, *itLumis)) {
1084  break;
1085  }
1086  }
1087  lumis.insert(itLumis, emptyLumi);
1088  }
1089 
1090  // Create a map of RunItems that gives the order of first appearance in the list.
1091  // Also fill in the vector of process history IDs
1092  typedef std::map<RunItem, int, RunItemSortByRunPhid> RunCountMap;
1093  RunCountMap runCountMap; // Declare (17)
1094  std::vector<ProcessHistoryID>& phids = indexIntoFile_.setProcessHistoryIDs();
1095  assert(phids.empty());
1096  std::vector<IndexIntoFile::RunOrLumiEntry>& entries = indexIntoFile_.setRunOrLumiEntries();
1097  assert(entries.empty());
1098  int rcount = 0;
1099  for (auto& run : runs) {
1100  RunCountMap::const_iterator countMapItem = runCountMap.find(run);
1101  if (countMapItem == runCountMap.end()) {
1102  countMapItem = runCountMap.insert(std::make_pair(run, rcount)).first; // Insert (17)
1103  assert(countMapItem != runCountMap.end());
1104  ++rcount;
1105  }
1106  std::vector<ProcessHistoryID>::const_iterator phidItem = find_in_all(phids, run.phid_);
1107  if (phidItem == phids.end()) {
1108  phids.push_back(run.phid_);
1109  phidItem = phids.end() - 1;
1110  }
1111  entries.emplace_back(countMapItem->second, // use (17)
1113  runMap[run.run_], // use (11)
1114  phidItem - phids.begin(),
1115  run.run_,
1116  0U,
1119  }
1120 
1121  // Create a map of LumiItems that gives the order of first appearance in the list.
1122  typedef std::map<LumiItem, int, LumiItemSortByRunLumiPhid> LumiCountMap;
1123  LumiCountMap lumiCountMap; // Declare (19)
1124  int lcount = 0;
1125  for (auto& lumi : lumis) {
1126  RunCountMap::const_iterator runCountMapItem = runCountMap.find(RunItem(lumi.phid_, lumi.run_));
1127  assert(runCountMapItem != runCountMap.end());
1128  LumiCountMap::const_iterator countMapItem = lumiCountMap.find(lumi);
1129  if (countMapItem == lumiCountMap.end()) {
1130  countMapItem = lumiCountMap.insert(std::make_pair(lumi, lcount)).first; // Insert (17)
1131  assert(countMapItem != lumiCountMap.end());
1132  ++lcount;
1133  }
1134  std::vector<ProcessHistoryID>::const_iterator phidItem = find_in_all(phids, lumi.phid_);
1135  assert(phidItem != phids.end());
1136  entries.emplace_back(runCountMapItem->second,
1137  countMapItem->second,
1138  runLumiMap[LuminosityBlockID(lumi.run_, lumi.lumi_)],
1139  phidItem - phids.begin(),
1140  lumi.run_,
1141  lumi.lumi_,
1142  lumi.firstEventEntry_,
1143  lumi.lastEventEntry_);
1144  }
1145  stable_sort_all(entries);
1146  }
1147 
1148  void RootFile::validateFile(InputType inputType, bool usingGoToEvent) {
1149  if (!fid_.isValid()) {
1151  }
1152  if (!eventTree_.isValid()) {
1153  throw Exception(errors::EventCorruption) << "'Events' tree is corrupted or not present\n"
1154  << "in the input file.\n";
1155  }
1156  if (enforceGUIDInFileName_) {
1157  auto guidFromName = stemFromPath(file_);
1158  if (guidFromName != fid_.fid()) {
1160  << "GUID " << guidFromName << " extracted from file name " << file_
1161  << " is inconsistent with the GUID read from the file " << fid_.fid();
1162  }
1163  }
1164 
1165  if (fileFormatVersion().hasIndexIntoFile()) {
1166  if (runTree().entries() > 0) {
1167  assert(!indexIntoFile_.empty());
1168  }
1170  if (daqProvenanceHelper_) {
1171  std::vector<ProcessHistoryID>& phidVec = indexIntoFile_.setProcessHistoryIDs();
1172  for (auto& phid : phidVec) {
1173  phid = daqProvenanceHelper_->mapProcessHistoryID(phid);
1174  }
1175  }
1177  }
1178  } else {
1179  assert(indexIntoFile_.empty());
1181  }
1182 
1186  std::shared_ptr<IndexIntoFile::EventFinder>(std::make_shared<RootFileEventFinder>(eventTree_)));
1187  // We fill the event numbers explicitly if we need to find events in closed files,
1188  // such as for secondary files (or secondary sources) or if duplicate checking across files.
1189  bool needEventNumbers = false;
1190  bool needIndexesForDuplicateChecker =
1191  duplicateChecker_ && duplicateChecker_->checkingAllFiles() && !duplicateChecker_->checkDisabled();
1192  if (inputType != InputType::Primary || needIndexesForDuplicateChecker || usingGoToEvent) {
1193  needEventNumbers = true;
1194  }
1195  bool needEventEntries = false;
1196  if (inputType != InputType::Primary || !noEventSort_) {
1197  // We need event entries for sorting or for secondary files or sources.
1198  needEventEntries = true;
1199  }
1200  indexIntoFile_.fillEventNumbersOrEntries(needEventNumbers, needEventEntries);
1201  }
1202 
1203  void RootFile::reportOpened(std::string const& inputType) {
1204  // Report file opened.
1205  std::string const label = "source";
1206  std::string moduleName = "PoolSource";
1207  filePtr_->inputFileOpened(logicalFile_, inputType, moduleName, label, fid_.fid(), eventTree_.branchNames());
1208  }
1209 
1211  // Just to play it safe, zero all pointers to objects in the InputFile to be closed.
1212  eventHistoryTree_ = nullptr;
1213  for (auto& treePointer : treePointers_) {
1214  treePointer->close();
1215  treePointer = nullptr;
1216  }
1217  filePtr_->Close();
1218  filePtr_ = nullptr; // propagate_const<T> has no reset() function
1219  }
1220 
1223  // Already read.
1224  return;
1225  }
1226  if (fileFormatVersion().newAuxiliary()) {
1227  EventAuxiliary* pEvAux = &eventAux_;
1229  } else {
1230  // for backward compatibility.
1232  EventAux* pEvAux = &eventAux;
1233  eventTree_.fillAux<EventAux>(pEvAux);
1234  conversion(eventAux, eventAux_);
1235  }
1237  }
1238 
1240  if (!eventTree_.current(entry)) {
1241  return false;
1242  }
1243  eventTree_.setEntryNumber(entry);
1245  return true;
1246  }
1247 
1249  // We could consider doing delayed reading, but because we have to
1250  // store this History object in a different tree than the event
1251  // data tree, this is too hard to do in this first version.
1252 
1253  if (fileFormatVersion().eventHistoryBranch()) {
1254  // Lumi block number was not in EventID for the relevant releases.
1255  EventID id(eventAux().id().run(), 0, eventAux().id().event());
1256  if (eventProcessHistoryIter_->eventID() != id) {
1259  assert(eventProcessHistoryIter_->eventID() == id);
1260  }
1263  } else if (fileFormatVersion().eventHistoryTree()) {
1264  // for backward compatibility.
1265  History* pHistory = history_.get();
1266  TBranch* eventHistoryBranch = eventHistoryTree_->GetBranch(poolNames::eventHistoryBranchName().c_str());
1267  if (!eventHistoryBranch) {
1268  throw Exception(errors::EventCorruption) << "Failed to find history branch in event history tree.\n";
1269  }
1270  eventHistoryBranch->SetAddress(&pHistory);
1272  eventAux_.setProcessHistoryID(history_->processHistoryID());
1273  eventSelectionIDs_.swap(history_->eventSelectionIDs());
1274  branchListIndexes_.swap(history_->branchListIndexes());
1275  } else if (fileFormatVersion().noMetaDataTrees()) {
1276  // Current format
1278  TBranch* eventSelectionIDBranch = eventTree_.tree()->GetBranch(poolNames::eventSelectionsBranchName().c_str());
1279  assert(eventSelectionIDBranch != nullptr);
1280  eventTree_.fillBranchEntry(eventSelectionIDBranch, pESV);
1282  TBranch* branchListIndexesBranch = eventTree_.tree()->GetBranch(poolNames::branchListIndexesBranchName().c_str());
1283  assert(branchListIndexesBranch != nullptr);
1284  eventTree_.fillBranchEntry(branchListIndexesBranch, pBLI);
1285  }
1286  if (provenanceAdaptor_) {
1287  eventAux_.setProcessHistoryID(provenanceAdaptor_->convertID(eventAux().processHistoryID()));
1288  for (auto& esID : eventSelectionIDs_) {
1289  esID = provenanceAdaptor_->convertID(esID);
1290  }
1291  }
1292  if (daqProvenanceHelper_) {
1294  }
1296  // old format. branchListIndexes_ must be filled in from the ProvenanceAdaptor.
1297  provenanceAdaptor_->branchListIndexes(branchListIndexes_);
1298  }
1299  if (branchIDListHelper_) {
1300  branchIDListHelper_->fixBranchListIndexes(branchListIndexes_);
1301  }
1302  }
1303 
1304  std::shared_ptr<LuminosityBlockAuxiliary> RootFile::fillLumiAuxiliary() {
1305  auto lumiAuxiliary = std::make_shared<LuminosityBlockAuxiliary>();
1306  if (fileFormatVersion().newAuxiliary()) {
1307  LuminosityBlockAuxiliary* pLumiAux = lumiAuxiliary.get();
1309  } else {
1310  LuminosityBlockAux lumiAux;
1311  LuminosityBlockAux* pLumiAux = &lumiAux;
1313  conversion(lumiAux, *lumiAuxiliary);
1314  }
1315  if (provenanceAdaptor_) {
1316  lumiAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(lumiAuxiliary->processHistoryID()));
1317  }
1318  if (daqProvenanceHelper_) {
1319  lumiAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(lumiAuxiliary->processHistoryID()));
1320  }
1321  if (lumiAuxiliary->luminosityBlock() == 0 && !fileFormatVersion().runsAndLumis()) {
1322  lumiAuxiliary->id() = LuminosityBlockID(RunNumber_t(1), LuminosityBlockNumber_t(1));
1323  }
1324  return lumiAuxiliary;
1325  }
1326 
1327  std::shared_ptr<RunAuxiliary> RootFile::fillRunAuxiliary() {
1328  auto runAuxiliary = std::make_shared<RunAuxiliary>();
1329  if (fileFormatVersion().newAuxiliary()) {
1330  RunAuxiliary* pRunAux = runAuxiliary.get();
1331  runTree_.fillAux<RunAuxiliary>(pRunAux);
1332  } else {
1333  RunAux runAux;
1334  RunAux* pRunAux = &runAux;
1335  runTree_.fillAux<RunAux>(pRunAux);
1336  conversion(runAux, *runAuxiliary);
1337  }
1338  if (provenanceAdaptor_) {
1339  runAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(runAuxiliary->processHistoryID()));
1340  }
1341  if (daqProvenanceHelper_) {
1342  runAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(runAuxiliary->processHistoryID()));
1343  }
1344  return runAuxiliary;
1345  }
1346 
1348  while (offset > 0 && indexIntoFileIter_ != indexIntoFileEnd_) {
1349  int phIndexOfSkippedEvent = IndexIntoFile::invalidIndex;
1350  RunNumber_t runOfSkippedEvent = IndexIntoFile::invalidRun;
1353 
1355  phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, skippedEventEntry);
1356 
1357  // At the end of the file and there were no more events to skip
1358  if (skippedEventEntry == IndexIntoFile::invalidEntry)
1359  break;
1360 
1361  if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
1362  fillEventAuxiliary(skippedEventEntry);
1363  if (eventSkipperByID_->skipIt(runOfSkippedEvent, lumiOfSkippedEvent, eventAux_.id().event())) {
1364  continue;
1365  }
1366  }
1367  if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) {
1368  fillEventAuxiliary(skippedEventEntry);
1369  if (duplicateChecker_->isDuplicateAndCheckActive(
1370  phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, eventAux_.id().event(), file_)) {
1371  continue;
1372  }
1373  }
1374  --offset;
1375  }
1376 
1377  while (offset < 0) {
1378  if (duplicateChecker_) {
1379  duplicateChecker_->disable();
1380  }
1381 
1382  int phIndexOfEvent = IndexIntoFile::invalidIndex;
1386 
1387  indexIntoFileIter_.skipEventBackward(phIndexOfEvent, runOfEvent, lumiOfEvent, eventEntry);
1388 
1389  if (eventEntry == IndexIntoFile::invalidEntry)
1390  break;
1391 
1392  if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
1393  fillEventAuxiliary(eventEntry);
1394  if (eventSkipperByID_->skipIt(runOfEvent, lumiOfEvent, eventAux_.id().event())) {
1395  continue;
1396  }
1397  }
1398  ++offset;
1399  }
1401  }
1402 
1405 
1406  if (duplicateChecker_) {
1407  duplicateChecker_->disable();
1408  }
1409 
1411  if (noEventSort_)
1413 
1415  indexIntoFile_.findPosition(sortOrder, eventID.run(), eventID.luminosityBlock(), eventID.event());
1416 
1417  if (iter == indexIntoFile_.end(sortOrder)) {
1418  return false;
1419  }
1420  indexIntoFileIter_ = iter;
1421  return true;
1422  }
1423 
1424  // readEvent() is responsible for creating, and setting up, the
1425  // EventPrincipal.
1426  //
1427  // 1. create an EventPrincipal with a unique EventID
1428  // 2. For each entry in the provenance, put in one ProductResolver,
1429  // holding the Provenance for the corresponding EDProduct.
1430  // 3. set up the the EventPrincipal to know about this ProductResolver.
1431  //
1432  // We do *not* create the EDProduct instance (the equivalent of reading
1433  // the branch containing this EDProduct. That will be done by the Delayed Reader,
1434  // when it is asked to do so.
1435  //
1439  // read the event auxiliary if not alrady read.
1441 
1442  // read the event
1443  readCurrentEvent(principal);
1444 
1445  runHelper_->checkRunConsistency(eventAux().run(), indexIntoFileIter_.run());
1446  runHelper_->checkLumiConsistency(eventAux().luminosityBlock(), indexIntoFileIter_.lumi());
1447 
1448  // If this next assert shows up in performance profiling or significantly affects memory, then these three lines should be deleted.
1449  // The IndexIntoFile should guarantee that it never fails.
1451  ? *daqProvenanceHelper_->oldProcessHistoryID()
1452  : eventAux().processHistoryID());
1453  ProcessHistoryID const& reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(idToCheck);
1455 
1457  }
1458 
1459  // Reads event at the current entry in the event tree
1461  if (!eventTree_.current()) {
1462  return false;
1463  }
1465  if (!fileFormatVersion().lumiInEventID()) {
1466  //ugly, but will disappear when the backward compatibility is done with schema evolution.
1467  const_cast<EventID&>(eventAux_.id()).setLuminosityBlockNumber(eventAux_.oldLuminosityBlock());
1469  }
1470  fillEventHistory();
1471  runHelper_->overrideRunNumber(eventAux_.id(), eventAux().isRealData());
1472 
1473  // We're not done ... so prepare the EventPrincipal
1475  principal.fillEventPrincipal(eventAux(),
1479  *(makeProductProvenanceRetriever(principal.streamID().value())),
1481 
1482  // report event read from file
1483  filePtr_->eventReadFromFile();
1484  return true;
1485  }
1486 
1488 
1489  std::shared_ptr<RunAuxiliary> RootFile::readRunAuxiliary_() {
1490  if (runHelper_->fakeNewRun()) {
1491  runHelper_->overrideRunNumber(savedRunAuxiliary_->id());
1492  return savedRunAuxiliary();
1493  }
1496 
1497  // Begin code for backward compatibility before the existence of run trees.
1498  if (!runTree_.isValid()) {
1499  // prior to the support of run trees.
1500  // RunAuxiliary did not contain a valid timestamp. Take it from the next event.
1502  assert(eventEntry != IndexIntoFile::invalidEntry);
1503  assert(eventTree_.current(eventEntry));
1504  fillEventAuxiliary(eventEntry);
1505 
1507  runHelper_->overrideRunNumber(run);
1508  savedRunAuxiliary_ = std::make_shared<RunAuxiliary>(run.run(), eventAux().time(), Timestamp::invalidTimestamp());
1509  return savedRunAuxiliary();
1510  }
1511  // End code for backward compatibility before the existence of run trees.
1513  std::shared_ptr<RunAuxiliary> runAuxiliary = fillRunAuxiliary();
1514  assert(runAuxiliary->run() == indexIntoFileIter_.run());
1515  runHelper_->overrideRunNumber(runAuxiliary->id());
1516  filePtr_->reportInputRunNumber(runAuxiliary->run());
1517  // If RunAuxiliary did not contain a valid begin timestamp, invalidate any end timestamp.
1518  if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1519  runAuxiliary->setEndTime(Timestamp::invalidTimestamp());
1520  }
1521 
1522  // If RunAuxiliary did not contain a valid timestamp, or if this an old format file from
1523  // when the Run's ProcessHistory included only processes where products were added to the Run itself,
1524  // we attempt to read the first event in the run to get appropriate info.
1525  if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp() ||
1528  // If we have a valid event, use its information.
1529  if (eventEntry != IndexIntoFile::invalidEntry) {
1530  assert(eventTree_.current(eventEntry));
1531  fillEventAuxiliary(eventEntry);
1532 
1533  // RunAuxiliary did not contain a valid timestamp. Take it from the next event in this run if there is one.
1534  if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1535  runAuxiliary->setBeginTime(eventAux().time());
1536  }
1537 
1538  // For backwards compatibility when the Run's ProcessHistory included only processes where products were added to the
1539  // Run, and then the Run and Event auxiliaries could be different. Use the event ProcessHistoryID if there is one. It should
1540  // almost always be correct by the current definition (processes included if any products are added. This makes the run, lumi,
1541  // and event ProcessHistory's always be the same if no file merging occurs).
1542  if (!fileFormatVersion().processHistorySameWithinRun()) {
1543  fillEventHistory();
1544  runAuxiliary->setProcessHistoryID(eventAux().processHistoryID());
1545  }
1546  }
1547  }
1548  savedRunAuxiliary_ = runAuxiliary;
1549  return runAuxiliary;
1550  }
1551 
1552  void RootFile::readRun_(RunPrincipal& runPrincipal) {
1553  MergeableRunProductMetadata* mergeableRunProductMetadata = nullptr;
1554  if (inputType_ == InputType::Primary) {
1555  mergeableRunProductMetadata = runPrincipal.mergeableRunProductMetadata();
1556  RootTree::EntryNumber const& entryNumber = runTree_.entryNumber();
1557  assert(entryNumber >= 0);
1558  mergeableRunProductMetadata->readRun(
1560  }
1561 
1562  if (!runHelper_->fakeNewRun()) {
1566  }
1567  // Begin code for backward compatibility before the existence of run trees.
1568  if (!runTree_.isValid()) {
1569  return;
1570  }
1571  // End code for backward compatibility before the existence of run trees.
1572  // NOTE: we use 0 for the index since do not do delayed reads for RunPrincipals
1575  // Read in all the products now.
1576  runPrincipal.readAllFromSourceAndMergeImmediately(mergeableRunProductMetadata);
1577  }
1578 
1579  std::shared_ptr<LuminosityBlockAuxiliary> RootFile::readLuminosityBlockAuxiliary_() {
1582  // Begin code for backward compatibility before the existence of lumi trees.
1583  if (!lumiTree_.isValid()) {
1585  assert(eventEntry != IndexIntoFile::invalidEntry);
1586  assert(eventTree_.current(eventEntry));
1587  fillEventAuxiliary(eventEntry);
1588 
1590  runHelper_->overrideRunNumber(lumi);
1591  return std::make_shared<LuminosityBlockAuxiliary>(
1593  }
1594  // End code for backward compatibility before the existence of lumi trees.
1596  std::shared_ptr<LuminosityBlockAuxiliary> lumiAuxiliary = fillLumiAuxiliary();
1597  assert(lumiAuxiliary->run() == indexIntoFileIter_.run());
1598  assert(lumiAuxiliary->luminosityBlock() == indexIntoFileIter_.lumi());
1599  runHelper_->overrideRunNumber(lumiAuxiliary->id());
1600  filePtr_->reportInputLumiSection(lumiAuxiliary->run(), lumiAuxiliary->luminosityBlock());
1601  if (lumiAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1603  if (eventEntry != IndexIntoFile::invalidEntry) {
1604  assert(eventTree_.current(eventEntry));
1605  fillEventAuxiliary(eventEntry);
1606 
1607  lumiAuxiliary->setBeginTime(eventAux().time());
1608  }
1609  lumiAuxiliary->setEndTime(Timestamp::invalidTimestamp());
1610  }
1611  if (!fileFormatVersion().processHistorySameWithinRun() && savedRunAuxiliary_) {
1612  lumiAuxiliary->setProcessHistoryID(savedRunAuxiliary_->processHistoryID());
1613  }
1614  return lumiAuxiliary;
1615  }
1616 
1620  // Begin code for backward compatibility before the existence of lumi trees.
1621  if (!lumiTree_.isValid()) {
1623  return;
1624  }
1625  // End code for backward compatibility before the existence of lumi trees.
1627  // NOTE: we use 0 for the index since do not do delayed reads for LuminosityBlockPrincipals
1630  // Read in all the products now.
1631  lumiPrincipal.readAllFromSourceAndMergeImmediately();
1633  }
1634 
1638  return false;
1640  return true;
1641  }
1642 
1646  return false;
1648  return true;
1649  }
1650 
1654  return false;
1656  return true;
1657  }
1658 
1662  }
1665  return false;
1666  if (run != indexIntoFileIter_.run())
1667  return false;
1668  if (lumi != indexIntoFileIter_.lumi())
1669  return false;
1671  return true;
1672  }
1673 
1675  // Read in the event history tree, if we have one...
1676  if (fileFormatVersion().eventHistoryTree()) {
1677  history_ = std::make_unique<History>(); // propagate_const<T> has no reset() function
1678  eventHistoryTree_ = dynamic_cast<TTree*>(filePtr_->Get(poolNames::eventHistoryTreeName().c_str()));
1679  if (!eventHistoryTree_) {
1680  throw Exception(errors::EventCorruption) << "Failed to find the event history tree.\n";
1681  }
1682  }
1683  }
1684 
1686  std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
1687  std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile) {
1688  if (duplicateChecker_ && !duplicateChecker_->checkDisabled()) {
1689  if (eventTree_.next()) {
1691  duplicateChecker_->inputFileOpened(
1692  eventAux().isRealData(), indexIntoFile_, indexesIntoFiles, currentIndexIntoFile);
1693  }
1695  }
1696  }
1697 
1698  void RootFile::markBranchToBeDropped(bool dropDescendants,
1699  BranchDescription const& branch,
1700  std::set<BranchID>& branchesToDrop,
1701  std::map<BranchID, BranchID> const& droppedToKeptAlias) const {
1702  if (dropDescendants) {
1703  branchChildren_->appendToDescendants(branch, branchesToDrop, droppedToKeptAlias);
1704  } else {
1705  branchesToDrop.insert(branch.branchID());
1706  }
1707  }
1708 
1710  ProductSelectorRules const& rules,
1711  bool dropDescendants,
1712  InputType inputType) {
1713  // This is the selector for drop on input.
1714  ProductSelector productSelector;
1715  productSelector.initialize(rules, reg.allBranchDescriptions());
1716 
1717  std::vector<BranchDescription const*> associationDescriptions;
1718 
1720  // Do drop on input. On the first pass, just fill in a set of branches to be dropped.
1721  std::set<BranchID> branchesToDrop;
1722  std::map<BranchID, BranchID> droppedToKeptAlias;
1723  for (auto const& product : prodList) {
1724  BranchDescription const& prod = product.second;
1725  if (prod.branchID() != prod.originalBranchID() && prod.present()) {
1726  droppedToKeptAlias[prod.originalBranchID()] = prod.branchID();
1727  }
1728  }
1729  for (auto const& product : prodList) {
1730  BranchDescription const& prod = product.second;
1731  // Special handling for ThinnedAssociations
1732  if (prod.unwrappedType() == typeid(ThinnedAssociation) && prod.present()) {
1733  if (inputType != InputType::SecondarySource) {
1734  associationDescriptions.push_back(&prod);
1735  } else {
1736  markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1737  }
1738  } else if (!productSelector.selected(prod)) {
1739  markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1740  }
1741  }
1742 
1743  if (inputType != InputType::SecondarySource) {
1744  // Decide whether to keep the thinned associations and corresponding
1745  // entries in the helper. For secondary source they are all dropped,
1746  // but in other cases we look for thinned collections the associations
1747  // redirect a Ref or Ptr to when dereferencing them.
1748 
1749  // Need a list of kept products in order to determine which thinned associations
1750  // are kept.
1751  std::set<BranchID> keptProductsInEvent;
1752  for (auto const& product : prodList) {
1753  BranchDescription const& prod = product.second;
1754  if (branchesToDrop.find(prod.branchID()) == branchesToDrop.end() && prod.present() &&
1755  prod.branchType() == InEvent) {
1756  keptProductsInEvent.insert(prod.branchID());
1757  }
1758  }
1759 
1760  // Decide which ThinnedAssociations to keep and store the decision in keepAssociation
1761  std::map<BranchID, bool> keepAssociation;
1762  fileThinnedAssociationsHelper_->selectAssociationProducts(
1763  associationDescriptions, keptProductsInEvent, keepAssociation);
1764 
1765  for (auto association : associationDescriptions) {
1766  if (!keepAssociation[association->branchID()]) {
1767  markBranchToBeDropped(dropDescendants, *association, branchesToDrop, droppedToKeptAlias);
1768  }
1769  }
1770 
1771  // Also delete the dropped associations from the ThinnedAssociationsHelper
1772  auto temp = std::make_unique<ThinnedAssociationsHelper>();
1773  for (auto const& associationBranches : fileThinnedAssociationsHelper_->data()) {
1774  auto iter = keepAssociation.find(associationBranches.association());
1775  if (iter != keepAssociation.end() && iter->second) {
1776  temp->addAssociation(associationBranches);
1777  }
1778  }
1779  // propagate_const<T> has no reset() function
1780  fileThinnedAssociationsHelper_ = std::unique_ptr<ThinnedAssociationsHelper>(temp.release());
1781  }
1782 
1783  // On this pass, actually drop the branches.
1784  std::set<BranchID>::const_iterator branchesToDropEnd = branchesToDrop.end();
1785  for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) {
1786  BranchDescription const& prod = it->second;
1787  bool drop = branchesToDrop.find(prod.branchID()) != branchesToDropEnd;
1788  if (drop) {
1789  if (!prod.dropped()) {
1790  if (productSelector.selected(prod) && prod.unwrappedType() != typeid(ThinnedAssociation)) {
1791  LogWarning("RootFile") << "Branch '" << prod.branchName() << "' is being dropped from the input\n"
1792  << "of file '" << file_ << "' because it is dependent on a branch\n"
1793  << "that was explicitly dropped.\n";
1794  }
1795  treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName()));
1796  hasNewlyDroppedBranch_[prod.branchType()] = true;
1797  }
1798  ProductRegistry::ProductList::iterator icopy = it;
1799  ++it;
1800  prodList.erase(icopy);
1801  } else {
1802  ++it;
1803  }
1804  }
1805 
1806  // Drop on input mergeable run and lumi products, this needs to be invoked for secondary file input
1807  if (inputType == InputType::SecondaryFile) {
1808  TString tString;
1809  for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) {
1810  BranchDescription const& prod = it->second;
1811  if (prod.branchType() != InEvent) {
1812  TClass* cp = prod.wrappedType().getClass();
1813  void* p = cp->New();
1814  int offset = cp->GetBaseClassOffset(edProductClass_);
1815  std::unique_ptr<WrapperBase> edp = getWrapperBasePtr(p, offset);
1816  if (edp->isMergeable()) {
1817  treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName()));
1818  ProductRegistry::ProductList::iterator icopy = it;
1819  ++it;
1820  prodList.erase(icopy);
1821  } else {
1822  ++it;
1823  }
1824  } else
1825  ++it;
1826  }
1827  }
1828  }
1829 
1831  signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* preEventReadSource,
1832  signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* postEventReadSource) {
1833  eventTree_.setSignals(preEventReadSource, postEventReadSource);
1834  }
1835 
1836  std::unique_ptr<MakeProvenanceReader> RootFile::makeProvenanceReaderMaker(InputType inputType) {
1838  readParentageTree(inputType);
1839  return std::make_unique<MakeReducedProvenanceReader>(parentageIDLookup_);
1840  } else if (fileFormatVersion_.splitProductIDs()) {
1841  readParentageTree(inputType);
1842  return std::make_unique<MakeFullProvenanceReader>();
1843  } else if (fileFormatVersion_.perEventProductIDs()) {
1844  auto entryDescriptionMap = std::make_unique<EntryDescriptionMap>();
1845  readEntryDescriptionTree(*entryDescriptionMap, inputType);
1846  return std::make_unique<MakeOldProvenanceReader>(std::move(entryDescriptionMap));
1847  } else {
1848  return std::make_unique<MakeDummyProvenanceReader>();
1849  }
1850  }
1851 
1852  std::shared_ptr<ProductProvenanceRetriever> RootFile::makeProductProvenanceRetriever(unsigned int iStreamID) {
1853  if (eventProductProvenanceRetrievers_.size() <= iStreamID) {
1854  eventProductProvenanceRetrievers_.resize(iStreamID + 1);
1855  }
1856  if (!eventProductProvenanceRetrievers_[iStreamID]) {
1857  // propagate_const<T> has no reset() function
1858  eventProductProvenanceRetrievers_[iStreamID] = std::make_shared<ProductProvenanceRetriever>(
1860  }
1861  eventProductProvenanceRetrievers_[iStreamID]->reset();
1862  return eventProductProvenanceRetriever(iStreamID);
1863  }
1864 
1866  public:
1867  ReducedProvenanceReader(RootTree* iRootTree,
1868  std::vector<ParentageID> const& iParentageIDLookup,
1869  DaqProvenanceHelper const* daqProvenanceHelper);
1870 
1871  std::set<ProductProvenance> readProvenance(unsigned int) const override;
1872 
1873  private:
1874  void readProvenanceAsync(WaitingTask* task,
1875  ModuleCallingContext const* moduleCallingContext,
1876  unsigned int transitionIndex,
1877  std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
1878 
1883  std::vector<ParentageID> const& parentageIDLookup_;
1885  std::shared_ptr<std::recursive_mutex> mutex_;
1887  };
1888 
1890  std::vector<ParentageID> const& iParentageIDLookup,
1891  DaqProvenanceHelper const* daqProvenanceHelper)
1893  rootTree_(iRootTree),
1894  pProvVector_(&provVector_),
1895  parentageIDLookup_(iParentageIDLookup),
1896  daqProvenanceHelper_(daqProvenanceHelper),
1897  mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
1898  acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {
1899  provBranch_ =
1900  rootTree_->tree()->GetBranch(BranchTypeToProductProvenanceBranchName(rootTree_->branchType()).c_str());
1901  }
1902 
1903  namespace {
1905  template <typename R>
1906  void readProvenanceAsyncImpl(R const* iThis,
1908  WaitingTask* task,
1909  unsigned int transitionIndex,
1910  std::atomic<const std::set<ProductProvenance>*>& writeTo,
1911  ModuleCallingContext const* iContext,
1912  SignalType const* pre,
1913  SignalType const* post) {
1914  if (nullptr == writeTo.load()) {
1915  //need to be sure the task isn't run until after the read
1916  WaitingTaskHolder taskHolder{task};
1917  auto pWriteTo = &writeTo;
1918 
1919  auto serviceToken = ServiceRegistry::instance().presentToken();
1920 
1921  chain.push([holder = std::move(taskHolder),
1922  pWriteTo,
1923  iThis,
1924  transitionIndex,
1925  iContext,
1926  pre,
1927  post,
1928  serviceToken]() mutable {
1929  if (nullptr == pWriteTo->load()) {
1930  ServiceRegistry::Operate operate(serviceToken);
1931  std::unique_ptr<const std::set<ProductProvenance>> prov;
1932  try {
1933  if (pre) {
1934  pre->emit(*(iContext->getStreamContext()), *iContext);
1935  }
1936  prov = std::make_unique<const std::set<ProductProvenance>>(iThis->readProvenance(transitionIndex));
1937  if (post) {
1938  post->emit(*(iContext->getStreamContext()), *iContext);
1939  }
1940 
1941  } catch (...) {
1942  if (post) {
1943  post->emit(*(iContext->getStreamContext()), *iContext);
1944  }
1945 
1946  holder.doneWaiting(std::current_exception());
1947  return;
1948  }
1949  const std::set<ProductProvenance>* expected = nullptr;
1950 
1951  if (pWriteTo->compare_exchange_strong(expected, prov.get())) {
1952  prov.release();
1953  }
1954  }
1955  holder.doneWaiting(std::exception_ptr());
1956  });
1957  }
1958  }
1959  } // namespace
1960 
1962  ModuleCallingContext const* moduleCallingContext,
1963  unsigned int transitionIndex,
1964  std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
1965  readProvenanceAsyncImpl(this,
1967  task,
1968  transitionIndex,
1969  writeTo,
1970  moduleCallingContext,
1971  rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
1972  rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
1973  }
1974 
1975  std::set<ProductProvenance> ReducedProvenanceReader::readProvenance(unsigned int transitionIndex) const {
1976  {
1977  std::lock_guard<std::recursive_mutex> guard(*mutex_);
1978  ReducedProvenanceReader* me = const_cast<ReducedProvenanceReader*>(this);
1979  me->rootTree_->fillBranchEntry(
1980  me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_);
1981  }
1982  std::set<ProductProvenance> retValue;
1983  if (daqProvenanceHelper_) {
1984  for (auto const& prov : provVector_) {
1985  BranchID bid(prov.branchID_);
1986  retValue.emplace(daqProvenanceHelper_->mapBranchID(BranchID(prov.branchID_)),
1987  daqProvenanceHelper_->mapParentageID(parentageIDLookup_[prov.parentageIDIndex_]));
1988  }
1989  } else {
1990  for (auto const& prov : provVector_) {
1991  if (prov.parentageIDIndex_ >= parentageIDLookup_.size()) {
1993  << "ReducedProvenanceReader::ReadProvenance\n"
1994  << "The parentage ID index value " << prov.parentageIDIndex_
1995  << " is out of bounds. The maximum value is " << parentageIDLookup_.size() - 1 << ".\n"
1996  << "This should never happen.\n"
1997  << "Please report this to the framework hypernews forum 'hn-cms-edmFramework@cern.ch'.\n";
1998  }
1999  retValue.emplace(BranchID(prov.branchID_), parentageIDLookup_[prov.parentageIDIndex_]);
2000  }
2001  }
2002  return retValue;
2003  }
2004 
2006  public:
2007  explicit FullProvenanceReader(RootTree* rootTree, DaqProvenanceHelper const* daqProvenanceHelper);
2009  std::set<ProductProvenance> readProvenance(unsigned int transitionIndex) const override;
2010 
2011  private:
2013  ModuleCallingContext const* moduleCallingContext,
2014  unsigned int transitionIndex,
2015  std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
2016 
2021  std::shared_ptr<std::recursive_mutex> mutex_;
2023  };
2024 
2027  rootTree_(rootTree),
2028  infoVector_(),
2029  pInfoVector_(&infoVector_),
2030  daqProvenanceHelper_(daqProvenanceHelper),
2031  mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2032  acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {}
2033 
2035  ModuleCallingContext const* moduleCallingContext,
2036  unsigned int transitionIndex,
2037  std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
2038  readProvenanceAsyncImpl(this,
2040  task,
2041  transitionIndex,
2042  writeTo,
2043  moduleCallingContext,
2046  }
2047 
2048  std::set<ProductProvenance> FullProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2049  {
2050  std::lock_guard<std::recursive_mutex> guard(*mutex_);
2053  }
2054  std::set<ProductProvenance> retValue;
2055  if (daqProvenanceHelper_) {
2056  for (auto const& info : infoVector_) {
2057  retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()),
2058  daqProvenanceHelper_->mapParentageID(info.parentageID()));
2059  }
2060  } else {
2061  for (auto const& info : infoVector_) {
2062  retValue.emplace(info);
2063  }
2064  }
2065  return retValue;
2066  }
2067 
2069  public:
2070  explicit OldProvenanceReader(RootTree* rootTree,
2071  EntryDescriptionMap const& theMap,
2072  DaqProvenanceHelper const* daqProvenanceHelper);
2073  ~OldProvenanceReader() override {}
2074  std::set<ProductProvenance> readProvenance(unsigned int transitionIndex) const override;
2075 
2076  private:
2078  ModuleCallingContext const* moduleCallingContext,
2079  unsigned int transitionIndex,
2080  std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
2081 
2083  std::vector<EventEntryInfo> infoVector_;
2084  mutable std::vector<EventEntryInfo>* pInfoVector_;
2087  std::shared_ptr<std::recursive_mutex> mutex_;
2089  };
2090 
2092  EntryDescriptionMap const& theMap,
2093  DaqProvenanceHelper const* daqProvenanceHelper)
2095  rootTree_(rootTree),
2096  infoVector_(),
2098  entryDescriptionMap_(theMap),
2099  daqProvenanceHelper_(daqProvenanceHelper),
2100  mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2101  acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {}
2102 
2104  ModuleCallingContext const* moduleCallingContext,
2105  unsigned int transitionIndex,
2106  std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
2107  readProvenanceAsyncImpl(this,
2109  task,
2110  transitionIndex,
2111  writeTo,
2112  moduleCallingContext,
2113  rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
2114  rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
2115  }
2116 
2117  std::set<ProductProvenance> OldProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2118  {
2119  std::lock_guard<std::recursive_mutex> guard(*mutex_);
2120  rootTree_->branchEntryInfoBranch()->SetAddress(&pInfoVector_);
2121  roottree::getEntry(rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex));
2122  }
2123  std::set<ProductProvenance> retValue;
2124  for (auto const& info : infoVector_) {
2125  EntryDescriptionMap::const_iterator iter = entryDescriptionMap_.find(info.entryDescriptionID());
2126  assert(iter != entryDescriptionMap_.end());
2127  Parentage parentage(iter->second.parents());
2128  if (daqProvenanceHelper_) {
2129  retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()),
2130  daqProvenanceHelper_->mapParentageID(parentage.id()));
2131  } else {
2132  retValue.emplace(info.branchID(), parentage.id());
2133  }
2134  }
2135  return retValue;
2136  }
2137 
2139  public:
2142 
2143  private:
2144  std::set<ProductProvenance> readProvenance(unsigned int) const override;
2146  ModuleCallingContext const* moduleCallingContext,
2147  unsigned int transitionIndex,
2148  std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
2149  };
2150 
2152 
2153  std::set<ProductProvenance> DummyProvenanceReader::readProvenance(unsigned int) const {
2154  // Not providing parentage!!!
2155  return std::set<ProductProvenance>{};
2156  }
2158  ModuleCallingContext const* moduleCallingContext,
2159  unsigned int transitionIndex,
2160  std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
2161  if (nullptr == writeTo.load()) {
2162  auto emptyProv = std::make_unique<const std::set<ProductProvenance>>();
2163  const std::set<ProductProvenance>* expected = nullptr;
2164  if (writeTo.compare_exchange_strong(expected, emptyProv.get())) {
2165  emptyProv.release();
2166  }
2167  }
2168  }
2169 
2170  std::unique_ptr<ProvenanceReaderBase> MakeDummyProvenanceReader::makeReader(RootTree&,
2171  DaqProvenanceHelper const*) const {
2172  return std::make_unique<DummyProvenanceReader>();
2173  }
2174 
2175  std::unique_ptr<ProvenanceReaderBase> MakeOldProvenanceReader::makeReader(
2176  RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2177  return std::make_unique<OldProvenanceReader>(&rootTree, *entryDescriptionMap_, daqProvenanceHelper);
2178  }
2179 
2180  std::unique_ptr<ProvenanceReaderBase> MakeFullProvenanceReader::makeReader(
2181  RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2182  return std::make_unique<FullProvenanceReader>(&rootTree, daqProvenanceHelper);
2183  }
2184 
2185  std::unique_ptr<ProvenanceReaderBase> MakeReducedProvenanceReader::makeReader(
2186  RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2187  return std::make_unique<ReducedProvenanceReader>(&rootTree, parentageIDLookup_, daqProvenanceHelper);
2188  }
2189 } // namespace edm
void dropOnInput(ProductRegistry &reg, ProductSelectorRules const &rules, bool dropDescendants, InputType inputType)
Definition: RootFile.cc:1709
RunNumber_t run() const
Definition: EventID.h:39
Code categoryCode() const
Definition: EDMException.h:97
EventID const & eventID() const
Definition: RootFile.h:220
Int_t getEntry(TBranch *branch, EntryNumber entryNumber)
Definition: RootTree.cc:495
void fillEventNumbersOrEntries(bool needEventNumbers, bool needEventEntries) const
EventNumber_t event() const
Definition: EventID.h:41
std::string const & idToParameterSetBlobsBranchName()
Definition: BranchType.cc:261
edm::propagate_const< std::unique_ptr< ThinnedAssociationsHelper > > fileThinnedAssociationsHelper_
Definition: RootFile.h:346
std::vector< ProcessConfiguration > ProcessConfigurationVector
void setEventFinder(std::shared_ptr< EventFinder > ptr) const
std::string const & branchName() const
StoredProductProvenanceVector provVector_
Definition: RootFile.cc:1881
bool isRealData() const
bool selected(BranchDescription const &desc) const
std::string const & BranchTypeToAuxiliaryBranchName(BranchType const &branchType)
Definition: BranchType.cc:116
unsigned int const defaultNonEventLearningEntries
Definition: RootTree.h:49
InputType
Definition: InputType.h:5
static const TGPicture * info(bool iBackgroundIsBlack)
void doneFileInitialization() const
Clears the temporary vector of event numbers to reduce memory usage.
static constexpr int invalidIndex
TPRegexp parents
Definition: eve_filter.cc:21
BranchType const & branchType() const
std::string const & parentageTreeName()
Definition: BranchType.cc:160
std::vector< BranchIDList > BranchIDLists
Definition: BranchIDList.h:19
std::shared_ptr< BranchIDLists const > branchIDLists_
Definition: RootFile.h:344
std::string const & entryDescriptionBranchName()
Definition: BranchType.cc:155
bool skipEvents(int &offset)
Definition: RootFile.cc:1347
StreamContext const * getStreamContext() const
edm::propagate_const< std::shared_ptr< DuplicateChecker > > duplicateChecker_
Definition: RootFile.h:356
FileFormatVersion fileFormatVersion() const
Definition: RootFile.h:224
std::shared_ptr< LuminosityBlockAuxiliary > fillLumiAuxiliary()
Definition: RootFile.cc:1304
ProductProvenanceVector infoVector_
Definition: RootFile.cc:2018
static std::string const source("source")
static Timestamp invalidTimestamp()
Definition: Timestamp.h:101
ForwardSequence::const_iterator lower_bound_all(ForwardSequence const &s, Datum const &d)
wrappers for std::lower_bound
Definition: Algorithms.h:69
Definition: chain.py:1
void readEntryDescriptionTree(EntryDescriptionMap &entryDescriptionMap, InputType inputType)
Definition: RootFile.cc:597
bool branchListIndexesUnchanged() const
Definition: RootFile.h:227
RunNumber_t run() const
Definition: RunID.h:39
std::vector< std::string > const & branchNames() const
Definition: RootTree.h:138
bool setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi)
Definition: RootFile.cc:1659
bool enforceGUIDInFileName_
Definition: RootFile.h:333
static PFTauRenderPlugin instance
MakeOldProvenanceReader(std::unique_ptr< EntryDescriptionMap > &&entryDescriptionMap)
Definition: RootFile.cc:78
int whyNotFastClonable_
Definition: RootFile.h:334
std::vector< BranchID > & parentsForUpdate()
Definition: Parentage.h:45
edm::propagate_const< std::shared_ptr< EventSkipperByID > > eventSkipperByID_
Definition: RootFile.h:318
edm::propagate_const< std::shared_ptr< ThinnedAssociationsHelper > > thinnedAssociationsHelper_
Definition: RootFile.h:347
ParentageID id() const
Definition: Parentage.cc:23
DaqProvenanceHelper const * daqProvenanceHelper_
Definition: RootFile.cc:2020
std::map< std::string, std::string > newBranchToOldBranch_
Definition: RootFile.h:350
bool empty() const
True if no runs, lumis, or events are in the file.
std::map< BranchKey, BranchDescription > ProductList
BranchID const & mapBranchID(BranchID const &branchID) const
#define nullptr
bool registerProcessHistory(ProcessHistory const &processHistory)
bool setEntryAtRun(RunNumber_t run)
Definition: RootFile.cc:1651
void readRun_(RunPrincipal &runPrincipal)
Definition: RootFile.cc:1552
std::string_view stemFromPath(std::string_view path)
Definition: stemFromPath.cc:4
ProductProvenanceVector * pInfoVector_
Definition: RootFile.cc:2019
RootTree lumiTree_
Definition: RootFile.h:339
Timestamp const & time() const
InputType inputType_
Definition: RootFile.h:363
edm::propagate_const< ProcessHistoryRegistry * > processHistoryRegistry_
Definition: RootFile.h:316
edm::propagate_const< std::unique_ptr< DaqProvenanceHelper > > daqProvenanceHelper_
Definition: RootFile.h:361
edm::propagate_const< std::shared_ptr< RunAuxiliary > > savedRunAuxiliary_
Definition: RootFile.h:330
unsigned long long EventNumber_t
EntryNumber const & entries() const
Definition: RootTree.h:135
RunNumber_t run() const
void setSignals(signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)> const *preEventReadSource, signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)> const *postEventReadSource)
Definition: RootTree.cc:488
std::string const & fileFormatVersionBranchName()
Definition: BranchType.cc:219
std::string message() const
Definition: Exception.cc:145
IndexIntoFileItr begin(SortOrder sortOrder) const
FullProvenanceReader(RootTree *rootTree, DaqProvenanceHelper const *daqProvenanceHelper)
Definition: RootFile.cc:2025
EntryDescriptionMap const & entryDescriptionMap_
Definition: RootFile.cc:2085
std::vector< BranchID > const & parents() const
std::string const & eventSelectionsBranchName()
Definition: BranchType.cc:249
edm::propagate_const< std::unique_ptr< EntryDescriptionMap > > entryDescriptionMap_
Definition: RootFile.cc:84
void initializeDuplicateChecker(std::vector< std::shared_ptr< IndexIntoFile >> const &indexesIntoFiles, std::vector< std::shared_ptr< IndexIntoFile >>::size_type currentIndexIntoFile)
Definition: RootFile.cc:1685
void push(T &&iAction)
asynchronously pushes functor iAction into queue
InputSource::ProcessingMode processingMode_
Definition: RootFile.h:348
~RootFileEventFinder() override
Definition: RootFile.cc:123
IndexIntoFile::IndexIntoFileItr indexIntoFileIter_
Definition: RootFile.h:326
std::shared_ptr< ProductProvenanceRetriever const > eventProductProvenanceRetriever(size_t index) const
Definition: RootFile.h:306
bool current() const
Definition: RootTree.h:128
void fillRunPrincipal(ProcessHistoryRegistry const &processHistoryRegistry, DelayedReader *reader=0)
Definition: RunPrincipal.cc:26
void stable_sort_all(RandomAccessSequence &s)
wrappers for std::stable_sort
Definition: Algorithms.h:103
std::shared_ptr< std::recursive_mutex > mutex_
Definition: RootFile.cc:2087
LuminosityBlockNumber_t lumi() const
void fillBranchEntryMeta(TBranch *branch, T *&pbuf)
Definition: RootTree.h:147
std::unique_ptr< FileBlock > createFileBlock() const
Definition: RootFile.cc:722
LuminosityBlockNumber_t luminosityBlock() const
Definition: EventID.h:40
void trainCache(char const *branchNames)
Definition: RootTree.cc:460
bool readCurrentEvent(EventPrincipal &cache)
Definition: RootFile.cc:1460
SharedResourcesAcquirer acquirer_
Definition: RootFile.cc:2088
~FullProvenanceReader() override
Definition: RootFile.cc:2008
ServiceToken presentToken() const
std::shared_ptr< ProductRegistry const > productRegistry() const
Definition: RootFile.h:215
uint16_t size_type
unsigned int LuminosityBlockNumber_t
IndexIntoFileItr findRunPosition(RunNumber_t run) const
Same as findPosition.
void readProvenanceAsync(WaitingTask *task, ModuleCallingContext const *moduleCallingContext, unsigned int transitionIndex, std::atomic< const std::set< ProductProvenance > * > &writeTo) const override
Definition: RootFile.cc:2103
void readAllFromSourceAndMergeImmediately(MergeableRunProductMetadata const *mergeableRunProductMetadata=0)
Definition: Principal.cc:904
TBranch * branchEntryInfoBranch() const
Definition: RootTree.h:186
bool fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry)
Definition: RootFile.cc:1239
MakeReducedProvenanceReader(std::vector< ParentageID > const &parentageIDLookup)
Definition: RootFile.cc:93
IndexIntoFileItr findPosition(RunNumber_t run, LuminosityBlockNumber_t lumi=0U, EventNumber_t event=0U) const
std::vector< EventSelectionID > EventSelectionIDVector
edm::propagate_const< TClass * > edProductClass_
Definition: RootFile.h:362
LuminosityBlockNumber_t luminosityBlock() const
FileFormatVersion fileFormatVersion_
Definition: RootFile.h:319
TTree const * metaTree() const
Definition: RootTree.h:182
std::string const & parameterSetsTreeName()
Definition: BranchType.cc:257
void setPosition(IndexIntoFile::IndexIntoFileItr const &position)
Definition: RootFile.cc:748
std::vector< edm::propagate_const< std::shared_ptr< ProductProvenanceRetriever > > > eventProductProvenanceRetrievers_
Definition: RootFile.h:359
edm::propagate_const< RootTree * > rootTree_
Definition: RootFile.cc:2082
ProductList const & productList() const
U second(std::pair< T, U > const &p)
std::vector< EventProcessHistoryID >::const_iterator eventProcessHistoryIter_
Definition: RootFile.h:329
void reduceProcessHistoryIDs(ProcessHistoryRegistry const &processHistoryRegistry)
bool skipThisEntry()
Definition: RootFile.cc:757
char const * label
TTree const * tree() const
Definition: RootTree.h:180
FileID fid_
Definition: RootFile.h:320
long long EntryNumber_t
bool containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const
Definition: RootFile.cc:808
void setParents(std::vector< BranchID > const &parents)
Definition: Parentage.h:46
DaqProvenanceHelper const * daqProvenanceHelper_
Definition: RootFile.cc:1884
std::string const logicalFile_
Definition: RootFile.h:314
def principal(options)
std::vector< BranchListIndex > BranchListIndexes
IndexIntoFile::IndexIntoFileItr indexIntoFileEnd_
Definition: RootFile.h:325
std::string moduleName(Provenance const &provenance)
Definition: Provenance.cc:27
void fillLuminosityBlockPrincipal(ProcessHistoryRegistry const &processHistoryRegistry, DelayedReader *reader=0)
void readParentageTree(InputType inputType)
Definition: RootFile.cc:643
std::string const & processHistoryMapBranchName()
Definition: BranchType.cc:194
std::vector< ProcessHistoryID > & setProcessHistoryIDs()
bool next()
Definition: RootTree.h:125
edm::propagate_const< std::unique_ptr< StoredMergeableRunProductMetadata > > storedMergeableRunProductMetadata_
Definition: RootFile.h:327
TClass * getClass() const
bool noEventSort_
Definition: RootFile.h:332
std::string const & className() const
bool wasLastEventJustRead() const
Definition: RootFile.cc:857
void insertEntryForIndex(unsigned int index)
Definition: RootTree.cc:104
DelayedReader * resetAndGetRootDelayedReader() const
Definition: RootTree.cc:122
bool setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi)
Definition: RootFile.cc:1643
std::string friendlyName(std::string const &iFullName)
static constexpr RunNumber_t invalidRun
std::string const & entryDescriptionTreeName()
Definition: BranchType.cc:147
void close()
Definition: RootFile.cc:1210
virtual signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)> const * postEventReadFromSourceSignal() const =0
std::string const & fid() const
Definition: FileID.h:19
std::shared_ptr< std::recursive_mutex > mutex_
Definition: RootFile.cc:2021
std::string const & mergeableRunProductMetadataBranchName()
Definition: BranchType.cc:239
RootTree const & runTree() const
Definition: RootFile.h:223
bool modifiedIDs() const
Definition: RootFile.h:228
IndexIntoFile::EntryNumber_t lastEventEntryNumberRead_
Definition: RootFile.h:342
void copyPosition(IndexIntoFileItr const &position)
Copy the position without modifying the pointer to the IndexIntoFile or size.
std::string const & indexIntoFileBranchName()
Definition: BranchType.cc:234
IndexIntoFileItr end(SortOrder sortOrder) const
Used to end an iteration over the Runs, Lumis, and Events in a file.
std::string const & eventHistoryBranchName()
Definition: BranchType.cc:244
std::set< ProductProvenance > readProvenance(unsigned int transitionIndex) const override
Definition: RootFile.cc:2048
void skipEventBackward(int &phIndexOfEvent, RunNumber_t &runOfEvent, LuminosityBlockNumber_t &lumiOfEvent, EntryNumber_t &eventEntry)
Long64_t numEntries(TFile *hdl, std::string const &trname)
Definition: CollUtil.cc:50
edm::propagate_const< RootTree * > rootTree_
Definition: RootFile.cc:1879
std::shared_ptr< RunAuxiliary > readRunAuxiliary_()
Definition: RootFile.cc:1489
EventAuxiliary const & eventAux() const
Definition: RootFile.h:216
EventSelectionIDVector eventSelectionIDs_
Definition: RootFile.h:352
std::shared_ptr< LuminosityBlockAuxiliary > readLuminosityBlockAuxiliary_()
Definition: RootFile.cc:1579
bool eventHistoryTree() const
bool isEarlierRelease(std::string const &a, std::string const &b)
StoredProductProvenanceVector const * pProvVector_
Definition: RootFile.cc:1882
int whyNotFastClonable() const
Definition: RootFile.h:225
edm::propagate_const< std::unique_ptr< ProvenanceAdaptor > > provenanceAdaptor_
Definition: RootFile.h:357
StreamID streamID() const
static ServiceRegistry & instance()
RunNumber_t run() const
std::shared_ptr< RunAuxiliary > fillRunAuxiliary()
Definition: RootFile.cc:1327
bool skipAnyEvents_
Definition: RootFile.h:331
void setSignals(signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)> const *preEventReadSource, signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)> const *postEventReadSource)
Definition: RootFile.cc:1830
RootTree eventTree_
Definition: RootFile.h:338
std::vector< BranchDescription const * > allBranchDescriptions() const
EventNumber_t getEventNumberOfEntry(roottree::EntryNumber entry) const override
Definition: RootFile.cc:125
std::map< ParameterSetID, ParameterSetID > ParameterSetIdConverter
bool isValid() const
Definition: FileID.h:18
bool iterationWillBeInEntryOrder(SortOrder sortOrder) const
Used to determine whether or not to disable fast cloning.
std::string const & friendlyClassName() const
BranchID const & branchID() const
TypeWithDict const & unwrappedType() const
bool insertMapped(value_type const &v, bool forceUpdate=false)
Definition: Registry.cc:32
std::array< bool, NumBranchTypes > const & hasNewlyDroppedBranch() const
Definition: RootFile.h:226
IndexIntoFile::EntryType getNextItemType(RunNumber_t &run, LuminosityBlockNumber_t &lumi, EventNumber_t &event)
Definition: RootFile.cc:812
BranchListIndexes branchListIndexes_
Definition: RootFile.h:353
std::string const & metaDataTreeName()
Definition: BranchType.cc:169
virtual signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)> const * preEventReadFromSourceSignal() const =0
bool isDuplicateEvent()
Definition: RootFile.cc:795
LuminosityBlockNumber_t oldLuminosityBlock() const
EntryDescriptionID id() const
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
IndexIntoFile::EntryNumber_t EntryNumber
Definition: RootTree.h:50
bool wasFirstEventJustRead() const
Definition: RootFile.cc:863
std::string const & parameterSetMapBranchName()
Definition: BranchType.cc:184
Hash< ProcessHistoryType > ProcessHistoryID
std::unique_ptr< ProvenanceReaderBase > makeReader(RootTree &eventTree, DaqProvenanceHelper const *daqProvenanceHelper) const override
Definition: RootFile.cc:2170
edm::propagate_const< std::unique_ptr< History > > history_
Definition: RootFile.h:354
SerialTaskQueueChain & serialQueueChain() const
std::vector< ProductProvenance > ProductProvenanceVector
const int drop
std::string const & processHistoryBranchName()
Definition: BranchType.cc:199
IndexIntoFile & indexIntoFile_
Definition: RootFile.h:322
void readEvent(EventPrincipal &cache)
Definition: RootFile.cc:1436
RootTree runTree_
Definition: RootFile.h:340
SharedResourcesAcquirer acquirer_
Definition: RootFile.cc:2022
edm::propagate_const< std::unique_ptr< MakeProvenanceReader > > provenanceReaderMaker_
Definition: RootFile.h:358
EntryNumber const & entryNumber() const
Definition: RootTree.h:133
bool containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const
bool goToEvent(EventID const &eventID)
Definition: RootFile.cc:1403
std::string getReleaseVersion()
std::shared_ptr< RunAuxiliary const > savedRunAuxiliary() const
Definition: RootFile.h:300
std::set< ProductProvenance > readProvenance(unsigned int) const override
Definition: RootFile.cc:1975
RootFile(std::string const &fileName, ProcessConfiguration const &processConfiguration, std::string const &logicalFileName, std::shared_ptr< InputFile > filePtr, std::shared_ptr< EventSkipperByID > eventSkipperByID, bool skipAnyEvents, int remainingEvents, int remainingLumis, unsigned int nStreams, unsigned int treeCacheSize, int treeMaxVirtualSize, InputSource::ProcessingMode processingMode, RunHelperBase *runHelper, bool noEventSort, ProductSelectorRules const &productSelectorRules, InputType inputType, std::shared_ptr< BranchIDListHelper > branchIDListHelper, std::shared_ptr< ThinnedAssociationsHelper > thinnedAssociationsHelper, std::vector< BranchID > const *associationsFromSecondary, std::shared_ptr< DuplicateChecker > duplicateChecker, bool dropDescendantsOfDroppedProducts, ProcessHistoryRegistry &processHistoryRegistry, std::vector< std::shared_ptr< IndexIntoFile >> const &indexesIntoFiles, std::vector< std::shared_ptr< IndexIntoFile >>::size_type currentIndexIntoFile, std::vector< ProcessHistoryID > &orderedProcessHistoryIDs, bool bypassVersionCheck, bool labelRawDataLikeMC, bool usingGoToEvent, bool enablePrefetching, bool enforceGUIDInFileName)
Definition: RootFile.cc:140
void setAtEventEntry(IndexIntoFile::EntryNumber_t entry)
Definition: RootFile.cc:1487
bool storedProductProvenanceUsed() const
void fixIndexes(std::vector< ProcessHistoryID > &processHistoryIDs)
DelayedReader * rootDelayedReader() const
Definition: RootTree.cc:127
std::vector< RunOrLumiEntry > & setRunOrLumiEntries()
EntryNumber const & entryNumberForIndex(unsigned int index) const
Definition: RootTree.cc:99
void conversion(EventAux const &from, EventAuxiliary &to)
Definition: EventAux.cc:9
std::array< bool, NumBranchTypes > hasNewlyDroppedBranch_
Definition: RootFile.h:335
std::vector< EventNumber_t > & unsortedEventNumbers() const
unsigned int value() const
Definition: StreamID.h:42
static constexpr EntryNumber_t invalidEntry
ForwardSequence::const_iterator find_in_all(ForwardSequence const &s, Datum const &d)
wrappers for std::find
Definition: Algorithms.h:26
std::shared_ptr< ProductRegistry const > productRegistry_
Definition: RootFile.h:343
void setIfFastClonable(int remainingEvents, int remainingLumis)
Definition: RootFile.cc:678
std::unique_ptr< ProvenanceReaderBase > makeReader(RootTree &eventTree, DaqProvenanceHelper const *daqProvenanceHelper) const override
Definition: RootFile.cc:2185
MergeableRunProductMetadata * mergeableRunProductMetadata()
Definition: RunPrincipal.h:78
void fillBranchEntry(TBranch *branch, T *&pbuf)
Definition: RootTree.h:158
static constexpr LuminosityBlockNumber_t invalidLumi
IndexIntoFileItr findLumiPosition(RunNumber_t run, LuminosityBlockNumber_t lumi) const
std::map< EntryDescriptionID, EventEntryDescription > EntryDescriptionMap
Definition: RootFile.h:52
edm::propagate_const< std::shared_ptr< BranchChildren > > branchChildren_
Definition: RootFile.h:355
std::string const & parentageBranchName()
Definition: BranchType.cc:164
edm::propagate_const< std::shared_ptr< InputFile > > filePtr_
Definition: RootFile.h:317
std::shared_ptr< BranchChildren const > branchChildren() const
Definition: RootFile.h:303
double b
Definition: hdecay.h:120
LuminosityBlockNumber_t luminosityBlock() const
bool processHistorySameWithinRun() const
void fillEventHistory()
Definition: RootFile.cc:1248
ProcessHistoryID const & processHistoryID(int i) const
ProductList & productListUpdator()
void addContext(std::string const &context)
Definition: Exception.cc:165
void setProcessHistoryID(ProcessHistoryID const &phid)
unsigned int transitionIndex() const
void readProvenanceAsync(WaitingTask *task, ModuleCallingContext const *moduleCallingContext, unsigned int transitionIndex, std::atomic< const std::set< ProductProvenance > * > &writeTo) const override
Definition: RootFile.cc:2157
void readProvenanceAsync(WaitingTask *task, ModuleCallingContext const *moduleCallingContext, unsigned int transitionIndex, std::atomic< const std::set< ProductProvenance > * > &writeTo) const override
Definition: RootFile.cc:1961
std::vector< ParentageID > const & parentageIDLookup_
Definition: RootFile.cc:1883
std::shared_ptr< std::recursive_mutex > mutex_
Definition: RootFile.cc:1885
std::unique_ptr< ProvenanceReaderBase > makeReader(RootTree &eventTree, DaqProvenanceHelper const *daqProvenanceHelper) const override
Definition: RootFile.cc:2175
bool isValid() const
Definition: RootTree.cc:109
~OldProvenanceReader() override
Definition: RootFile.cc:2073
std::shared_ptr< ProductProvenanceRetriever > makeProductProvenanceRetriever(unsigned int iStreamIndex)
Definition: RootFile.cc:1852
std::vector< StoredProductProvenance > StoredProductProvenanceVector
std::string const & file() const
Definition: RootFile.h:214
std::string const & productDescriptionBranchName()
Definition: BranchType.cc:174
std::string const & processConfigurationBranchName()
Definition: BranchType.cc:204
IndexIntoFile::IndexIntoFileItr indexIntoFileBegin_
Definition: RootFile.h:324
ProcessHistoryID const & processHistoryID() const
EventID const & id() const
edm::propagate_const< RunHelperBase * > runHelper_
Definition: RootFile.h:349
bool branchListIndexesUnchanged_
Definition: RootFile.h:336
void fillEventPrincipal(EventAuxiliary const &aux, ProcessHistoryRegistry const &processHistoryRegistry, DelayedReader *reader=0)
void fillIndexIntoFile()
Definition: RootFile.cc:924
#define begin
Definition: vmac.h:32
HLT enums.
edm::propagate_const< TTree * > eventHistoryTree_
Definition: RootFile.h:351
void readLuminosityBlock_(LuminosityBlockPrincipal &lumiPrincipal)
Definition: RootFile.cc:1617
std::vector< EventEntryInfo > * pInfoVector_
Definition: RootFile.cc:2084
std::unique_ptr< ProvenanceReaderBase > makeReader(RootTree &eventTree, DaqProvenanceHelper const *daqProvenanceHelper) const override
Definition: RootFile.cc:2180
unsigned int const defaultNonEventCacheSize
Definition: RootTree.h:47
RootTreePtrArray treePointers_
Definition: RootFile.h:341
void fillEventNumbers() const
double a
Definition: hdecay.h:121
void initialize(ProductSelectorRules const &rules, std::vector< BranchDescription const * > const &branchDescriptions)
static int position[264][3]
Definition: ReadPGInfo.cc:509
bool perEventProductIDs() const
std::string const & BranchTypeToProductProvenanceBranchName(BranchType const &BranchType)
Definition: BranchType.cc:132
std::vector< ProcessHistoryID > & orderedProcessHistoryIDs_
Definition: RootFile.h:323
std::string const & productDependenciesBranchName()
Definition: BranchType.cc:179
void setNumberOfEvents(EntryNumber_t nevents) const
void initAssociationsFromSecondary(std::vector< BranchID > const &)
Definition: RootFile.cc:752
std::string const & thinnedAssociationsHelperBranchName()
Definition: BranchType.cc:214
std::string const & entryDescriptionIDBranchName()
Definition: BranchType.cc:151
~DummyProvenanceReader() override
Definition: RootFile.cc:2141
bool useReducedProcessHistoryID() const
edm::propagate_const< TBranch * > provBranch_
Definition: RootFile.cc:1880
std::unique_ptr< MakeProvenanceReader > makeProvenanceReaderMaker(InputType inputType)
Definition: RootFile.cc:1836
std::unique_ptr< WrapperBase > getWrapperBasePtr(void *p, int offset)
std::string const & branchIDListBranchName()
Definition: BranchType.cc:209
BranchID const & originalBranchID() const
unsigned int RunNumber_t
std::string const & branchListIndexesBranchName()
Definition: BranchType.cc:253
std::vector< ParentageID > parentageIDLookup_
Definition: RootFile.h:360
EventAuxiliary eventAux_
Definition: RootFile.h:337
void readProvenanceAsync(WaitingTask *task, ModuleCallingContext const *moduleCallingContext, unsigned int transitionIndex, std::atomic< const std::set< ProductProvenance > * > &writeTo) const override
Definition: RootFile.cc:2034
void fillThisEventAuxiliary()
Definition: RootFile.cc:1221
void resetTraining()
Definition: RootTree.h:192
bool setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event)
Definition: RootFile.cc:1635
std::vector< ParentageID > const & parentageIDLookup_
Definition: RootFile.cc:99
ReducedProvenanceReader(RootTree *iRootTree, std::vector< ParentageID > const &iParentageIDLookup, DaqProvenanceHelper const *daqProvenanceHelper)
Definition: RootFile.cc:1889
std::string const & eventHistoryTreeName()
Definition: BranchType.cc:274
static Interceptor::Registry registry("Interceptor")
RootFileEventFinder(RootTree &eventTree)
Definition: RootFile.cc:122
ParentageID const & mapParentageID(ParentageID const &phid) const
edm::propagate_const< std::shared_ptr< BranchIDListHelper > > branchIDListHelper_
Definition: RootFile.h:345
void validateFile(InputType inputType, bool usingGoToEvent)
Definition: RootFile.cc:1148
IndexIntoFile::IndexIntoFileItr indexIntoFileIter() const
Definition: RootFile.cc:746
std::string const & newBranchToOldBranch(std::string const &newBranch) const
Definition: RootFile.cc:738
void readRun(long long inputRunEntry, StoredMergeableRunProductMetadata const &inputStoredMergeableRunProductMetadata, IndexIntoFileItrHolder const &inputIndexIntoFileItr)
T first(std::pair< T, U > const &p)
std::unique_ptr< TTreeCache > trainCache(TTree *tree, InputFile &file, unsigned int cacheSize, char const *branchNames)
Definition: RootTree.cc:515
static ParentageRegistry * instance()
OldProvenanceReader(RootTree *rootTree, EntryDescriptionMap const &theMap, DaqProvenanceHelper const *daqProvenanceHelper)
Definition: RootFile.cc:2091
SharedResourcesAcquirer acquirer_
Definition: RootFile.cc:1886
IndexIntoFileItr findEventPosition(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const
unsigned int const defaultLearningEntries
Definition: RootTree.h:48
std::vector< EventEntryInfo > infoVector_
Definition: RootFile.cc:2083
void markBranchToBeDropped(bool dropDescendants, BranchDescription const &branch, std::set< BranchID > &branchesToDrop, std::map< BranchID, BranchID > const &droppedToKeptAlias) const
Definition: RootFile.cc:1698
LuminosityBlockNumber_t peekAheadAtLumi() const
bool hasThinnedAssociations() const
std::string const & fileIdentifierBranchName()
Definition: BranchType.cc:224
void setIsMergeable(BranchDescription &)
EventNumber_t event() const
std::string const & moduleDescriptionMapBranchName()
Definition: BranchType.cc:189
bool insertMapped(value_type const &v)
std::string const file_
Definition: RootFile.h:313
def move(src, dest)
Definition: eostools.py:511
void skipEventForward(int &phIndexOfSkippedEvent, RunNumber_t &runOfSkippedEvent, LuminosityBlockNumber_t &lumiOfSkippedEvent, EntryNumber_t &skippedEventEntry)
std::vector< EventProcessHistoryID > eventProcessHistoryIDs_
Definition: RootFile.h:328
static Registry * instance()
Definition: Registry.cc:12
std::set< ProductProvenance > readProvenance(unsigned int transitionIndex) const override
Definition: RootFile.cc:2117
DaqProvenanceHelper const * daqProvenanceHelper_
Definition: RootFile.cc:2086
std::string createGlobalIdentifier()
Definition: event.py:1
TypeWithDict const & wrappedType() const
void copyProduct(BranchDescription const &productdesc)
void readEventHistoryTree()
Definition: RootFile.cc:1674
void fillAux(T *&pAux)
Definition: RootTree.h:142
std::set< ProductProvenance > readProvenance(unsigned int) const override
Definition: RootFile.cc:2153
void setEntryNumber(EntryNumber theEntryNumber)
Definition: RootTree.cc:201
void reportOpened(std::string const &inputType)
Definition: RootFile.cc:1203
def operate(timelog, memlog, json_f, num)
roottree::EntryNumber EntryNumber
Definition: RootTree.h:104