CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Types | Public Member Functions | Static Public Member Functions | Private Member Functions | Private Attributes
edm::RootInputFileSequence Class Reference

#include <RootInputFileSequence.h>

Public Types

typedef std::shared_ptr< RootFileRootFileSharedPtr
 

Public Member Functions

void closeFile_ ()
 
bool containedInCurrentFile (RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const
 
void dropUnwantedBranches_ (std::vector< std::string > const &wantedBranches)
 
void endJob ()
 
std::shared_ptr
< BranchIDListHelper const > 
fileBranchIDListHelper () const
 
std::shared_ptr
< ProductRegistry const > 
fileProductRegistry () const
 
ProcessingController::ForwardState forwardState () const
 
InputSource::ItemType getNextItemType (RunNumber_t &run, LuminosityBlockNumber_t &lumi, EventNumber_t &event)
 
bool goToEvent (EventID const &eventID)
 
void initAssociationsFromSecondary (std::set< BranchID > const &)
 
RootInputFileSequenceoperator= (RootInputFileSequence const &)=delete
 
ProcessHistoryRegistry const & processHistoryRegistry () const
 
ProcessHistoryRegistryprocessHistoryRegistryForUpdate ()
 
void readEvent (EventPrincipal &cache)
 
std::unique_ptr< FileBlockreadFile_ ()
 
void readLuminosityBlock_ (LuminosityBlockPrincipal &lumiPrincipal)
 
std::shared_ptr
< LuminosityBlockAuxiliary
readLuminosityBlockAuxiliary_ ()
 
void readOneRandom (EventPrincipal &cache, size_t &fileNameHash, CLHEP::HepRandomEngine *)
 
bool readOneRandomWithID (EventPrincipal &cache, size_t &fileNameHash, LuminosityBlockID const &id, CLHEP::HepRandomEngine *)
 
bool readOneSequential (EventPrincipal &cache, size_t &fileNameHash)
 
bool readOneSequentialWithID (EventPrincipal &cache, size_t &fileNameHash, LuminosityBlockID const &id)
 
void readOneSpecified (EventPrincipal &cache, size_t &fileNameHash, SecondaryEventIDAndFileInfo const &id)
 
void readRun_ (RunPrincipal &runPrincipal)
 
std::shared_ptr< RunAuxiliaryreadRunAuxiliary_ ()
 
ProcessingController::ReverseState reverseState () const
 
void rewind_ ()
 
 RootInputFileSequence (ParameterSet const &pset, PoolSource &input, InputFileCatalog const &catalog, unsigned int nStreams, InputType inputType)
 
 RootInputFileSequence (RootInputFileSequence const &)=delete
 
void skipEntries (unsigned int offset)
 
bool skipEvents (int offset)
 
bool skipToItem (RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash=0U, bool currentFileFirst=true)
 
bool skipToItemInNewFile (RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event)
 
bool skipToItemInNewFile (RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash)
 
virtual ~RootInputFileSequence ()
 

Static Public Member Functions

static void fillDescription (ParameterSetDescription &desc)
 

Private Member Functions

std::vector< FileCatalogItem >
const & 
fileCatalogItems () const
 
void initFile (bool skipBadFiles)
 
bool nextFile ()
 
bool previousFile ()
 
ProcessConfiguration const & processConfiguration () const
 
std::shared_ptr
< ProductRegistry const > 
productRegistry () const
 
ProductRegistryproductRegistryUpdate () const
 
int remainingEvents () const
 
int remainingLuminosityBlocks () const
 
void rewindFile ()
 

Private Attributes

std::vector< BranchIDassociationsFromSecondary_
 
BranchDescription::MatchMode branchesMustMatch_
 
bool bypassVersionCheck_
 
InputFileCatalog const & catalog_
 
bool dropDescendants_
 
std::shared_ptr< DuplicateCheckerduplicateChecker_
 
bool enablePrefetching_
 
std::shared_ptr< EventSkipperByIDeventSkipperByID_
 
int eventsRemainingInFile_
 
std::vector< FileCatalogItem >
::const_iterator 
fileIter_
 
std::vector< FileCatalogItem >
::const_iterator 
fileIterBegin_
 
std::vector< FileCatalogItem >
::const_iterator 
fileIterEnd_
 
std::vector< FileCatalogItem >
::const_iterator 
fileIterLastOpened_
 
std::unique_ptr
< std::unordered_multimap
< size_t, size_t > > 
findFileForSpecifiedID_
 
bool firstFile_
 
std::vector< std::shared_ptr
< IndexIntoFile > > 
indexesIntoFiles_
 
int initialNumberOfEventsToSkip_
 
PoolSourceinput_
 
InputType inputType_
 
bool labelRawDataLikeMC_
 
std::string lfn_
 
size_t lfnHash_
 
bool noEventSort_
 
unsigned int nStreams_
 
std::vector< ProcessHistoryIDorderedProcessHistoryIDs_
 
ProductSelectorRules productSelectorRules_
 
RootFileSharedPtr rootFile_
 
RunNumber_t setRun_
 
bool skipBadFiles_
 
unsigned int treeCacheSize_
 
int const treeMaxVirtualSize_
 
bool usedFallback_
 
bool usingGoToEvent_
 

Detailed Description

Definition at line 40 of file RootInputFileSequence.h.

Member Typedef Documentation

Definition at line 52 of file RootInputFileSequence.h.

Constructor & Destructor Documentation

edm::RootInputFileSequence::RootInputFileSequence ( ParameterSet const &  pset,
PoolSource input,
InputFileCatalog const &  catalog,
unsigned int  nStreams,
InputType  inputType 
)
explicit

Definition at line 30 of file RootInputFileSequence.cc.

References StorageFactory::activateTimeout(), branchesMustMatch_, prof2calltree::count, enablePrefetching_, f, fileIter_, fileIterBegin_, fileIterEnd_, StorageFactory::get(), edm::ParameterSet::getUntrackedParameter(), initFile(), initialNumberOfEventsToSkip_, inputType_, edm::Service< T >::isAvailable(), hltrates_dqm_sourceclient-live_cfg::offset, edm::Primary, productRegistryUpdate(), rootFile_, edm::SecondaryFile, edm::SecondarySource, fileCollector::seed, skipBadFiles_, skipEvents(), StorageFactory::stagein(), edm::BranchDescription::Strict, AlCaHLTBitMon_QueryRunRegistry::string, treeCacheSize_, and edm::ProductRegistry::updateFromInput().

35  :
36  input_(input),
37  inputType_(inputType),
39  firstFile_(true),
40  lfn_("unknown"),
41  lfnHash_(0U),
42  findFileForSpecifiedID_(nullptr),
47  rootFile_(),
51  nStreams_(nStreams),
54  // The default value provided as the second argument to the getUntrackedParameter function call
55  // is not used when the ParameterSet has been validated and the parameters are not optional
56  // in the description. This is currently true when PoolSource is the primary input source.
57  // The modules that use PoolSource as a SecSource have not defined their fillDescriptions function
58  // yet, so the ParameterSet does not get validated yet. As soon as all the modules with a SecSource
59  // have defined descriptions, the defaults in the getUntrackedParameterSet function calls can
60  // and should be deleted from the code.
61  initialNumberOfEventsToSkip_(inputType != InputType::SecondaryFile ? pset.getUntrackedParameter<unsigned int>("skipEvents", 0U) : 0U),
62  noEventSort_(inputType == InputType::Primary ? pset.getUntrackedParameter<bool>("noEventSort", true) : false),
63  skipBadFiles_(pset.getUntrackedParameter<bool>("skipBadFiles", false)),
64  bypassVersionCheck_(pset.getUntrackedParameter<bool>("bypassVersionCheck", false)),
65  treeCacheSize_(noEventSort_ ? pset.getUntrackedParameter<unsigned int>("cacheSize", roottree::defaultCacheSize) : 0U),
66  treeMaxVirtualSize_(pset.getUntrackedParameter<int>("treeMaxVirtualSize", -1)),
67  setRun_(pset.getUntrackedParameter<unsigned int>("setRunNumber", 0U)),
68  productSelectorRules_(pset, "inputCommands", "InputSource"),
69  duplicateChecker_(inputType == InputType::Primary ? new DuplicateChecker(pset) : 0),
70  dropDescendants_(pset.getUntrackedParameter<bool>("dropDescendantsOfDroppedBranches", inputType != InputType::SecondarySource)),
71  labelRawDataLikeMC_(pset.getUntrackedParameter<bool>("labelRawDataLikeMC", true)),
72  usingGoToEvent_(false),
73  enablePrefetching_(false),
74  usedFallback_(false) {
75 
76  // The SiteLocalConfig controls the TTreeCache size and the prefetching settings.
78  if(pSLC.isAvailable()) {
79  if(treeCacheSize_ != 0U && pSLC->sourceTTreeCacheSize()) {
80  treeCacheSize_ = *(pSLC->sourceTTreeCacheSize());
81  }
82  enablePrefetching_ = pSLC->enablePrefetching();
83  }
84 
85  std::string branchesMustMatch = pset.getUntrackedParameter<std::string>("branchesMustMatch", std::string("permissive"));
86  if(branchesMustMatch == std::string("strict")) branchesMustMatch_ = BranchDescription::Strict;
87 
89 
90  if(inputType == InputType::SecondarySource) {
91  // For the secondary input source we do not stage in,
92  // and we randomly choose the first file to open.
93  // We cannot use the random number service yet.
94  std::ifstream f("/dev/urandom");
95  unsigned int seed;
96  f.read(reinterpret_cast<char*>(&seed), sizeof(seed));
97  std::default_random_engine dre(seed);
99  std::uniform_int_distribution<int> distribution(0, count - 1);
100  while(!rootFile_ && count != 0) {
101  --count;
102  int offset = distribution(dre);
103  fileIter_ = fileIterBegin_ + offset;
105  }
106  } else {
107  // Prestage the files
108  for(fileIter_ = fileIterBegin_; fileIter_ != fileIterEnd_; ++fileIter_) {
109  factory->activateTimeout(fileIter_->fileName());
110  factory->stagein(fileIter_->fileName());
111  //NOTE: we do not want to stage in all secondary files since we can be given a list of
112  // thousands of files and prestaging all those files can cause a site to fail.
113  // So, we stage in the first secondary file only.
115  break;
116  }
117  }
118  // Open the first file.
119  for(fileIter_ = fileIterBegin_; fileIter_ != fileIterEnd_; ++fileIter_) {
121  if(rootFile_) break;
122  }
123  }
124  if(rootFile_) {
125  productRegistryUpdate().updateFromInput(rootFile_->productRegistry()->productList());
126  if(inputType == InputType::Primary && initialNumberOfEventsToSkip_ != 0) {
128  }
129  }
130  }
void stagein(const std::string &url)
void initFile(bool skipBadFiles)
ProductSelectorRules productSelectorRules_
std::vector< FileCatalogItem >::const_iterator fileIter_
std::shared_ptr< EventSkipperByID > eventSkipperByID_
ProductRegistry & productRegistryUpdate() const
unsigned int const defaultCacheSize
Definition: RootTree.h:37
static std::string const input
Definition: EdmProvDump.cc:43
static StorageFactory * get(void)
std::shared_ptr< DuplicateChecker > duplicateChecker_
std::vector< FileCatalogItem >::const_iterator fileIterLastOpened_
std::vector< FileCatalogItem > const & fileCatalogItems() const
double f[11][100]
#define end
Definition: vmac.h:37
std::vector< std::shared_ptr< IndexIntoFile > > indexesIntoFiles_
std::unique_ptr< std::unordered_multimap< size_t, size_t > > findFileForSpecifiedID_
std::vector< ProcessHistoryID > orderedProcessHistoryIDs_
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
#define begin
Definition: vmac.h:30
void updateFromInput(ProductList const &other)
void activateTimeout(const std::string &url)
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
BranchDescription::MatchMode branchesMustMatch_
static std::auto_ptr< EventSkipperByID > create(ParameterSet const &pset)
tuple size
Write out results.
InputFileCatalog const & catalog_
edm::RootInputFileSequence::~RootInputFileSequence ( )
virtual

Definition at line 395 of file RootInputFileSequence.cc.

395  {
396  }
edm::RootInputFileSequence::RootInputFileSequence ( RootInputFileSequence const &  )
delete

Member Function Documentation

void edm::RootInputFileSequence::closeFile_ ( )

Definition at line 161 of file RootInputFileSequence.cc.

References duplicateChecker_, input_, inputType_, lfn_, edm::Primary, rootFile_, edm::SecondarySource, and usedFallback_.

Referenced by endJob(), initFile(), and rewind_().

161  {
162  // close the currently open file, if any, and delete the RootFile object.
163  if(rootFile_) {
165  std::unique_ptr<InputSource::FileCloseSentry>
166  sentry((inputType_ == InputType::Primary) ? new InputSource::FileCloseSentry(input_, lfn_, usedFallback_) : 0);
167  rootFile_->close();
168  if(duplicateChecker_) duplicateChecker_->inputFileClosed();
169  }
170  rootFile_.reset();
171  }
172  }
std::shared_ptr< DuplicateChecker > duplicateChecker_
bool edm::RootInputFileSequence::containedInCurrentFile ( RunNumber_t  run,
LuminosityBlockNumber_t  lumi,
EventNumber_t  event 
) const

Definition at line 467 of file RootInputFileSequence.cc.

References rootFile_.

467  {
468  if(!rootFile_) return false;
469  return rootFile_->containsItem(run, lumi, event);
470  }
tuple lumi
Definition: fjr2json.py:35
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
void edm::RootInputFileSequence::dropUnwantedBranches_ ( std::vector< std::string > const &  wantedBranches)

Definition at line 692 of file RootInputFileSequence.cc.

References edm::ParameterSet::addUntrackedParameter(), productSelectorRules_, cmsCodeRulesChecker::rules, and AlCaHLTBitMon_QueryRunRegistry::string.

692  {
693  std::vector<std::string> rules;
694  rules.reserve(wantedBranches.size() + 1);
695  rules.emplace_back("drop *");
696  for(std::string const& branch : wantedBranches) {
697  rules.push_back("keep " + branch + "_*");
698  }
699  ParameterSet pset;
700  pset.addUntrackedParameter("inputCommands", rules);
701  productSelectorRules_ = ProductSelectorRules(pset, "inputCommands", "InputSource");
702  }
ProductSelectorRules productSelectorRules_
void edm::RootInputFileSequence::endJob ( void  )

Definition at line 138 of file RootInputFileSequence.cc.

References closeFile_().

138  {
139  closeFile_();
140  }
std::shared_ptr< BranchIDListHelper const > edm::RootInputFileSequence::fileBranchIDListHelper ( ) const

Definition at line 341 of file RootInputFileSequence.cc.

References assert(), and rootFile_.

341  {
342  assert(rootFile_);
343  return rootFile_->branchIDListHelper();
344  }
assert(m_qm.get())
std::vector< FileCatalogItem > const & edm::RootInputFileSequence::fileCatalogItems ( ) const
private

Definition at line 133 of file RootInputFileSequence.cc.

References catalog_, and edm::InputFileCatalog::fileCatalogItems().

Referenced by readOneRandom().

133  {
134  return catalog_.fileCatalogItems();
135  }
std::vector< FileCatalogItem > const & fileCatalogItems() const
InputFileCatalog const & catalog_
std::shared_ptr< ProductRegistry const > edm::RootInputFileSequence::fileProductRegistry ( ) const

Definition at line 335 of file RootInputFileSequence.cc.

References assert(), and rootFile_.

335  {
336  assert(rootFile_);
337  return rootFile_->productRegistry();
338  }
assert(m_qm.get())
void edm::RootInputFileSequence::fillDescription ( ParameterSetDescription desc)
static

Definition at line 904 of file RootInputFileSequence.cc.

References edm::ParameterSetDescription::addUntracked(), edm::roottree::defaultCacheSize, edm::EventSkipperByID::fillDescription(), edm::ProductSelectorRules::fillDescription(), edm::DuplicateChecker::fillDescription(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by edm::PoolSource::fillDescriptions().

904  {
905  desc.addUntracked<unsigned int>("skipEvents", 0U)
906  ->setComment("Skip the first 'skipEvents' events that otherwise would have been processed.");
907  desc.addUntracked<bool>("noEventSort", true)
908  ->setComment("True: Process runs, lumis and events in the order they appear in the file (but see notes 1 and 2).\n"
909  "False: Process runs, lumis and events in each file in numerical order (run#, lumi#, event#) (but see note 3).\n"
910  "Note 1: Events within the same lumi will always be processed contiguously.\n"
911  "Note 2: Lumis within the same run will always be processed contiguously.\n"
912  "Note 3: Any sorting occurs independently in each input file (no sorting across input files).");
913  desc.addUntracked<bool>("skipBadFiles", false)
914  ->setComment("True: Ignore any missing or unopenable input file.\n"
915  "False: Throw exception if missing or unopenable input file.");
916  desc.addUntracked<bool>("bypassVersionCheck", false)
917  ->setComment("True: Bypass release version check.\n"
918  "False: Throw exception if reading file in a release prior to the release in which the file was written.");
919  desc.addUntracked<unsigned int>("cacheSize", roottree::defaultCacheSize)
920  ->setComment("Size of ROOT TTree prefetch cache. Affects performance.");
921  desc.addUntracked<int>("treeMaxVirtualSize", -1)
922  ->setComment("Size of ROOT TTree TBasket cache. Affects performance.");
923  desc.addUntracked<unsigned int>("setRunNumber", 0U)
924  ->setComment("If non-zero, change number of first run to this number. Apply same offset to all runs. Allowed only for simulation.");
925  desc.addUntracked<bool>("dropDescendantsOfDroppedBranches", true)
926  ->setComment("If True, also drop on input any descendent of any branch dropped on input.");
927  std::string defaultString("permissive");
928  desc.addUntracked<std::string>("branchesMustMatch", defaultString)
929  ->setComment("'strict': Branches in each input file must match those in the first file.\n"
930  "'permissive': Branches in each input file may be any subset of those in the first file.");
931  desc.addUntracked<bool>("labelRawDataLikeMC", true)
932  ->setComment("If True: replace module label for raw data to match MC. Also use 'LHC' as process.");
933 
934  ProductSelectorRules::fillDescription(desc, "inputCommands");
937  }
static void fillDescription(ParameterSetDescription &desc, char const *parameterName)
static void fillDescription(ParameterSetDescription &desc)
unsigned int const defaultCacheSize
Definition: RootTree.h:37
static void fillDescription(ParameterSetDescription &desc)
ProcessingController::ForwardState edm::RootInputFileSequence::forwardState ( ) const
InputSource::ItemType edm::RootInputFileSequence::getNextItemType ( RunNumber_t run,
LuminosityBlockNumber_t lumi,
EventNumber_t event 
)

Definition at line 442 of file RootInputFileSequence.cc.

References assert(), fileIter_, fileIterEnd_, firstFile_, edm::InputSource::IsEvent, edm::InputSource::IsFile, edm::InputSource::IsLumi, edm::InputSource::IsRun, edm::InputSource::IsStop, edm::IndexIntoFile::kEnd, edm::IndexIntoFile::kEvent, edm::IndexIntoFile::kLumi, edm::IndexIntoFile::kRun, and rootFile_.

442  {
443  if(fileIter_ == fileIterEnd_) {
444  return InputSource::IsStop;
445  }
446  if(firstFile_) {
447  return InputSource::IsFile;
448  }
449  if(rootFile_) {
450  IndexIntoFile::EntryType entryType = rootFile_->getNextItemType(run, lumi, event);
451  if(entryType == IndexIntoFile::kEvent) {
452  return InputSource::IsEvent;
453  } else if(entryType == IndexIntoFile::kLumi) {
454  return InputSource::IsLumi;
455  } else if(entryType == IndexIntoFile::kRun) {
456  return InputSource::IsRun;
457  }
458  assert(entryType == IndexIntoFile::kEnd);
459  }
460  if(fileIter_ + 1 == fileIterEnd_) {
461  return InputSource::IsStop;
462  }
463  return InputSource::IsFile;
464  }
std::vector< FileCatalogItem >::const_iterator fileIter_
tuple lumi
Definition: fjr2json.py:35
assert(m_qm.get())
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
bool edm::RootInputFileSequence::goToEvent ( EventID const &  eventID)

Definition at line 519 of file RootInputFileSequence.cc.

References assert(), edm::EventID::event(), fileIter_, fileIterBegin_, newFWLiteAna::found, indexesIntoFiles_, initFile(), edm::EventID::luminosityBlock(), rootFile_, edm::EventID::run(), and usingGoToEvent_.

519  {
520  usingGoToEvent_ = true;
521  if(rootFile_) {
522  if(rootFile_->goToEvent(eventID)) {
523  return true;
524  }
525  // If only one input file, give up now, to save time.
526  if(rootFile_ && indexesIntoFiles_.size() == 1) {
527  return false;
528  }
529  // Save the current file and position so that we can restore them
530  // if we fail to restore the desired event
531  bool closedOriginalFile = false;
532  std::vector<FileCatalogItem>::const_iterator originalFile = fileIter_;
533  IndexIntoFile::IndexIntoFileItr originalPosition = rootFile_->indexIntoFileIter();
534 
535  // Look for item (run/lumi/event) in files previously opened without reopening unnecessary files.
536  typedef std::vector<std::shared_ptr<IndexIntoFile> >::const_iterator Iter;
537  for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
538  if(*it && (*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) {
539  // We found it. Close the currently open file, and open the correct one.
540  fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
541  initFile(false);
542  // Now get the item from the correct file.
543  assert(rootFile_);
544  bool found = rootFile_->goToEvent(eventID);
545  assert(found);
546  return true;
547  }
548  }
549  // Look for item in files not yet opened.
550  for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
551  if(!*it) {
552  fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
553  initFile(false);
554  closedOriginalFile = true;
555  if((*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) {
556  assert(rootFile_);
557  if(rootFile_->goToEvent(eventID)) {
558  return true;
559  }
560  }
561  }
562  }
563  if(closedOriginalFile) {
564  fileIter_ = originalFile;
565  initFile(false);
566  assert(rootFile_);
567  rootFile_->setPosition(originalPosition);
568  }
569  }
570  return false;
571  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
assert(m_qm.get())
std::vector< std::shared_ptr< IndexIntoFile > > indexesIntoFiles_
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
void edm::RootInputFileSequence::initAssociationsFromSecondary ( std::set< BranchID > const &  associationsFromSecondary)

Definition at line 969 of file RootInputFileSequence.cc.

References associationsFromSecondary_, and rootFile_.

969  {
970  for(auto const& branchID : associationsFromSecondary) {
971  associationsFromSecondary_.push_back(branchID);
972  }
973  rootFile_->initAssociationsFromSecondary(associationsFromSecondary_);
974  }
std::vector< BranchID > associationsFromSecondary_
void edm::RootInputFileSequence::initFile ( bool  skipBadFiles)
private

Definition at line 174 of file RootInputFileSequence.cc.

References StorageFactory::activateTimeout(), cms::Exception::additionalInfo(), assert(), associationsFromSecondary_, edm::InputSource::branchIDListHelper(), bypassVersionCheck_, closeFile_(), dropDescendants_, duplicateChecker_, alignCSCRings::e, enablePrefetching_, eventSkipperByID_, edm::hlt::Exception, cms::Exception::explainSelf(), edm::errors::FallbackFileOpenError, fileIter_, fileIterBegin_, fileIterEnd_, fileIterLastOpened_, edm::errors::FileOpenError, StorageFactory::get(), indexesIntoFiles_, initialNumberOfEventsToSkip_, input_, RPC_Client_on_RootFile::InputFile, inputType_, labelRawDataLikeMC_, lfn_, lfnHash_, noEventSort_, nStreams_, orderedProcessHistoryIDs_, dbtoconf::out, dbtoconf::pfn, edm::Primary, processConfiguration(), processHistoryRegistryForUpdate(), edm::InputSource::processingMode(), productSelectorRules_, remainingEvents(), remainingLuminosityBlocks(), edm::InputFile::reportFallbackAttempt(), edm::InputFile::reportSkippedFile(), rootFile_, alignCSCRings::s, edm::SecondaryFile, edm::SecondarySource, setRun_, AlCaHLTBitMon_QueryRunRegistry::string, edm::InputSource::thinnedAssociationsHelper(), treeCacheSize_, treeMaxVirtualSize_, usedFallback_, and usingGoToEvent_.

Referenced by goToEvent(), nextFile(), previousFile(), readFile_(), readOneRandom(), readOneSequential(), readOneSequentialWithID(), rewind_(), RootInputFileSequence(), skipEntries(), skipToItem(), and skipToItemInNewFile().

174  {
175  // We are really going to close the open file.
176 
177  // If this is the primary sequence, we are not duplicate checking across files
178  // and we are not using random access to find events, then we can delete the
179  // IndexIntoFile for the file we are closing. If we can't delete all of it,
180  // then we can delete the parts we do not need.
182  size_t currentIndexIntoFile = fileIterLastOpened_ - fileIterBegin_;
183  bool needIndexesForDuplicateChecker = duplicateChecker_ && duplicateChecker_->checkingAllFiles() && !duplicateChecker_->checkDisabled();
184  bool deleteIndexIntoFile = inputType_ == InputType::Primary &&
185  !needIndexesForDuplicateChecker &&
187  if(deleteIndexIntoFile) {
188  indexesIntoFiles_[currentIndexIntoFile].reset();
189  } else {
190  if(indexesIntoFiles_[currentIndexIntoFile]) indexesIntoFiles_[currentIndexIntoFile]->inputFileClosed();
191  }
193  }
194  closeFile_();
195 
196  if(fileIter_ == fileIterEnd_) {
197  // No files specified
198  return;
199  }
200 
201  // Check if the logical file name was found.
202  if(fileIter_->fileName().empty()) {
203  // LFN not found in catalog.
204  InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
205  if(!skipBadFiles) {
206  throw cms::Exception("LogicalFileNameNotFound", "RootInputFileSequence::initFile()\n")
207  << "Logical file name '" << fileIter_->logicalFileName() << "' was not found in the file catalog.\n"
208  << "If you wanted a local file, you forgot the 'file:' prefix\n"
209  << "before the file name in your configuration file.\n";
210  }
211  LogWarning("") << "Input logical file: " << fileIter_->logicalFileName() << " was not found in the catalog, and will be skipped.\n";
212  return;
213  }
214 
215  lfn_ = fileIter_->logicalFileName().empty() ? fileIter_->fileName() : fileIter_->logicalFileName();
216  lfnHash_ = std::hash<std::string>()(lfn_);
217  usedFallback_ = false;
218 
219  // Determine whether we have a fallback URL specified; if so, prepare it;
220  // Only valid if it is non-empty and differs from the original filename.
221  std::string fallbackName = fileIter_->fallbackFileName();
222  bool hasFallbackUrl = !fallbackName.empty() && fallbackName != fileIter_->fileName();
223 
224  std::shared_ptr<InputFile> filePtr;
225  std::list<std::string> originalInfo;
226  try {
227  std::unique_ptr<InputSource::FileOpenSentry>
228  sentry(inputType_ == InputType::Primary ? new InputSource::FileOpenSentry(input_, lfn_, usedFallback_) : 0);
229  filePtr.reset(new InputFile(gSystem->ExpandPathName(fileIter_->fileName().c_str()), " Initiating request to open file ", inputType_));
230  }
231  catch (cms::Exception const& e) {
232  if(!skipBadFiles) {
233  if(hasFallbackUrl) {
234  std::ostringstream out;
235  out << e.explainSelf();
236  std::string pfn(gSystem->ExpandPathName(fallbackName.c_str()));
237  InputFile::reportFallbackAttempt(pfn, fileIter_->logicalFileName(), out.str());
238  originalInfo = e.additionalInfo();
239  } else {
240  InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
241  Exception ex(errors::FileOpenError, "", e);
242  ex.addContext("Calling RootInputFileSequence::initFile()");
243  std::ostringstream out;
244  out << "Input file " << fileIter_->fileName() << " could not be opened.";
245  ex.addAdditionalInfo(out.str());
246  throw ex;
247  }
248  }
249  }
250  if(!filePtr && (hasFallbackUrl)) {
251  try {
252  usedFallback_ = true;
253  std::unique_ptr<InputSource::FileOpenSentry>
254  sentry(inputType_ == InputType::Primary ? new InputSource::FileOpenSentry(input_, lfn_, usedFallback_) : 0);
255  std::string fallbackFullName = gSystem->ExpandPathName(fallbackName.c_str());
256  StorageFactory *factory = StorageFactory::get();
257  if (factory) {factory->activateTimeout(fallbackFullName);}
258  filePtr.reset(new InputFile(fallbackFullName.c_str(), " Fallback request to file ", inputType_));
259  }
260  catch (cms::Exception const& e) {
261  if(!skipBadFiles) {
262  InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
264  ex.addContext("Calling RootInputFileSequence::initFile()");
265  std::ostringstream out;
266  out << "Input file " << fileIter_->fileName() << " could not be opened.\n";
267  out << "Fallback Input file " << fallbackName << " also could not be opened.";
268  if (originalInfo.size()) {
269  out << std::endl << "Original exception info is above; fallback exception info is below.";
270  ex.addAdditionalInfo(out.str());
271  for (auto const & s : originalInfo) {
272  ex.addAdditionalInfo(s);
273  }
274  } else {
275  ex.addAdditionalInfo(out.str());
276  }
277  throw ex;
278  }
279  }
280  }
281  if(filePtr) {
282  std::vector<std::shared_ptr<IndexIntoFile> >::size_type currentIndexIntoFile = fileIter_ - fileIterBegin_;
283  rootFile_ = RootFileSharedPtr(new RootFile(
284  fileIter_->fileName(),
286  fileIter_->logicalFileName(),
287  filePtr,
290  remainingEvents(),
292  nStreams_,
296  setRun_,
297  noEventSort_,
299  inputType_,
300  (inputType_ == InputType::SecondarySource ? std::make_shared<BranchIDListHelper>() : input_.branchIDListHelper()),
301  (inputType_ == InputType::SecondarySource ? std::shared_ptr<ThinnedAssociationsHelper>() : input_.thinnedAssociationsHelper()),
307  currentIndexIntoFile,
313 
314  assert(rootFile_);
316  indexesIntoFiles_[currentIndexIntoFile] = rootFile_->indexIntoFileSharedPtr();
317  char const* inputType = 0;
318  switch(inputType_) {
319  case InputType::Primary: inputType = "primaryFiles"; break;
320  case InputType::SecondaryFile: inputType = "secondaryFiles"; break;
321  case InputType::SecondarySource: inputType = "mixingFiles"; break;
322  }
323  rootFile_->reportOpened(inputType);
324  } else {
325  InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
326  if(!skipBadFiles) {
328  "RootInputFileSequence::initFile(): Input file " << fileIter_->fileName() << " was not found or could not be opened.\n";
329  }
330  LogWarning("") << "Input file: " << fileIter_->fileName() << " was not found or could not be opened, and will be skipped.\n";
331  }
332  }
list pfn
Definition: dbtoconf.py:76
ProductSelectorRules productSelectorRules_
ProcessConfiguration const & processConfiguration() const
virtual std::string explainSelf() const
Definition: Exception.cc:146
std::vector< FileCatalogItem >::const_iterator fileIter_
std::shared_ptr< EventSkipperByID > eventSkipperByID_
assert(m_qm.get())
std::shared_ptr< ThinnedAssociationsHelper > thinnedAssociationsHelper() const
Accessor for thinnedAssociationsHelper.
Definition: InputSource.h:181
ProcessingMode processingMode() const
RunsLumisAndEvents (default), RunsAndLumis, or Runs.
Definition: InputSource.h:257
uint16_t size_type
std::list< std::string > const & additionalInfo() const
Definition: Exception.cc:195
static StorageFactory * get(void)
std::shared_ptr< RootFile > RootFileSharedPtr
tuple InputFile
Open Root file and provide MEs ############.
std::vector< BranchID > associationsFromSecondary_
static void reportFallbackAttempt(std::string const &pfn, std::string const &logicalFileName, std::string const &errorMessage)
Definition: InputFile.cc:86
std::shared_ptr< DuplicateChecker > duplicateChecker_
std::vector< FileCatalogItem >::const_iterator fileIterLastOpened_
std::shared_ptr< BranchIDListHelper > branchIDListHelper() const
Accessor for branchIDListHelper.
Definition: InputSource.h:178
static void reportSkippedFile(std::string const &fileName, std::string const &logicalFileName)
Definition: InputFile.cc:80
std::vector< std::shared_ptr< IndexIntoFile > > indexesIntoFiles_
tuple out
Definition: dbtoconf.py:99
std::vector< ProcessHistoryID > orderedProcessHistoryIDs_
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
tuple skipBadFiles
Definition: example_cfg.py:64
ProcessHistoryRegistry & processHistoryRegistryForUpdate()
void activateTimeout(const std::string &url)
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
bool edm::RootInputFileSequence::nextFile ( )
private

Definition at line 346 of file RootInputFileSequence.cc.

References branchesMustMatch_, edm::hlt::Exception, fileIter_, fileIterBegin_, fileIterEnd_, initFile(), inputType_, edm::ProductRegistry::merge(), edm::errors::MismatchedInputFiles, edm::Primary, productRegistryUpdate(), rootFile_, skipBadFiles_, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by readFile_(), and skipEvents().

346  {
348  if(fileIter_ == fileIterEnd_) {
350  return false;
351  } else {
353  }
354  }
355 
357 
359  // make sure the new product registry is compatible with the main one
360  std::string mergeInfo = productRegistryUpdate().merge(*rootFile_->productRegistry(),
361  fileIter_->fileName(),
363  if(!mergeInfo.empty()) {
364  throw Exception(errors::MismatchedInputFiles,"RootInputFileSequence::nextFile()") << mergeInfo;
365  }
366  }
367  return true;
368  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
ProductRegistry & productRegistryUpdate() const
std::string merge(ProductRegistry const &other, std::string const &fileName, BranchDescription::MatchMode branchesMustMatch=BranchDescription::Permissive)
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
BranchDescription::MatchMode branchesMustMatch_
RootInputFileSequence& edm::RootInputFileSequence::operator= ( RootInputFileSequence const &  )
delete
bool edm::RootInputFileSequence::previousFile ( )
private

Definition at line 370 of file RootInputFileSequence.cc.

References branchesMustMatch_, edm::hlt::Exception, fileIter_, fileIterBegin_, fileIterEnd_, initFile(), inputType_, edm::ProductRegistry::merge(), edm::errors::MismatchedInputFiles, edm::Primary, productRegistryUpdate(), rootFile_, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by skipEvents().

370  {
371  if(fileIter_ == fileIterBegin_) {
373  return false;
374  } else {
376  }
377  }
378  --fileIter_;
379 
380  initFile(false);
381 
383  // make sure the new product registry is compatible to the main one
384  std::string mergeInfo = productRegistryUpdate().merge(*rootFile_->productRegistry(),
385  fileIter_->fileName(),
387  if(!mergeInfo.empty()) {
388  throw Exception(errors::MismatchedInputFiles,"RootInputFileSequence::previousEvent()") << mergeInfo;
389  }
390  }
391  if(rootFile_) rootFile_->setToLastEntry();
392  return true;
393  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
ProductRegistry & productRegistryUpdate() const
std::string merge(ProductRegistry const &other, std::string const &fileName, BranchDescription::MatchMode branchesMustMatch=BranchDescription::Permissive)
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
BranchDescription::MatchMode branchesMustMatch_
ProcessConfiguration const & edm::RootInputFileSequence::processConfiguration ( ) const
private

Definition at line 667 of file RootInputFileSequence.cc.

References input_, and edm::InputSource::processConfiguration().

Referenced by initFile().

667  {
668  return input_.processConfiguration();
669  }
ProcessConfiguration const & processConfiguration() const
Accessor for Process Configuration.
Definition: InputSource.h:212
ProcessHistoryRegistry const & edm::RootInputFileSequence::processHistoryRegistry ( ) const

Definition at line 657 of file RootInputFileSequence.cc.

References input_, and edm::InputSource::processHistoryRegistry().

657  {
659  }
ProcessHistoryRegistry const & processHistoryRegistry() const
Const accessor for process history registry.
Definition: InputSource.h:172
ProcessHistoryRegistry & edm::RootInputFileSequence::processHistoryRegistryForUpdate ( )

Definition at line 662 of file RootInputFileSequence.cc.

References input_, and edm::InputSource::processHistoryRegistryForUpdate().

Referenced by initFile().

662  {
664  }
ProcessHistoryRegistry & processHistoryRegistryForUpdate()
Non-const accessor for process history registry.
Definition: InputSource.h:175
std::shared_ptr< ProductRegistry const > edm::RootInputFileSequence::productRegistry ( ) const
private

Definition at line 687 of file RootInputFileSequence.cc.

References input_, and edm::InputSource::productRegistry().

687  {
688  return input_.productRegistry();
689  }
std::shared_ptr< ProductRegistry const > productRegistry() const
Accessor for product registry.
Definition: InputSource.h:169
ProductRegistry & edm::RootInputFileSequence::productRegistryUpdate ( ) const
private

Definition at line 682 of file RootInputFileSequence.cc.

References input_, and edm::InputSource::productRegistryUpdate().

Referenced by nextFile(), previousFile(), and RootInputFileSequence().

682  {
683  return input_.productRegistryUpdate();
684  }
ProductRegistry & productRegistryUpdate() const
Definition: InputSource.h:351
void edm::RootInputFileSequence::readEvent ( EventPrincipal cache)

Definition at line 436 of file RootInputFileSequence.cc.

References assert(), and rootFile_.

436  {
437  assert(rootFile_);
438  rootFile_->readEvent(eventPrincipal);
439  }
assert(m_qm.get())
std::unique_ptr< FileBlock > edm::RootInputFileSequence::readFile_ ( )

Definition at line 143 of file RootInputFileSequence.cc.

References assert(), firstFile_, initFile(), nextFile(), rootFile_, and skipBadFiles_.

143  {
144  if(firstFile_) {
145  // The first input file has already been opened.
146  firstFile_ = false;
147  if(!rootFile_) {
149  }
150  } else {
151  if(!nextFile()) {
152  assert(0);
153  }
154  }
155  if(!rootFile_) {
156  return std::unique_ptr<FileBlock>(new FileBlock);
157  }
158  return rootFile_->createFileBlock();
159  }
void initFile(bool skipBadFiles)
assert(m_qm.get())
void edm::RootInputFileSequence::readLuminosityBlock_ ( LuminosityBlockPrincipal lumiPrincipal)

Definition at line 417 of file RootInputFileSequence.cc.

References assert(), and rootFile_.

417  {
418  assert(rootFile_);
419  rootFile_->readLuminosityBlock_(lumiPrincipal);
420  }
assert(m_qm.get())
std::shared_ptr< LuminosityBlockAuxiliary > edm::RootInputFileSequence::readLuminosityBlockAuxiliary_ ( )

Definition at line 405 of file RootInputFileSequence.cc.

References assert(), and rootFile_.

405  {
406  assert(rootFile_);
407  return rootFile_->readLuminosityBlockAuxiliary_();
408  }
assert(m_qm.get())
void edm::RootInputFileSequence::readOneRandom ( EventPrincipal cache,
size_t &  fileNameHash,
CLHEP::HepRandomEngine *  engine 
)

Definition at line 825 of file RootInputFileSequence.cc.

References assert(), edm::errors::Configuration, eventsRemainingInFile_, edm::hlt::Exception, fileCatalogItems(), fileIter_, fileIterBegin_, fileIterEnd_, firstFile_, newFWLiteAna::found, initFile(), lfnHash_, edm::errors::NotFound, rootFile_, findQualityFiles::size, and skipBadFiles_.

825  {
826  firstFile_ = false;
828  throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandom(): no input files specified for secondary input source.\n";
829  }
830  assert(rootFile_);
831  skipBadFiles_ = false;
832  unsigned int currentSeqNumber = fileIter_ - fileIterBegin_;
833  while(eventsRemainingInFile_ == 0) {
834 
835  fileIter_ = fileIterBegin_ + CLHEP::RandFlat::shootInt(engine, fileCatalogItems().size());
836  unsigned int newSeqNumber = fileIter_ - fileIterBegin_;
837  if(newSeqNumber != currentSeqNumber) {
838  initFile(false);
839  currentSeqNumber = newSeqNumber;
840  }
841  eventsRemainingInFile_ = rootFile_->eventTree().entries();
842  if(eventsRemainingInFile_ == 0) {
843  throw Exception(errors::NotFound) <<
844  "RootInputFileSequence::readOneRandom(): Secondary Input file " << fileIter_->fileName() << " contains no events.\n";
845  }
846  rootFile_->setAtEventEntry(CLHEP::RandFlat::shootInt(engine, eventsRemainingInFile_) - 1);
847  }
848  rootFile_->nextEventEntry();
849 
850  bool found = rootFile_->readCurrentEvent(cache);
851  if(!found) {
852  rootFile_->setAtEventEntry(0);
853  bool found = rootFile_->readCurrentEvent(cache);
854  assert(found);
855  }
856  fileNameHash = lfnHash_;
858  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
assert(m_qm.get())
std::vector< FileCatalogItem > const & fileCatalogItems() const
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
tuple size
Write out results.
bool edm::RootInputFileSequence::readOneRandomWithID ( EventPrincipal cache,
size_t &  fileNameHash,
LuminosityBlockID const &  id,
CLHEP::HepRandomEngine *  engine 
)

Definition at line 863 of file RootInputFileSequence.cc.

References assert(), edm::errors::Configuration, edm::hlt::Exception, fileIter_, fileIterBegin_, fileIterEnd_, firstFile_, newFWLiteAna::found, i, lfnHash_, rootFile_, DTTTrigCorrFirst::run, skipBadFiles_, and skipToItem().

863  {
864  firstFile_ = false;
866  throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandomWithID(): no input files specified for secondary input source.\n";
867  }
868  skipBadFiles_ = false;
869  if(fileIter_ == fileIterEnd_ || !rootFile_ ||
870  rootFile_->indexIntoFileIter().run() != id.run() ||
871  rootFile_->indexIntoFileIter().lumi() != id.luminosityBlock()) {
872  bool found = skipToItem(id.run(), id.luminosityBlock(), 0);
873  if(!found) {
874  return false;
875  }
876  int eventsInLumi = 0;
877  assert(rootFile_);
878  while(rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock())) ++eventsInLumi;
879  found = skipToItem(id.run(), id.luminosityBlock(), 0);
880  assert(found);
881  int eventInLumi = CLHEP::RandFlat::shootInt(engine, eventsInLumi);
882  for(int i = 0; i < eventInLumi; ++i) {
883  bool found = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
884  assert(found);
885  }
886  }
887  assert(rootFile_);
888  bool found = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
889  if(found) {
890  found = rootFile_->readCurrentEvent(cache);
891  }
892  if(!found) {
893  bool found = rootFile_->setEntryAtItem(id.run(), id.luminosityBlock(), 0);
894  if(!found) {
895  return false;
896  }
897  return readOneRandomWithID(cache, fileNameHash, id, engine);
898  }
899  fileNameHash = lfnHash_;
900  return true;
901  }
int i
Definition: DBlmapReader.cc:9
std::vector< FileCatalogItem >::const_iterator fileIter_
assert(m_qm.get())
bool readOneRandomWithID(EventPrincipal &cache, size_t &fileNameHash, LuminosityBlockID const &id, CLHEP::HepRandomEngine *)
bool skipToItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash=0U, bool currentFileFirst=true)
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
bool edm::RootInputFileSequence::readOneSequential ( EventPrincipal cache,
size_t &  fileNameHash 
)

Definition at line 721 of file RootInputFileSequence.cc.

References assert(), edm::errors::Configuration, edm::hlt::Exception, fileIter_, fileIterBegin_, fileIterEnd_, firstFile_, newFWLiteAna::found, initFile(), initialNumberOfEventsToSkip_, edm::IndexIntoFile::invalidEntry, lfnHash_, rootFile_, skipBadFiles_, and skipEntries().

721  {
722  skipBadFiles_ = false;
723  if(firstFile_) {
724  firstFile_ = false;
726  throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequential(): no input files specified for secondary input source.\n";
727  }
728  if(fileIter_ != fileIterBegin_) {
730  initFile(false);
731  }
732  assert(rootFile_);
733  rootFile_->setAtEventEntry(IndexIntoFile::invalidEntry);
735  }
736  assert(rootFile_);
737  rootFile_->nextEventEntry();
738  bool found = rootFile_->readCurrentEvent(cache);
739  if(!found) {
740  ++fileIter_;
741  if(fileIter_ == fileIterEnd_) {
743  }
744  initFile(false);
745  assert(rootFile_);
746  rootFile_->setAtEventEntry(IndexIntoFile::invalidEntry);
747  return readOneSequential(cache, fileNameHash);
748  }
749  fileNameHash = lfnHash_;
750  return true;
751  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
assert(m_qm.get())
static EntryNumber_t const invalidEntry
bool readOneSequential(EventPrincipal &cache, size_t &fileNameHash)
void skipEntries(unsigned int offset)
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
bool edm::RootInputFileSequence::readOneSequentialWithID ( EventPrincipal cache,
size_t &  fileNameHash,
LuminosityBlockID const &  id 
)

Definition at line 754 of file RootInputFileSequence.cc.

References assert(), edm::errors::Configuration, edm::hlt::Exception, fileIter_, fileIterBegin_, fileIterEnd_, firstFile_, newFWLiteAna::found, initFile(), initialNumberOfEventsToSkip_, edm::IndexIntoFile::invalidEntry, lfnHash_, hltrates_dqm_sourceclient-live_cfg::offset, rootFile_, DTTTrigCorrFirst::run, skipBadFiles_, skipToItem(), and skipToItemInNewFile().

754  {
755  skipBadFiles_ = false;
756  if(firstFile_) {
757  firstFile_ = false;
759  throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequential(): no input files specified for secondary input source.\n";
760  }
761  if(fileIter_ != fileIterBegin_) {
763  initFile(false);
764  }
765  assert(rootFile_);
766  rootFile_->setAtEventEntry(IndexIntoFile::invalidEntry);
768  while(offset > 0) {
769  bool found = readOneSequentialWithID(cache, fileNameHash, id);
770  if(!found) {
771  return false;
772  }
773  --offset;
774  }
775  }
776  assert(rootFile_);
777  if(fileIter_ == fileIterEnd_ ||
778  rootFile_->indexIntoFileIter().run() != id.run() ||
779  rootFile_->indexIntoFileIter().lumi() != id.luminosityBlock()) {
780  bool found = skipToItem(id.run(), id.luminosityBlock(), 0, 0, false);
781  if(!found) {
782  return false;
783  }
784  }
785  assert(rootFile_);
786  bool found = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
787  if(found) {
788  found = rootFile_->readCurrentEvent(cache);
789  }
790  if(!found) {
791  found = skipToItemInNewFile(id.run(), id.luminosityBlock(), 0);
792  if(!found) {
793  return false;
794  }
795  return readOneSequentialWithID(cache, fileNameHash, id);
796  }
797  fileNameHash = lfnHash_;
798  return true;
799  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
assert(m_qm.get())
bool skipToItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash=0U, bool currentFileFirst=true)
static EntryNumber_t const invalidEntry
bool skipToItemInNewFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event)
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
bool readOneSequentialWithID(EventPrincipal &cache, size_t &fileNameHash, LuminosityBlockID const &id)
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
void edm::RootInputFileSequence::readOneSpecified ( EventPrincipal cache,
size_t &  fileNameHash,
SecondaryEventIDAndFileInfo const &  id 
)

Definition at line 802 of file RootInputFileSequence.cc.

References assert(), edm::errors::Configuration, event(), edm::SecondaryEventIDAndFileInfo::eventID(), edm::hlt::Exception, fileIterBegin_, fileIterEnd_, edm::SecondaryEventIDAndFileInfo::fileNameHash(), firstFile_, newFWLiteAna::found, lfnHash_, edm::errors::NotFound, rootFile_, DTTTrigCorrFirst::run, skipBadFiles_, and skipToItem().

802  {
803  firstFile_ = false;
805  throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSpecified(): no input files specified for secondary input source.\n";
806  }
807  skipBadFiles_ = false;
808  EventID const& id = idx.eventID();
809  bool found = skipToItem(id.run(), id.luminosityBlock(), id.event(), idx.fileNameHash());
810  if(!found) {
811  throw Exception(errors::NotFound) <<
812  "RootInputFileSequence::readOneSpecified(): Secondary Input files" <<
813  " do not contain specified event:\n" << id << "\n";
814  }
815  assert(rootFile_);
816  found = rootFile_->readCurrentEvent(cache);
817  assert(found);
818  fileNameHash = idx.fileNameHash();
819  if(fileNameHash == 0U) {
820  fileNameHash = lfnHash_;
821  }
822  }
assert(m_qm.get())
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
bool skipToItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash=0U, bool currentFileFirst=true)
tuple idx
DEBUGGING if hasattr(process,&quot;trackMonIterativeTracking2012&quot;): print &quot;trackMonIterativeTracking2012 D...
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
void edm::RootInputFileSequence::readRun_ ( RunPrincipal runPrincipal)

Definition at line 411 of file RootInputFileSequence.cc.

References assert(), and rootFile_.

411  {
412  assert(rootFile_);
413  rootFile_->readRun_(runPrincipal);
414  }
assert(m_qm.get())
std::shared_ptr< RunAuxiliary > edm::RootInputFileSequence::readRunAuxiliary_ ( )

Definition at line 399 of file RootInputFileSequence.cc.

References assert(), and rootFile_.

399  {
400  assert(rootFile_);
401  return rootFile_->readRunAuxiliary_();
402  }
assert(m_qm.get())
int edm::RootInputFileSequence::remainingEvents ( ) const
private

Definition at line 672 of file RootInputFileSequence.cc.

References input_, and edm::InputSource::remainingEvents().

Referenced by initFile().

672  {
673  return input_.remainingEvents();
674  }
int remainingEvents() const
Definition: InputSource.h:198
int edm::RootInputFileSequence::remainingLuminosityBlocks ( ) const
private

Definition at line 677 of file RootInputFileSequence.cc.

References input_, and edm::InputSource::remainingLuminosityBlocks().

Referenced by initFile().

677  {
679  }
int remainingLuminosityBlocks() const
Definition: InputSource.h:206
ProcessingController::ReverseState edm::RootInputFileSequence::reverseState ( ) const
void edm::RootInputFileSequence::rewind_ ( )

Definition at line 474 of file RootInputFileSequence.cc.

References closeFile_(), fileIter_, fileIterBegin_, firstFile_, initFile(), initialNumberOfEventsToSkip_, rewindFile(), rootFile_, and skipEvents().

474  {
475  if(fileIter_ != fileIterBegin_) {
476  closeFile_();
478  }
479  if(!rootFile_) {
480  initFile(false);
481  }
482  rewindFile();
483  firstFile_ = true;
484  if(rootFile_) {
487  }
488  }
489  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
void edm::RootInputFileSequence::rewindFile ( )
private

Definition at line 493 of file RootInputFileSequence.cc.

References rootFile_.

Referenced by rewind_().

493  {
494  if(rootFile_) rootFile_->rewind();
495  }
void edm::RootInputFileSequence::skipEntries ( unsigned int  offset)

Definition at line 705 of file RootInputFileSequence.cc.

References assert(), fileIter_, fileIterBegin_, fileIterEnd_, initFile(), edm::IndexIntoFile::invalidEntry, and rootFile_.

Referenced by readOneSequential().

705  {
706  // offset is decremented by the number of events actually skipped.
707  bool completed = rootFile_->skipEntries(offset);
708  while(!completed) {
709  ++fileIter_;
710  if(fileIter_ == fileIterEnd_) {
712  }
713  initFile(false);
714  assert(rootFile_);
715  rootFile_->setAtEventEntry(IndexIntoFile::invalidEntry);
716  completed = rootFile_->skipEntries(offset);
717  }
718  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
assert(m_qm.get())
static EntryNumber_t const invalidEntry
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
bool edm::RootInputFileSequence::skipEvents ( int  offset)

Definition at line 499 of file RootInputFileSequence.cc.

References assert(), fileIter_, fileIterEnd_, inputType_, nextFile(), previousFile(), rootFile_, and edm::SecondaryFile.

Referenced by rewind_(), and RootInputFileSequence().

499  {
500  // We never call skipEvents for secondary input files. If we did,
501  // we would have to implement synchronization if a new file is opened.
502  // To avoid this, just assert.
504  assert(rootFile_);
505  while(offset != 0) {
506  bool atEnd = rootFile_->skipEvents(offset);
507  if((offset > 0 || atEnd) && !nextFile()) {
508  return false;
509  }
510  if(offset < 0 && !previousFile()) {
512  return false;
513  }
514  }
515  return true;
516  }
std::vector< FileCatalogItem >::const_iterator fileIter_
assert(m_qm.get())
std::vector< FileCatalogItem >::const_iterator fileIterEnd_
bool edm::RootInputFileSequence::skipToItem ( RunNumber_t  run,
LuminosityBlockNumber_t  lumi,
EventNumber_t  event,
size_t  fileNameHash = 0U,
bool  currentFileFirst = true 
)

Definition at line 626 of file RootInputFileSequence.cc.

References assert(), fileIter_, fileIterBegin_, newFWLiteAna::found, indexesIntoFiles_, initFile(), rootFile_, and skipToItemInNewFile().

Referenced by readOneRandomWithID(), readOneSequentialWithID(), and readOneSpecified().

626  {
627  // Attempt to find item in currently open input file.
628  bool found = currentFileFirst && rootFile_ && rootFile_->setEntryAtItem(run, lumi, event);
629  if(!found) {
630  // If only one input file, give up now, to save time.
631  if(currentFileFirst && rootFile_ && indexesIntoFiles_.size() == 1) {
632  return false;
633  }
634  // Look for item (run/lumi/event) in files previously opened without reopening unnecessary files.
635  typedef std::vector<std::shared_ptr<IndexIntoFile> >::const_iterator Iter;
636  for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
637  if(*it && (*it)->containsItem(run, lumi, event)) {
638  // We found it. Close the currently open file, and open the correct one.
639  std::vector<FileCatalogItem>::const_iterator currentIter = fileIter_;
640  fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
641  if(fileIter_ != currentIter) {
642  initFile(false);
643  }
644  // Now get the item from the correct file.
645  assert(rootFile_);
646  found = rootFile_->setEntryAtItem(run, lumi, event);
647  assert(found);
648  return true;
649  }
650  }
651  return (fileNameHash != 0U && skipToItemInNewFile(run, lumi, event, fileNameHash)) || skipToItemInNewFile(run, lumi, event);
652  }
653  return true;
654  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
tuple lumi
Definition: fjr2json.py:35
assert(m_qm.get())
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
std::vector< std::shared_ptr< IndexIntoFile > > indexesIntoFiles_
bool skipToItemInNewFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event)
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
bool edm::RootInputFileSequence::skipToItemInNewFile ( RunNumber_t  run,
LuminosityBlockNumber_t  lumi,
EventNumber_t  event 
)

Definition at line 606 of file RootInputFileSequence.cc.

References assert(), fileIter_, fileIterBegin_, newFWLiteAna::found, indexesIntoFiles_, initFile(), and rootFile_.

Referenced by readOneSequentialWithID(), and skipToItem().

606  {
607  // Look for item in files not yet opened. We do not have a valid hash of the logical file name.
608  typedef std::vector<std::shared_ptr<IndexIntoFile> >::const_iterator Iter;
609  for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
610  if(!*it) {
611  // File not yet opened.
612  fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
613  initFile(false);
614  assert(rootFile_);
615  bool found = rootFile_->setEntryAtItem(run, lumi, event);
616  if(found) {
617  return true;
618  }
619  }
620  }
621  // Not found
622  return false;
623  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
tuple lumi
Definition: fjr2json.py:35
assert(m_qm.get())
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
std::vector< std::shared_ptr< IndexIntoFile > > indexesIntoFiles_
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
bool edm::RootInputFileSequence::skipToItemInNewFile ( RunNumber_t  run,
LuminosityBlockNumber_t  lumi,
EventNumber_t  event,
size_t  fileNameHash 
)

Definition at line 574 of file RootInputFileSequence.cc.

References assert(), fileIter_, fileIterBegin_, fileIterEnd_, findFileForSpecifiedID_, newFWLiteAna::found, indexesIntoFiles_, initFile(), getDQMSummary::iter, and rootFile_.

574  {
575  // Look for item in files not yet opened. We have a hash of the logical file name
576  assert(fileNameHash != 0U);
577  // If the lookup table is not yet filled in, fill it.
579  // We use a multimap because there may be hash collisions (Two different LFNs could have the same hash).
580  // We map the hash of the LFN to the index into the list of files.
581  findFileForSpecifiedID_.reset(new std::unordered_multimap<size_t, size_t>);
582  auto hasher = std::hash<std::string>();
583  for(auto fileIter = fileIterBegin_; fileIter != fileIterEnd_; ++fileIter) {
584  findFileForSpecifiedID_->insert(std::make_pair(hasher(fileIter->logicalFileName()), fileIter - fileIterBegin_));
585  }
586  }
587  // Look up the logical file name in the table
588  auto range = findFileForSpecifiedID_->equal_range(fileNameHash);
589  for(auto iter = range.first; iter != range.second; ++iter) {
590  // Don't look in files previously opened, because those have already been searched.
591  if(!indexesIntoFiles_[iter->second]) {
592  fileIter_ = fileIterBegin_ + iter->second;
593  initFile(false);
594  assert(rootFile_);
595  bool found = rootFile_->setEntryAtItem(run, lumi, event);
596  if(found) {
597  return true;
598  }
599  }
600  }
601  // Not found
602  return false;
603  }
void initFile(bool skipBadFiles)
std::vector< FileCatalogItem >::const_iterator fileIter_
tuple lumi
Definition: fjr2json.py:35
assert(m_qm.get())
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger but the state exists so we define the behavior If all triggers are the negative crieriion will lead to accepting the event(this again matches the behavior of"!*"before the partial wildcard feature was incorporated).The per-event"cost"of each negative criterion with multiple relevant triggers is about the same as!*was in the past
std::vector< std::shared_ptr< IndexIntoFile > > indexesIntoFiles_
std::unique_ptr< std::unordered_multimap< size_t, size_t > > findFileForSpecifiedID_
std::vector< FileCatalogItem >::const_iterator fileIterBegin_
std::vector< FileCatalogItem >::const_iterator fileIterEnd_

Member Data Documentation

std::vector<BranchID> edm::RootInputFileSequence::associationsFromSecondary_
private

Definition at line 114 of file RootInputFileSequence.h.

Referenced by initAssociationsFromSecondary(), and initFile().

BranchDescription::MatchMode edm::RootInputFileSequence::branchesMustMatch_
private

Definition at line 110 of file RootInputFileSequence.h.

Referenced by nextFile(), previousFile(), and RootInputFileSequence().

bool edm::RootInputFileSequence::bypassVersionCheck_
private

Definition at line 122 of file RootInputFileSequence.h.

Referenced by initFile().

InputFileCatalog const& edm::RootInputFileSequence::catalog_
private

Definition at line 100 of file RootInputFileSequence.h.

Referenced by fileCatalogItems().

bool edm::RootInputFileSequence::dropDescendants_
private

Definition at line 128 of file RootInputFileSequence.h.

Referenced by initFile().

std::shared_ptr<DuplicateChecker> edm::RootInputFileSequence::duplicateChecker_
private

Definition at line 127 of file RootInputFileSequence.h.

Referenced by closeFile_(), and initFile().

bool edm::RootInputFileSequence::enablePrefetching_
private

Definition at line 131 of file RootInputFileSequence.h.

Referenced by initFile(), and RootInputFileSequence().

std::shared_ptr<EventSkipperByID> edm::RootInputFileSequence::eventSkipperByID_
private

Definition at line 117 of file RootInputFileSequence.h.

Referenced by initFile().

int edm::RootInputFileSequence::eventsRemainingInFile_
private

Definition at line 118 of file RootInputFileSequence.h.

Referenced by readOneRandom().

std::vector<FileCatalogItem>::const_iterator edm::RootInputFileSequence::fileIter_
private
std::vector<FileCatalogItem>::const_iterator edm::RootInputFileSequence::fileIterBegin_
private
std::vector<FileCatalogItem>::const_iterator edm::RootInputFileSequence::fileIterEnd_
private
std::vector<FileCatalogItem>::const_iterator edm::RootInputFileSequence::fileIterLastOpened_
private

Definition at line 108 of file RootInputFileSequence.h.

Referenced by initFile().

std::unique_ptr<std::unordered_multimap<size_t, size_t> > edm::RootInputFileSequence::findFileForSpecifiedID_
private

Definition at line 104 of file RootInputFileSequence.h.

Referenced by skipToItemInNewFile().

bool edm::RootInputFileSequence::firstFile_
private
std::vector<std::shared_ptr<IndexIntoFile> > edm::RootInputFileSequence::indexesIntoFiles_
private

Definition at line 112 of file RootInputFileSequence.h.

Referenced by goToEvent(), initFile(), skipToItem(), and skipToItemInNewFile().

int edm::RootInputFileSequence::initialNumberOfEventsToSkip_
private
PoolSource& edm::RootInputFileSequence::input_
private
InputType edm::RootInputFileSequence::inputType_
private
bool edm::RootInputFileSequence::labelRawDataLikeMC_
private

Definition at line 129 of file RootInputFileSequence.h.

Referenced by initFile().

std::string edm::RootInputFileSequence::lfn_
private

Definition at line 102 of file RootInputFileSequence.h.

Referenced by closeFile_(), and initFile().

size_t edm::RootInputFileSequence::lfnHash_
private
bool edm::RootInputFileSequence::noEventSort_
private

Definition at line 120 of file RootInputFileSequence.h.

Referenced by initFile().

unsigned int edm::RootInputFileSequence::nStreams_
private

Definition at line 116 of file RootInputFileSequence.h.

Referenced by initFile().

std::vector<ProcessHistoryID> edm::RootInputFileSequence::orderedProcessHistoryIDs_
private

Definition at line 113 of file RootInputFileSequence.h.

Referenced by initFile().

ProductSelectorRules edm::RootInputFileSequence::productSelectorRules_
private

Definition at line 126 of file RootInputFileSequence.h.

Referenced by dropUnwantedBranches_(), and initFile().

RootFileSharedPtr edm::RootInputFileSequence::rootFile_
private
RunNumber_t edm::RootInputFileSequence::setRun_
private

Definition at line 125 of file RootInputFileSequence.h.

Referenced by initFile().

bool edm::RootInputFileSequence::skipBadFiles_
private
unsigned int edm::RootInputFileSequence::treeCacheSize_
private

Definition at line 123 of file RootInputFileSequence.h.

Referenced by initFile(), and RootInputFileSequence().

int const edm::RootInputFileSequence::treeMaxVirtualSize_
private

Definition at line 124 of file RootInputFileSequence.h.

Referenced by initFile().

bool edm::RootInputFileSequence::usedFallback_
private

Definition at line 132 of file RootInputFileSequence.h.

Referenced by closeFile_(), and initFile().

bool edm::RootInputFileSequence::usingGoToEvent_
private

Definition at line 130 of file RootInputFileSequence.h.

Referenced by goToEvent(), and initFile().