CMS 3D CMS Logo

/afs/cern.ch/work/a/aaltunda/public/www/CMSSW_6_2_5/src/IOPool/Input/src/RootInputFileSequence.cc

Go to the documentation of this file.
00001 /*----------------------------------------------------------------------
00002 ----------------------------------------------------------------------*/
00003 #include "DuplicateChecker.h"
00004 #include "PoolSource.h"
00005 #include "RootFile.h"
00006 #include "RootInputFileSequence.h"
00007 #include "RootTree.h"
00008 
00009 #include "DataFormats/Provenance/interface/BranchIDListHelper.h"
00010 #include "DataFormats/Provenance/interface/ProductRegistry.h"
00011 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
00012 #include "FWCore/Framework/interface/EventPrincipal.h"
00013 #include "FWCore/Framework/interface/FileBlock.h"
00014 #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h"
00015 #include "FWCore/Framework/interface/RunPrincipal.h"
00016 #include "FWCore/MessageLogger/interface/MessageLogger.h"
00017 #include "FWCore/ParameterSet/interface/ParameterSet.h"
00018 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
00019 #include "FWCore/ServiceRegistry/interface/Service.h"
00020 #include "FWCore/Utilities/interface/RandomNumberGenerator.h"
00021 #include "Utilities/StorageFactory/interface/StorageFactory.h"
00022 
00023 #include "CLHEP/Random/RandFlat.h"
00024 #include "InputFile.h"
00025 #include "TSystem.h"
00026 
00027 namespace edm {
00028   RootInputFileSequence::RootInputFileSequence(
00029                 ParameterSet const& pset,
00030                 PoolSource const& input,
00031                 InputFileCatalog const& catalog,
00032                 InputType::InputType inputType) :
00033     input_(input),
00034     inputType_(inputType),
00035     catalog_(catalog),
00036     firstFile_(true),
00037     lfn_("unknown"),
00038     fileIterBegin_(fileCatalogItems().begin()),
00039     fileIterEnd_(fileCatalogItems().end()),
00040     fileIter_(fileIterEnd_),
00041     fileIterLastOpened_(fileIterEnd_),
00042     rootFile_(),
00043     parametersMustMatch_(BranchDescription::Permissive),
00044     branchesMustMatch_(BranchDescription::Permissive),
00045     flatDistribution_(),
00046     indexesIntoFiles_(fileCatalogItems().size()),
00047     orderedProcessHistoryIDs_(),
00048     eventSkipperByID_(inputType == InputType::Primary ? EventSkipperByID::create(pset).release() : 0),
00049     eventsRemainingInFile_(0),
00050     // The default value provided as the second argument to the getUntrackedParameter function call
00051     // is not used when the ParameterSet has been validated and the parameters are not optional
00052     // in the description.  This is currently true when PoolSource is the primary input source.
00053     // The modules that use PoolSource as a SecSource have not defined their fillDescriptions function
00054     // yet, so the ParameterSet does not get validated yet.  As soon as all the modules with a SecSource
00055     // have defined descriptions, the defaults in the getUntrackedParameterSet function calls can
00056     // and should be deleted from the code.
00057     initialNumberOfEventsToSkip_(inputType == InputType::Primary ? pset.getUntrackedParameter<unsigned int>("skipEvents", 0U) : 0U),
00058     noEventSort_(inputType == InputType::Primary ? pset.getUntrackedParameter<bool>("noEventSort", true) : false),
00059     skipBadFiles_(pset.getUntrackedParameter<bool>("skipBadFiles", false)),
00060     treeCacheSize_(noEventSort_ ? pset.getUntrackedParameter<unsigned int>("cacheSize", roottree::defaultCacheSize) : 0U),
00061     treeMaxVirtualSize_(pset.getUntrackedParameter<int>("treeMaxVirtualSize", -1)),
00062     setRun_(pset.getUntrackedParameter<unsigned int>("setRunNumber", 0U)),
00063     productSelectorRules_(pset, "inputCommands", "InputSource"),
00064     duplicateChecker_(inputType == InputType::Primary ? new DuplicateChecker(pset) : 0),
00065     dropDescendants_(pset.getUntrackedParameter<bool>("dropDescendantsOfDroppedBranches", inputType != InputType::SecondarySource)),
00066     labelRawDataLikeMC_(pset.getUntrackedParameter<bool>("labelRawDataLikeMC", true)),
00067     usingGoToEvent_(false),
00068     enablePrefetching_(false),
00069     usedFallback_(false) {
00070 
00071     // The SiteLocalConfig controls the TTreeCache size and the prefetching settings.
00072     Service<SiteLocalConfig> pSLC;
00073     if(pSLC.isAvailable()) {
00074       if(treeCacheSize_ != 0U && pSLC->sourceTTreeCacheSize()) {
00075         treeCacheSize_ = *(pSLC->sourceTTreeCacheSize());
00076       }
00077       enablePrefetching_ = pSLC->enablePrefetching();
00078     }
00079 
00080     StorageFactory *factory = StorageFactory::get();
00081     for(fileIter_ = fileIterBegin_; fileIter_ != fileIterEnd_; ++fileIter_) {
00082       factory->activateTimeout(fileIter_->fileName());
00083       factory->stagein(fileIter_->fileName());
00084       //NOTE: we do not want to stage in all secondary files since we can be given a list of
00085       // thousands of files and prestaging all those files can cause a site to fail.
00086       // So, we stage in the first secondary file only.
00087       if(inputType_ != InputType::Primary) {
00088         break;
00089       }
00090     }
00091 
00092     std::string parametersMustMatch = pset.getUntrackedParameter<std::string>("parametersMustMatch", std::string("permissive"));
00093     if(parametersMustMatch == std::string("strict")) parametersMustMatch_ = BranchDescription::Strict;
00094 
00095     std::string branchesMustMatch = pset.getUntrackedParameter<std::string>("branchesMustMatch", std::string("permissive"));
00096     if(branchesMustMatch == std::string("strict")) branchesMustMatch_ = BranchDescription::Strict;
00097 
00098     for(fileIter_ = fileIterBegin_; fileIter_ != fileIterEnd_; ++fileIter_) {
00099       initFile(skipBadFiles_);
00100       if(rootFile_) break;
00101     }
00102     if(rootFile_) {
00103       productRegistryUpdate().updateFromInput(rootFile_->productRegistry()->productList());
00104       if(initialNumberOfEventsToSkip_ != 0) {
00105         skipEvents(initialNumberOfEventsToSkip_);
00106       }
00107     }
00108   }
00109 
00110   std::vector<FileCatalogItem> const&
00111   RootInputFileSequence::fileCatalogItems() const {
00112     return catalog_.fileCatalogItems();
00113   }
00114 
00115   void
00116   RootInputFileSequence::endJob() {
00117     closeFile_();
00118   }
00119 
00120   std::unique_ptr<FileBlock>
00121   RootInputFileSequence::readFile_() {
00122     if(firstFile_) {
00123       // The first input file has already been opened.
00124       firstFile_ = false;
00125       if(!rootFile_) {
00126         initFile(skipBadFiles_);
00127       }
00128     } else {
00129       if(!nextFile()) {
00130         assert(0);
00131       }
00132     }
00133     if(!rootFile_) {
00134       return std::unique_ptr<FileBlock>(new FileBlock);
00135     }
00136     return rootFile_->createFileBlock();
00137   }
00138 
00139   void RootInputFileSequence::closeFile_() {
00140     // close the currently open file, if any, and delete the RootFile object.
00141     if(rootFile_) {
00142       if(inputType_ != InputType::SecondarySource) {
00143         std::unique_ptr<InputSource::FileCloseSentry>
00144         sentry((inputType_ == InputType::Primary) ? new InputSource::FileCloseSentry(input_, lfn_, usedFallback_) : 0);
00145         rootFile_->close();
00146         if(duplicateChecker_) duplicateChecker_->inputFileClosed();
00147       }
00148       rootFile_.reset();
00149     }
00150   }
00151 
00152   void RootInputFileSequence::initFile(bool skipBadFiles) {
00153     // We are really going to close the open file.
00154 
00155     // If this is the primary sequence, we are not duplicate checking across files
00156     // and we are not using random access to find events, then we can delete the
00157     // IndexIntoFile for the file we are closing. If we can't delete all of it,
00158     // then we can delete the parts we do not need.
00159     if(fileIterLastOpened_ != fileIterEnd_) {
00160       size_t currentIndexIntoFile = fileIterLastOpened_ - fileIterBegin_;
00161       bool needIndexesForDuplicateChecker = duplicateChecker_ && duplicateChecker_->checkingAllFiles() && !duplicateChecker_->checkDisabled();
00162       bool deleteIndexIntoFile = inputType_ == InputType::Primary &&
00163                                  !needIndexesForDuplicateChecker &&
00164                                  !usingGoToEvent_;
00165       if(deleteIndexIntoFile) {
00166         indexesIntoFiles_[currentIndexIntoFile].reset();
00167       } else {
00168         if(indexesIntoFiles_[currentIndexIntoFile]) indexesIntoFiles_[currentIndexIntoFile]->inputFileClosed();
00169       }
00170       fileIterLastOpened_ = fileIterEnd_;
00171     }
00172     closeFile_();
00173 
00174     if(fileIter_ == fileIterEnd_) {
00175       // No files specified
00176       return;
00177     }
00178 
00179     // Check if the logical file name was found.
00180     if(fileIter_->fileName().empty()) {
00181       // LFN not found in catalog.
00182       InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00183       if(!skipBadFiles) {
00184         throw cms::Exception("LogicalFileNameNotFound", "RootInputFileSequence::initFile()\n")
00185           << "Logical file name '" << fileIter_->logicalFileName() << "' was not found in the file catalog.\n"
00186           << "If you wanted a local file, you forgot the 'file:' prefix\n"
00187           << "before the file name in your configuration file.\n";
00188       }
00189       LogWarning("") << "Input logical file: " << fileIter_->logicalFileName() << " was not found in the catalog, and will be skipped.\n";
00190       return;
00191     }
00192 
00193     lfn_ = fileIter_->logicalFileName().empty() ? fileIter_->fileName() : fileIter_->logicalFileName();
00194 
00195     // Determine whether we have a fallback URL specified; if so, prepare it;
00196     // Only valid if it is non-empty and differs from the original filename.
00197     std::string fallbackName = fileIter_->fallbackFileName();
00198     bool hasFallbackUrl = !fallbackName.empty() && fallbackName != fileIter_->fileName();
00199 
00200     boost::shared_ptr<InputFile> filePtr;
00201     try {
00202       std::unique_ptr<InputSource::FileOpenSentry>
00203         sentry(inputType_ == InputType::Primary ? new InputSource::FileOpenSentry(input_) : 0);
00204       filePtr.reset(new InputFile(gSystem->ExpandPathName(fileIter_->fileName().c_str()), "  Initiating request to open file "));
00205     }
00206     catch (cms::Exception const& e) {
00207       if(!skipBadFiles) {
00208         if(hasFallbackUrl) {
00209           std::ostringstream out;
00210           out << e.explainSelf();
00211           std::string pfn(gSystem->ExpandPathName(fallbackName.c_str()));
00212           InputFile::reportFallbackAttempt(pfn, fileIter_->logicalFileName(), out.str());
00213         } else {
00214           InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00215           Exception ex(errors::FileOpenError, "", e);
00216           ex.addContext("Calling RootInputFileSequence::initFile()");
00217           std::ostringstream out;
00218           out << "Input file " << fileIter_->fileName() << " could not be opened.";
00219           ex.addAdditionalInfo(out.str());
00220           throw ex;
00221         }
00222       }
00223     }
00224     if(!filePtr && (hasFallbackUrl)) {
00225       try {
00226         std::unique_ptr<InputSource::FileOpenSentry>
00227           sentry(inputType_ == InputType::Primary ? new InputSource::FileOpenSentry(input_) : 0);
00228         filePtr.reset(new InputFile(gSystem->ExpandPathName(fallbackName.c_str()), "  Fallback request to file "));
00229         usedFallback_ = true;
00230       }
00231       catch (cms::Exception const& e) {
00232         if(!skipBadFiles) {
00233           InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00234           Exception ex(errors::FallbackFileOpenError, "", e);
00235           ex.addContext("Calling RootInputFileSequence::initFile()");
00236           std::ostringstream out;
00237           out << "Input file " << fileIter_->fileName() << " could not be opened.\n";
00238           out << "Fallback Input file " << fallbackName << " also could not be opened.";
00239           ex.addAdditionalInfo(out.str());
00240           throw ex;
00241         }
00242       }
00243     }
00244     if(filePtr) {
00245       std::vector<boost::shared_ptr<IndexIntoFile> >::size_type currentIndexIntoFile = fileIter_ - fileIterBegin_;
00246       rootFile_ = RootFileSharedPtr(new RootFile(fileIter_->fileName(),
00247           processConfiguration(), fileIter_->logicalFileName(), filePtr,
00248           eventSkipperByID_, initialNumberOfEventsToSkip_ != 0,
00249           remainingEvents(), remainingLuminosityBlocks(), treeCacheSize_, treeMaxVirtualSize_,
00250           input_.processingMode(),
00251           setRun_,
00252           noEventSort_,
00253           productSelectorRules_,
00254           inputType_,
00255           (inputType_ == InputType::SecondarySource ?  boost::shared_ptr<BranchIDListHelper>(new BranchIDListHelper()) :  input_.branchIDListHelper()),
00256           duplicateChecker_,
00257           dropDescendants_,
00258           indexesIntoFiles_,
00259           currentIndexIntoFile,
00260           orderedProcessHistoryIDs_,
00261           labelRawDataLikeMC_,
00262           usingGoToEvent_,
00263           enablePrefetching_));
00264 
00265       fileIterLastOpened_ = fileIter_;
00266       indexesIntoFiles_[currentIndexIntoFile] = rootFile_->indexIntoFileSharedPtr();
00267       char const* inputType = 0;
00268       switch(inputType_) {
00269       case InputType::Primary: inputType = "primaryFiles"; break;
00270       case InputType::SecondaryFile: inputType = "secondaryFiles"; break;
00271       case InputType::SecondarySource: inputType = "mixingFiles"; break;
00272       }
00273       rootFile_->reportOpened(inputType);
00274     } else {
00275       InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00276       if(!skipBadFiles) {
00277         throw Exception(errors::FileOpenError) <<
00278            "RootInputFileSequence::initFile(): Input file " << fileIter_->fileName() << " was not found or could not be opened.\n";
00279       }
00280       LogWarning("") << "Input file: " << fileIter_->fileName() << " was not found or could not be opened, and will be skipped.\n";
00281     }
00282   }
00283 
00284   boost::shared_ptr<ProductRegistry const>
00285   RootInputFileSequence::fileProductRegistry() const {
00286     assert(rootFile_);
00287     return rootFile_->productRegistry();
00288   }
00289 
00290   boost::shared_ptr<BranchIDListHelper const>
00291   RootInputFileSequence::fileBranchIDListHelper() const {
00292     assert(rootFile_);
00293     return rootFile_->branchIDListHelper();
00294   }
00295 
00296   bool RootInputFileSequence::nextFile() {
00297     if(fileIter_ != fileIterEnd_) ++fileIter_;
00298     if(fileIter_ == fileIterEnd_) {
00299       if(inputType_ == InputType::Primary) {
00300         return false;
00301       } else {
00302         fileIter_ = fileIterBegin_;
00303       }
00304     }
00305 
00306     initFile(skipBadFiles_);
00307 
00308     if(inputType_ == InputType::Primary && rootFile_) {
00309       // make sure the new product registry is compatible with the main one
00310       std::string mergeInfo = productRegistryUpdate().merge(*rootFile_->productRegistry(),
00311                                                             fileIter_->fileName(),
00312                                                             parametersMustMatch_,
00313                                                             branchesMustMatch_);
00314       if(!mergeInfo.empty()) {
00315         throw Exception(errors::MismatchedInputFiles,"RootInputFileSequence::nextFile()") << mergeInfo;
00316       }
00317     }
00318     return true;
00319   }
00320 
00321   bool RootInputFileSequence::previousFile() {
00322     if(fileIter_ == fileIterBegin_) {
00323       if(inputType_ == InputType::Primary) {
00324         return false;
00325       } else {
00326         fileIter_ = fileIterEnd_;
00327       }
00328     }
00329     --fileIter_;
00330 
00331     initFile(false);
00332 
00333     if(inputType_ == InputType::Primary && rootFile_) {
00334       // make sure the new product registry is compatible to the main one
00335       std::string mergeInfo = productRegistryUpdate().merge(*rootFile_->productRegistry(),
00336                                                             fileIter_->fileName(),
00337                                                             parametersMustMatch_,
00338                                                             branchesMustMatch_);
00339       if(!mergeInfo.empty()) {
00340         throw Exception(errors::MismatchedInputFiles,"RootInputFileSequence::previousEvent()") << mergeInfo;
00341       }
00342     }
00343     if(rootFile_) rootFile_->setToLastEntry();
00344     return true;
00345   }
00346 
00347   RootInputFileSequence::~RootInputFileSequence() {
00348   }
00349 
00350   boost::shared_ptr<RunAuxiliary>
00351   RootInputFileSequence::readRunAuxiliary_() {
00352     return rootFile_->readRunAuxiliary_();
00353   }
00354 
00355   boost::shared_ptr<LuminosityBlockAuxiliary>
00356   RootInputFileSequence::readLuminosityBlockAuxiliary_() {
00357     return rootFile_->readLuminosityBlockAuxiliary_();
00358   }
00359 
00360   boost::shared_ptr<RunPrincipal>
00361   RootInputFileSequence::readRun_(boost::shared_ptr<RunPrincipal> runPrincipal) {
00362     return rootFile_->readRun_(runPrincipal);
00363   }
00364 
00365   boost::shared_ptr<LuminosityBlockPrincipal>
00366   RootInputFileSequence::readLuminosityBlock_(boost::shared_ptr<LuminosityBlockPrincipal> lumiPrincipal) {
00367     return rootFile_->readLumi(lumiPrincipal);
00368   }
00369 
00370   // readEvent() is responsible for setting up the EventPrincipal.
00371   //
00372   //   1. fill an EventPrincipal with a unique EventID
00373   //   2. For each entry in the provenance, put in one ProductHolder,
00374   //      holding the Provenance for the corresponding EDProduct.
00375   //   3. set up the caches in the EventPrincipal to know about this
00376   //      ProductHolder.
00377   //
00378   // We do *not* create the EDProduct instance (the equivalent of reading
00379   // the branch containing this EDProduct. That will be done by the Delayed Reader,
00380   //  when it is asked to do so.
00381   //
00382 
00383   EventPrincipal*
00384   RootInputFileSequence::readEvent(EventPrincipal& eventPrincipal) {
00385     return rootFile_->readEvent(eventPrincipal);
00386   }
00387 
00388   InputSource::ItemType
00389   RootInputFileSequence::getNextItemType() {
00390     if(fileIter_ == fileIterEnd_) {
00391       return InputSource::IsStop;
00392     }
00393     if(firstFile_) {
00394       return InputSource::IsFile;
00395     }
00396     if(rootFile_) {
00397       IndexIntoFile::EntryType entryType = rootFile_->getNextEntryTypeWanted();
00398       if(entryType == IndexIntoFile::kEvent) {
00399         return InputSource::IsEvent;
00400       } else if(entryType == IndexIntoFile::kLumi) {
00401         return InputSource::IsLumi;
00402       } else if(entryType == IndexIntoFile::kRun) {
00403         return InputSource::IsRun;
00404       }
00405       assert(entryType == IndexIntoFile::kEnd);
00406     }
00407     if(fileIter_ + 1 == fileIterEnd_) {
00408       return InputSource::IsStop;
00409     }
00410     return InputSource::IsFile;
00411   }
00412 
00413   // Rewind to before the first event that was read.
00414   void
00415   RootInputFileSequence::rewind_() {
00416     if(fileIter_ != fileIterBegin_) {
00417       closeFile_();
00418       fileIter_ = fileIterBegin_;
00419     }
00420     if(!rootFile_) {
00421       initFile(false);
00422     }
00423     rewindFile();
00424     firstFile_ = true;
00425     if(rootFile_) {
00426       if(initialNumberOfEventsToSkip_ != 0) {
00427         skipEvents(initialNumberOfEventsToSkip_);
00428       }
00429     }
00430   }
00431 
00432   // Rewind to the beginning of the current file
00433   void
00434   RootInputFileSequence::rewindFile() {
00435     if(rootFile_) rootFile_->rewind();
00436   }
00437 
00438   // Advance "offset" events.  Offset can be positive or negative (or zero).
00439   bool
00440   RootInputFileSequence::skipEvents(int offset) {
00441     while(offset != 0) {
00442       bool atEnd = rootFile_->skipEvents(offset);
00443       if((offset > 0 || atEnd) && !nextFile()) {
00444         return false;
00445       }
00446       if(offset < 0 && !previousFile()) {
00447         fileIter_ = fileIterEnd_;
00448         return false;
00449       }
00450     }
00451     return true;
00452   }
00453 
00454   bool
00455   RootInputFileSequence::goToEvent(EventID const& eventID) {
00456     usingGoToEvent_ = true;
00457     if(rootFile_) {
00458       if(rootFile_->goToEvent(eventID)) {
00459         return true;
00460       }
00461       // If only one input file, give up now, to save time.
00462       if(rootFile_ && indexesIntoFiles_.size() == 1) {
00463         return false;
00464       }
00465       // Save the current file and position so that we can restore them
00466       // if we fail to restore the desired event
00467       bool closedOriginalFile = false;
00468       std::vector<FileCatalogItem>::const_iterator originalFile = fileIter_;
00469       IndexIntoFile::IndexIntoFileItr originalPosition = rootFile_->indexIntoFileIter();
00470 
00471       // Look for item (run/lumi/event) in files previously opened without reopening unnecessary files.
00472       typedef std::vector<boost::shared_ptr<IndexIntoFile> >::const_iterator Iter;
00473       for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00474         if(*it && (*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) {
00475           // We found it. Close the currently open file, and open the correct one.
00476           fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00477           initFile(false);
00478           // Now get the item from the correct file.
00479           bool found = rootFile_->goToEvent(eventID);
00480           assert (found);
00481           return true;
00482         }
00483       }
00484       // Look for item in files not yet opened.
00485       for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00486         if(!*it) {
00487           fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00488           initFile(false);
00489           closedOriginalFile = true;
00490           if((*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) {
00491             if  (rootFile_->goToEvent(eventID)) {
00492               return true;
00493             }
00494           }
00495         }
00496       }
00497       if(closedOriginalFile) {
00498         fileIter_ = originalFile;
00499         initFile(false);
00500         rootFile_->setPosition(originalPosition);
00501       }
00502     }
00503     return false;
00504   }
00505 
00506   bool
00507   RootInputFileSequence::skipToItemInNewFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) {
00508     // Look for item in files not yet opened.
00509     typedef std::vector<boost::shared_ptr<IndexIntoFile> >::const_iterator Iter;
00510     for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00511       if(!*it) {
00512         fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00513         initFile(false);
00514         bool found = rootFile_->setEntryAtItem(run, lumi, event);
00515         if(found) {
00516           return true;
00517         }
00518       }
00519     }
00520     // Not found
00521     return false;
00522   }
00523 
00524   bool
00525   RootInputFileSequence::skipToItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, bool currentFileFirst) {
00526     // Attempt to find item in currently open input file.
00527     bool found = currentFileFirst && rootFile_ && rootFile_->setEntryAtItem(run, lumi, event);
00528     if(!found) {
00529       // If only one input file, give up now, to save time.
00530       if(currentFileFirst && rootFile_ && indexesIntoFiles_.size() == 1) {
00531         return false;
00532       }
00533       // Look for item (run/lumi/event) in files previously opened without reopening unnecessary files.
00534       typedef std::vector<boost::shared_ptr<IndexIntoFile> >::const_iterator Iter;
00535       for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00536         if(*it && (*it)->containsItem(run, lumi, event)) {
00537           // We found it. Close the currently open file, and open the correct one.
00538           std::vector<FileCatalogItem>::const_iterator currentIter = fileIter_;
00539           fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00540           if(fileIter_ != currentIter) {
00541             initFile(false);
00542           }
00543           // Now get the item from the correct file.
00544           found = rootFile_->setEntryAtItem(run, lumi, event);
00545           assert (found);
00546           return true;
00547         }
00548       }
00549       // Look for item in files not yet opened.
00550       return skipToItemInNewFile(run, lumi, event);
00551     }
00552     return true;
00553   }
00554 
00555   ProcessConfiguration const&
00556   RootInputFileSequence::processConfiguration() const {
00557     return input_.processConfiguration();
00558   }
00559 
00560   int
00561   RootInputFileSequence::remainingEvents() const {
00562     return input_.remainingEvents();
00563   }
00564 
00565   int
00566   RootInputFileSequence::remainingLuminosityBlocks() const {
00567     return input_.remainingLuminosityBlocks();
00568   }
00569 
00570   ProductRegistry &
00571   RootInputFileSequence::productRegistryUpdate() const{
00572     return input_.productRegistryUpdate();
00573   }
00574 
00575   boost::shared_ptr<ProductRegistry const>
00576   RootInputFileSequence::productRegistry() const{
00577     return input_.productRegistry();
00578   }
00579 
00580   void
00581   RootInputFileSequence::dropUnwantedBranches_(std::vector<std::string> const& wantedBranches) {
00582     std::vector<std::string> rules;
00583     rules.reserve(wantedBranches.size() + 1);
00584     rules.emplace_back("drop *");
00585     for(std::vector<std::string>::const_iterator it = wantedBranches.begin(), itEnd = wantedBranches.end();
00586         it != itEnd; ++it) {
00587       rules.push_back("keep " + *it + "_*");
00588     }
00589     ParameterSet pset;
00590     pset.addUntrackedParameter("inputCommands", rules);
00591     productSelectorRules_ = ProductSelectorRules(pset, "inputCommands", "InputSource");
00592   }
00593 
00594   EventPrincipal*
00595   RootInputFileSequence::readOneSequential(EventPrincipal& cache) {
00596     skipBadFiles_ = false;
00597     if(fileIter_ == fileIterEnd_ || !rootFile_) {
00598       if(fileIterEnd_ == fileIterBegin_) {
00599         throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequential(): no input files specified.\n";
00600       }
00601       fileIter_ = fileIterBegin_;
00602       initFile(false);
00603       rootFile_->setAtEventEntry(-1);
00604     }
00605     rootFile_->nextEventEntry();
00606     EventPrincipal* ep = rootFile_->readCurrentEvent(cache);
00607     if(ep == 0) {
00608       ++fileIter_;
00609       if(fileIter_ == fileIterEnd_) {
00610         return 0;
00611       }
00612       initFile(false);
00613       rootFile_->setAtEventEntry(-1);
00614       return readOneSequential(cache);
00615     }
00616     return ep;
00617   }
00618 
00619   EventPrincipal*
00620   RootInputFileSequence::readOneSequentialWithID(EventPrincipal& cache, LuminosityBlockID const& id) {
00621     if(fileIterEnd_ == fileIterBegin_) {
00622       throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequentialWithID(): no input files specified.\n";
00623     }
00624     skipBadFiles_ = false;
00625     if(fileIter_ == fileIterEnd_ || !rootFile_ ||
00626         rootFile_->indexIntoFileIter().run() != id.run() || 
00627         rootFile_->indexIntoFileIter().lumi() != id.luminosityBlock()) {
00628       bool found = skipToItem(id.run(), id.luminosityBlock(), 0, false);
00629       if(!found) {
00630         return 0;
00631       }
00632     }
00633     bool nextFound = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
00634     EventPrincipal* ep = (nextFound ? rootFile_->readCurrentEvent(cache) : 0);
00635     if(ep == 0) {
00636       bool found = skipToItemInNewFile(id.run(), id.luminosityBlock(), 0);
00637       if(found) {
00638         return readOneSequentialWithID(cache, id);
00639       }
00640     }
00641     return ep;
00642   }
00643 
00644   EventPrincipal*
00645   RootInputFileSequence::readOneSpecified(EventPrincipal& cache, EventID const& id) {
00646     skipBadFiles_ = false;
00647     bool found = skipToItem(id.run(), id.luminosityBlock(), id.event());
00648     if(!found) {
00649       throw Exception(errors::NotFound) <<
00650          "RootInputFileSequence::readOneSpecified(): Secondary Input file " <<
00651          fileIter_->fileName() <<
00652          " does not contain specified event:\n" << id << "\n";
00653     }
00654     EventPrincipal* ep = rootFile_->readCurrentEvent(cache);
00655     assert(ep != 0);
00656     return ep;
00657   }
00658 
00659   EventPrincipal*
00660   RootInputFileSequence::readOneRandom(EventPrincipal& cache) {
00661     if(fileIterEnd_ == fileIterBegin_) {
00662       throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandom(): no input files specified.\n";
00663     }
00664     if(!flatDistribution_) {
00665       Service<RandomNumberGenerator> rng;
00666       CLHEP::HepRandomEngine& engine = rng->getEngine();
00667       flatDistribution_.reset(new CLHEP::RandFlat(engine));
00668     }
00669     skipBadFiles_ = false;
00670     unsigned int currentSeqNumber = fileIter_ - fileIterBegin_;
00671     while(eventsRemainingInFile_ == 0) {
00672       fileIter_ = fileIterBegin_ + flatDistribution_->fireInt(fileCatalogItems().size());
00673       unsigned int newSeqNumber = fileIter_ - fileIterBegin_;
00674       if(newSeqNumber != currentSeqNumber) {
00675         initFile(false);
00676         currentSeqNumber = newSeqNumber;
00677       }
00678       eventsRemainingInFile_ = rootFile_->eventTree().entries();
00679       if(eventsRemainingInFile_ == 0) {
00680         throw Exception(errors::NotFound) <<
00681            "RootInputFileSequence::readOneRandom(): Secondary Input file " << fileIter_->fileName() << " contains no events.\n";
00682       }
00683       rootFile_->setAtEventEntry(flatDistribution_->fireInt(eventsRemainingInFile_) - 1);
00684     }
00685     rootFile_->nextEventEntry();
00686 
00687     EventPrincipal* ep = rootFile_->readCurrentEvent(cache);
00688     if(ep == 0) {
00689       rootFile_->setAtEventEntry(0);
00690       ep = rootFile_->readCurrentEvent(cache);
00691       assert(ep != 0);
00692     }
00693     --eventsRemainingInFile_;
00694     return ep;
00695   }
00696 
00697   // bool RootFile::setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) {
00698 
00699   EventPrincipal*
00700   RootInputFileSequence::readOneRandomWithID(EventPrincipal& cache, LuminosityBlockID const& id) {
00701     if(fileIterEnd_ == fileIterBegin_) {
00702       throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandomWithID(): no input files specified.\n";
00703     }
00704     if(!flatDistribution_) {
00705       Service<RandomNumberGenerator> rng;
00706       CLHEP::HepRandomEngine& engine = rng->getEngine();
00707       flatDistribution_.reset(new CLHEP::RandFlat(engine));
00708     }
00709     skipBadFiles_ = false;
00710     if(fileIter_ == fileIterEnd_ || !rootFile_ ||
00711         rootFile_->indexIntoFileIter().run() != id.run() || 
00712         rootFile_->indexIntoFileIter().lumi() != id.luminosityBlock()) {
00713       bool found = skipToItem(id.run(), id.luminosityBlock(), 0);
00714       if(!found) {
00715         return 0;
00716       }
00717       int eventsInLumi = 0;
00718       while(rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock())) ++eventsInLumi;
00719       found = skipToItem(id.run(), id.luminosityBlock(), 0);
00720       assert(found);
00721       int eventInLumi = flatDistribution_->fireInt(eventsInLumi);
00722       for(int i = 0; i < eventInLumi; ++i) {
00723         bool found = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
00724         assert(found);
00725       }
00726     }
00727     bool nextFound = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
00728     EventPrincipal* ep = (nextFound ? rootFile_->readCurrentEvent(cache) : 0);
00729     if(ep == 0) {
00730       bool found = rootFile_->setEntryAtItem(id.run(), id.luminosityBlock(), 0);
00731       if(found) {
00732         return readOneRandomWithID(cache, id);
00733       }
00734     }
00735     return ep;
00736   }
00737 
00738   void
00739   RootInputFileSequence::fillDescription(ParameterSetDescription & desc) {
00740     desc.addUntracked<unsigned int>("skipEvents", 0U)
00741         ->setComment("Skip the first 'skipEvents' events that otherwise would have been processed.");
00742     desc.addUntracked<bool>("noEventSort", true)
00743         ->setComment("True:  Process runs, lumis and events in the order they appear in the file (but see notes 1 and 2).\n"
00744                      "False: Process runs, lumis and events in each file in numerical order (run#, lumi#, event#) (but see note 3).\n"
00745                      "Note 1: Events within the same lumi will always be processed contiguously.\n"
00746                      "Note 2: Lumis within the same run will always be processed contiguously.\n"
00747                      "Note 3: Any sorting occurs independently in each input file (no sorting across input files).");
00748     desc.addUntracked<bool>("skipBadFiles", false)
00749         ->setComment("True:  Ignore any missing or unopenable input file.\n"
00750                      "False: Throw exception if missing or unopenable input file.");
00751     desc.addUntracked<unsigned int>("cacheSize", roottree::defaultCacheSize)
00752         ->setComment("Size of ROOT TTree prefetch cache.  Affects performance.");
00753     desc.addUntracked<int>("treeMaxVirtualSize", -1)
00754         ->setComment("Size of ROOT TTree TBasket cache.  Affects performance.");
00755     desc.addUntracked<unsigned int>("setRunNumber", 0U)
00756         ->setComment("If non-zero, change number of first run to this number. Apply same offset to all runs.  Allowed only for simulation.");
00757     desc.addUntracked<bool>("dropDescendantsOfDroppedBranches", true)
00758         ->setComment("If True, also drop on input any descendent of any branch dropped on input.");
00759     std::string defaultString("permissive");
00760     desc.addUntracked<std::string>("parametersMustMatch", defaultString)
00761         ->setComment("'strict':     Values of tracked parameters must be unique across all input files.\n"
00762                      "'permissive': Values of tracked parameters may differ across or within files.");
00763     desc.addUntracked<std::string>("branchesMustMatch", defaultString)
00764         ->setComment("'strict':     Branches in each input file must match those in the first file.\n"
00765                      "'permissive': Branches in each input file may be any subset of those in the first file.");
00766     desc.addUntracked<bool>("labelRawDataLikeMC", true)
00767         ->setComment("If True: replace module label for raw data to match MC. Also use 'LHC' as process.");
00768 
00769     ProductSelectorRules::fillDescription(desc, "inputCommands");
00770     EventSkipperByID::fillDescription(desc);
00771     DuplicateChecker::fillDescription(desc);
00772   }
00773 
00774   ProcessingController::ForwardState
00775   RootInputFileSequence::forwardState() const {
00776     if(rootFile_) {
00777       if(!rootFile_->wasLastEventJustRead()) {
00778         return ProcessingController::kEventsAheadInFile;
00779       }
00780       std::vector<FileCatalogItem>::const_iterator itr(fileIter_);
00781       if(itr != fileIterEnd_) ++itr;
00782       if(itr != fileIterEnd_) {
00783         return ProcessingController::kNextFileExists;
00784       }
00785       return ProcessingController::kAtLastEvent;
00786     }
00787     return ProcessingController::kUnknownForward;
00788   }
00789 
00790   ProcessingController::ReverseState
00791   RootInputFileSequence::reverseState() const {
00792     if(rootFile_) {
00793       if(!rootFile_->wasFirstEventJustRead()) {
00794         return ProcessingController::kEventsBackwardsInFile;
00795       }
00796       if(fileIter_ != fileIterBegin_) {
00797         return ProcessingController::kPreviousFileExists;
00798       }
00799       return ProcessingController::kAtFirstEvent;
00800     }
00801     return ProcessingController::kUnknownReverse;
00802   }
00803 }