00001
00002
00003 #include "DuplicateChecker.h"
00004 #include "PoolSource.h"
00005 #include "RootFile.h"
00006 #include "RootInputFileSequence.h"
00007 #include "RootTree.h"
00008
00009 #include "DataFormats/Provenance/interface/BranchIDListHelper.h"
00010 #include "DataFormats/Provenance/interface/ProductRegistry.h"
00011 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
00012 #include "FWCore/Framework/interface/EventPrincipal.h"
00013 #include "FWCore/Framework/interface/FileBlock.h"
00014 #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h"
00015 #include "FWCore/Framework/interface/RunPrincipal.h"
00016 #include "FWCore/MessageLogger/interface/MessageLogger.h"
00017 #include "FWCore/ParameterSet/interface/ParameterSet.h"
00018 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
00019 #include "FWCore/ServiceRegistry/interface/Service.h"
00020 #include "FWCore/Utilities/interface/RandomNumberGenerator.h"
00021 #include "Utilities/StorageFactory/interface/StorageFactory.h"
00022
00023 #include "CLHEP/Random/RandFlat.h"
00024 #include "InputFile.h"
00025 #include "TSystem.h"
00026
00027 namespace edm {
00028 RootInputFileSequence::RootInputFileSequence(
00029 ParameterSet const& pset,
00030 PoolSource const& input,
00031 InputFileCatalog const& catalog,
00032 InputType::InputType inputType) :
00033 input_(input),
00034 inputType_(inputType),
00035 catalog_(catalog),
00036 firstFile_(true),
00037 lfn_("unknown"),
00038 fileIterBegin_(fileCatalogItems().begin()),
00039 fileIterEnd_(fileCatalogItems().end()),
00040 fileIter_(fileIterEnd_),
00041 fileIterLastOpened_(fileIterEnd_),
00042 rootFile_(),
00043 parametersMustMatch_(BranchDescription::Permissive),
00044 branchesMustMatch_(BranchDescription::Permissive),
00045 flatDistribution_(),
00046 indexesIntoFiles_(fileCatalogItems().size()),
00047 orderedProcessHistoryIDs_(),
00048 eventSkipperByID_(inputType == InputType::Primary ? EventSkipperByID::create(pset).release() : 0),
00049 eventsRemainingInFile_(0),
00050
00051
00052
00053
00054
00055
00056
00057 initialNumberOfEventsToSkip_(inputType == InputType::Primary ? pset.getUntrackedParameter<unsigned int>("skipEvents", 0U) : 0U),
00058 noEventSort_(inputType == InputType::Primary ? pset.getUntrackedParameter<bool>("noEventSort", true) : false),
00059 skipBadFiles_(pset.getUntrackedParameter<bool>("skipBadFiles", false)),
00060 treeCacheSize_(noEventSort_ ? pset.getUntrackedParameter<unsigned int>("cacheSize", roottree::defaultCacheSize) : 0U),
00061 treeMaxVirtualSize_(pset.getUntrackedParameter<int>("treeMaxVirtualSize", -1)),
00062 setRun_(pset.getUntrackedParameter<unsigned int>("setRunNumber", 0U)),
00063 productSelectorRules_(pset, "inputCommands", "InputSource"),
00064 duplicateChecker_(inputType == InputType::Primary ? new DuplicateChecker(pset) : 0),
00065 dropDescendants_(pset.getUntrackedParameter<bool>("dropDescendantsOfDroppedBranches", inputType != InputType::SecondarySource)),
00066 labelRawDataLikeMC_(pset.getUntrackedParameter<bool>("labelRawDataLikeMC", true)),
00067 usingGoToEvent_(false),
00068 enablePrefetching_(false),
00069 usedFallback_(false) {
00070
00071
00072 Service<SiteLocalConfig> pSLC;
00073 if(pSLC.isAvailable()) {
00074 if(treeCacheSize_ != 0U && pSLC->sourceTTreeCacheSize()) {
00075 treeCacheSize_ = *(pSLC->sourceTTreeCacheSize());
00076 }
00077 enablePrefetching_ = pSLC->enablePrefetching();
00078 }
00079
00080 StorageFactory *factory = StorageFactory::get();
00081 for(fileIter_ = fileIterBegin_; fileIter_ != fileIterEnd_; ++fileIter_) {
00082 factory->activateTimeout(fileIter_->fileName());
00083 factory->stagein(fileIter_->fileName());
00084
00085
00086
00087 if(inputType_ != InputType::Primary) {
00088 break;
00089 }
00090 }
00091
00092 std::string parametersMustMatch = pset.getUntrackedParameter<std::string>("parametersMustMatch", std::string("permissive"));
00093 if(parametersMustMatch == std::string("strict")) parametersMustMatch_ = BranchDescription::Strict;
00094
00095 std::string branchesMustMatch = pset.getUntrackedParameter<std::string>("branchesMustMatch", std::string("permissive"));
00096 if(branchesMustMatch == std::string("strict")) branchesMustMatch_ = BranchDescription::Strict;
00097
00098 for(fileIter_ = fileIterBegin_; fileIter_ != fileIterEnd_; ++fileIter_) {
00099 initFile(skipBadFiles_);
00100 if(rootFile_) break;
00101 }
00102 if(rootFile_) {
00103 productRegistryUpdate().updateFromInput(rootFile_->productRegistry()->productList());
00104 if(initialNumberOfEventsToSkip_ != 0) {
00105 skipEvents(initialNumberOfEventsToSkip_);
00106 }
00107 }
00108 }
00109
00110 std::vector<FileCatalogItem> const&
00111 RootInputFileSequence::fileCatalogItems() const {
00112 return catalog_.fileCatalogItems();
00113 }
00114
00115 void
00116 RootInputFileSequence::endJob() {
00117 closeFile_();
00118 }
00119
00120 std::unique_ptr<FileBlock>
00121 RootInputFileSequence::readFile_() {
00122 if(firstFile_) {
00123
00124 firstFile_ = false;
00125 if(!rootFile_) {
00126 initFile(skipBadFiles_);
00127 }
00128 } else {
00129 if(!nextFile()) {
00130 assert(0);
00131 }
00132 }
00133 if(!rootFile_) {
00134 return std::unique_ptr<FileBlock>(new FileBlock);
00135 }
00136 return rootFile_->createFileBlock();
00137 }
00138
00139 void RootInputFileSequence::closeFile_() {
00140
00141 if(rootFile_) {
00142 if(inputType_ != InputType::SecondarySource) {
00143 std::unique_ptr<InputSource::FileCloseSentry>
00144 sentry((inputType_ == InputType::Primary) ? new InputSource::FileCloseSentry(input_, lfn_, usedFallback_) : 0);
00145 rootFile_->close();
00146 if(duplicateChecker_) duplicateChecker_->inputFileClosed();
00147 }
00148 rootFile_.reset();
00149 }
00150 }
00151
00152 void RootInputFileSequence::initFile(bool skipBadFiles) {
00153
00154
00155
00156
00157
00158
00159 if(fileIterLastOpened_ != fileIterEnd_) {
00160 size_t currentIndexIntoFile = fileIterLastOpened_ - fileIterBegin_;
00161 bool needIndexesForDuplicateChecker = duplicateChecker_ && duplicateChecker_->checkingAllFiles() && !duplicateChecker_->checkDisabled();
00162 bool deleteIndexIntoFile = inputType_ == InputType::Primary &&
00163 !needIndexesForDuplicateChecker &&
00164 !usingGoToEvent_;
00165 if(deleteIndexIntoFile) {
00166 indexesIntoFiles_[currentIndexIntoFile].reset();
00167 } else {
00168 if(indexesIntoFiles_[currentIndexIntoFile]) indexesIntoFiles_[currentIndexIntoFile]->inputFileClosed();
00169 }
00170 fileIterLastOpened_ = fileIterEnd_;
00171 }
00172 closeFile_();
00173
00174 if(fileIter_ == fileIterEnd_) {
00175
00176 return;
00177 }
00178
00179
00180 if(fileIter_->fileName().empty()) {
00181
00182 InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00183 if(!skipBadFiles) {
00184 throw cms::Exception("LogicalFileNameNotFound", "RootInputFileSequence::initFile()\n")
00185 << "Logical file name '" << fileIter_->logicalFileName() << "' was not found in the file catalog.\n"
00186 << "If you wanted a local file, you forgot the 'file:' prefix\n"
00187 << "before the file name in your configuration file.\n";
00188 }
00189 LogWarning("") << "Input logical file: " << fileIter_->logicalFileName() << " was not found in the catalog, and will be skipped.\n";
00190 return;
00191 }
00192
00193 lfn_ = fileIter_->logicalFileName().empty() ? fileIter_->fileName() : fileIter_->logicalFileName();
00194
00195
00196
00197 std::string fallbackName = fileIter_->fallbackFileName();
00198 bool hasFallbackUrl = !fallbackName.empty() && fallbackName != fileIter_->fileName();
00199
00200 boost::shared_ptr<InputFile> filePtr;
00201 try {
00202 std::unique_ptr<InputSource::FileOpenSentry>
00203 sentry(inputType_ == InputType::Primary ? new InputSource::FileOpenSentry(input_) : 0);
00204 filePtr.reset(new InputFile(gSystem->ExpandPathName(fileIter_->fileName().c_str()), " Initiating request to open file "));
00205 }
00206 catch (cms::Exception const& e) {
00207 if(!skipBadFiles) {
00208 if(hasFallbackUrl) {
00209 std::ostringstream out;
00210 out << e.explainSelf();
00211 std::string pfn(gSystem->ExpandPathName(fallbackName.c_str()));
00212 InputFile::reportFallbackAttempt(pfn, fileIter_->logicalFileName(), out.str());
00213 } else {
00214 InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00215 Exception ex(errors::FileOpenError, "", e);
00216 ex.addContext("Calling RootInputFileSequence::initFile()");
00217 std::ostringstream out;
00218 out << "Input file " << fileIter_->fileName() << " could not be opened.";
00219 ex.addAdditionalInfo(out.str());
00220 throw ex;
00221 }
00222 }
00223 }
00224 if(!filePtr && (hasFallbackUrl)) {
00225 try {
00226 std::unique_ptr<InputSource::FileOpenSentry>
00227 sentry(inputType_ == InputType::Primary ? new InputSource::FileOpenSentry(input_) : 0);
00228 filePtr.reset(new InputFile(gSystem->ExpandPathName(fallbackName.c_str()), " Fallback request to file "));
00229 usedFallback_ = true;
00230 }
00231 catch (cms::Exception const& e) {
00232 if(!skipBadFiles) {
00233 InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00234 Exception ex(errors::FallbackFileOpenError, "", e);
00235 ex.addContext("Calling RootInputFileSequence::initFile()");
00236 std::ostringstream out;
00237 out << "Input file " << fileIter_->fileName() << " could not be opened.\n";
00238 out << "Fallback Input file " << fallbackName << " also could not be opened.";
00239 ex.addAdditionalInfo(out.str());
00240 throw ex;
00241 }
00242 }
00243 }
00244 if(filePtr) {
00245 std::vector<boost::shared_ptr<IndexIntoFile> >::size_type currentIndexIntoFile = fileIter_ - fileIterBegin_;
00246 rootFile_ = RootFileSharedPtr(new RootFile(fileIter_->fileName(),
00247 processConfiguration(), fileIter_->logicalFileName(), filePtr,
00248 eventSkipperByID_, initialNumberOfEventsToSkip_ != 0,
00249 remainingEvents(), remainingLuminosityBlocks(), treeCacheSize_, treeMaxVirtualSize_,
00250 input_.processingMode(),
00251 setRun_,
00252 noEventSort_,
00253 productSelectorRules_,
00254 inputType_,
00255 (inputType_ == InputType::SecondarySource ? boost::shared_ptr<BranchIDListHelper>(new BranchIDListHelper()) : input_.branchIDListHelper()),
00256 duplicateChecker_,
00257 dropDescendants_,
00258 indexesIntoFiles_,
00259 currentIndexIntoFile,
00260 orderedProcessHistoryIDs_,
00261 labelRawDataLikeMC_,
00262 usingGoToEvent_,
00263 enablePrefetching_));
00264
00265 fileIterLastOpened_ = fileIter_;
00266 indexesIntoFiles_[currentIndexIntoFile] = rootFile_->indexIntoFileSharedPtr();
00267 char const* inputType = 0;
00268 switch(inputType_) {
00269 case InputType::Primary: inputType = "primaryFiles"; break;
00270 case InputType::SecondaryFile: inputType = "secondaryFiles"; break;
00271 case InputType::SecondarySource: inputType = "mixingFiles"; break;
00272 }
00273 rootFile_->reportOpened(inputType);
00274 } else {
00275 InputFile::reportSkippedFile(fileIter_->fileName(), fileIter_->logicalFileName());
00276 if(!skipBadFiles) {
00277 throw Exception(errors::FileOpenError) <<
00278 "RootInputFileSequence::initFile(): Input file " << fileIter_->fileName() << " was not found or could not be opened.\n";
00279 }
00280 LogWarning("") << "Input file: " << fileIter_->fileName() << " was not found or could not be opened, and will be skipped.\n";
00281 }
00282 }
00283
00284 boost::shared_ptr<ProductRegistry const>
00285 RootInputFileSequence::fileProductRegistry() const {
00286 assert(rootFile_);
00287 return rootFile_->productRegistry();
00288 }
00289
00290 boost::shared_ptr<BranchIDListHelper const>
00291 RootInputFileSequence::fileBranchIDListHelper() const {
00292 assert(rootFile_);
00293 return rootFile_->branchIDListHelper();
00294 }
00295
00296 bool RootInputFileSequence::nextFile() {
00297 if(fileIter_ != fileIterEnd_) ++fileIter_;
00298 if(fileIter_ == fileIterEnd_) {
00299 if(inputType_ == InputType::Primary) {
00300 return false;
00301 } else {
00302 fileIter_ = fileIterBegin_;
00303 }
00304 }
00305
00306 initFile(skipBadFiles_);
00307
00308 if(inputType_ == InputType::Primary && rootFile_) {
00309
00310 std::string mergeInfo = productRegistryUpdate().merge(*rootFile_->productRegistry(),
00311 fileIter_->fileName(),
00312 parametersMustMatch_,
00313 branchesMustMatch_);
00314 if(!mergeInfo.empty()) {
00315 throw Exception(errors::MismatchedInputFiles,"RootInputFileSequence::nextFile()") << mergeInfo;
00316 }
00317 }
00318 return true;
00319 }
00320
00321 bool RootInputFileSequence::previousFile() {
00322 if(fileIter_ == fileIterBegin_) {
00323 if(inputType_ == InputType::Primary) {
00324 return false;
00325 } else {
00326 fileIter_ = fileIterEnd_;
00327 }
00328 }
00329 --fileIter_;
00330
00331 initFile(false);
00332
00333 if(inputType_ == InputType::Primary && rootFile_) {
00334
00335 std::string mergeInfo = productRegistryUpdate().merge(*rootFile_->productRegistry(),
00336 fileIter_->fileName(),
00337 parametersMustMatch_,
00338 branchesMustMatch_);
00339 if(!mergeInfo.empty()) {
00340 throw Exception(errors::MismatchedInputFiles,"RootInputFileSequence::previousEvent()") << mergeInfo;
00341 }
00342 }
00343 if(rootFile_) rootFile_->setToLastEntry();
00344 return true;
00345 }
00346
00347 RootInputFileSequence::~RootInputFileSequence() {
00348 }
00349
00350 boost::shared_ptr<RunAuxiliary>
00351 RootInputFileSequence::readRunAuxiliary_() {
00352 return rootFile_->readRunAuxiliary_();
00353 }
00354
00355 boost::shared_ptr<LuminosityBlockAuxiliary>
00356 RootInputFileSequence::readLuminosityBlockAuxiliary_() {
00357 return rootFile_->readLuminosityBlockAuxiliary_();
00358 }
00359
00360 boost::shared_ptr<RunPrincipal>
00361 RootInputFileSequence::readRun_(boost::shared_ptr<RunPrincipal> runPrincipal) {
00362 return rootFile_->readRun_(runPrincipal);
00363 }
00364
00365 boost::shared_ptr<LuminosityBlockPrincipal>
00366 RootInputFileSequence::readLuminosityBlock_(boost::shared_ptr<LuminosityBlockPrincipal> lumiPrincipal) {
00367 return rootFile_->readLumi(lumiPrincipal);
00368 }
00369
00370
00371
00372
00373
00374
00375
00376
00377
00378
00379
00380
00381
00382
00383 EventPrincipal*
00384 RootInputFileSequence::readEvent(EventPrincipal& eventPrincipal) {
00385 return rootFile_->readEvent(eventPrincipal);
00386 }
00387
00388 InputSource::ItemType
00389 RootInputFileSequence::getNextItemType() {
00390 if(fileIter_ == fileIterEnd_) {
00391 return InputSource::IsStop;
00392 }
00393 if(firstFile_) {
00394 return InputSource::IsFile;
00395 }
00396 if(rootFile_) {
00397 IndexIntoFile::EntryType entryType = rootFile_->getNextEntryTypeWanted();
00398 if(entryType == IndexIntoFile::kEvent) {
00399 return InputSource::IsEvent;
00400 } else if(entryType == IndexIntoFile::kLumi) {
00401 return InputSource::IsLumi;
00402 } else if(entryType == IndexIntoFile::kRun) {
00403 return InputSource::IsRun;
00404 }
00405 assert(entryType == IndexIntoFile::kEnd);
00406 }
00407 if(fileIter_ + 1 == fileIterEnd_) {
00408 return InputSource::IsStop;
00409 }
00410 return InputSource::IsFile;
00411 }
00412
00413
00414 void
00415 RootInputFileSequence::rewind_() {
00416 if(fileIter_ != fileIterBegin_) {
00417 closeFile_();
00418 fileIter_ = fileIterBegin_;
00419 }
00420 if(!rootFile_) {
00421 initFile(false);
00422 }
00423 rewindFile();
00424 firstFile_ = true;
00425 if(rootFile_) {
00426 if(initialNumberOfEventsToSkip_ != 0) {
00427 skipEvents(initialNumberOfEventsToSkip_);
00428 }
00429 }
00430 }
00431
00432
00433 void
00434 RootInputFileSequence::rewindFile() {
00435 if(rootFile_) rootFile_->rewind();
00436 }
00437
00438
00439 bool
00440 RootInputFileSequence::skipEvents(int offset) {
00441 while(offset != 0) {
00442 bool atEnd = rootFile_->skipEvents(offset);
00443 if((offset > 0 || atEnd) && !nextFile()) {
00444 return false;
00445 }
00446 if(offset < 0 && !previousFile()) {
00447 fileIter_ = fileIterEnd_;
00448 return false;
00449 }
00450 }
00451 return true;
00452 }
00453
00454 bool
00455 RootInputFileSequence::goToEvent(EventID const& eventID) {
00456 usingGoToEvent_ = true;
00457 if(rootFile_) {
00458 if(rootFile_->goToEvent(eventID)) {
00459 return true;
00460 }
00461
00462 if(rootFile_ && indexesIntoFiles_.size() == 1) {
00463 return false;
00464 }
00465
00466
00467 bool closedOriginalFile = false;
00468 std::vector<FileCatalogItem>::const_iterator originalFile = fileIter_;
00469 IndexIntoFile::IndexIntoFileItr originalPosition = rootFile_->indexIntoFileIter();
00470
00471
00472 typedef std::vector<boost::shared_ptr<IndexIntoFile> >::const_iterator Iter;
00473 for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00474 if(*it && (*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) {
00475
00476 fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00477 initFile(false);
00478
00479 bool found = rootFile_->goToEvent(eventID);
00480 assert (found);
00481 return true;
00482 }
00483 }
00484
00485 for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00486 if(!*it) {
00487 fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00488 initFile(false);
00489 closedOriginalFile = true;
00490 if((*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) {
00491 if (rootFile_->goToEvent(eventID)) {
00492 return true;
00493 }
00494 }
00495 }
00496 }
00497 if(closedOriginalFile) {
00498 fileIter_ = originalFile;
00499 initFile(false);
00500 rootFile_->setPosition(originalPosition);
00501 }
00502 }
00503 return false;
00504 }
00505
00506 bool
00507 RootInputFileSequence::skipToItemInNewFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) {
00508
00509 typedef std::vector<boost::shared_ptr<IndexIntoFile> >::const_iterator Iter;
00510 for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00511 if(!*it) {
00512 fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00513 initFile(false);
00514 bool found = rootFile_->setEntryAtItem(run, lumi, event);
00515 if(found) {
00516 return true;
00517 }
00518 }
00519 }
00520
00521 return false;
00522 }
00523
00524 bool
00525 RootInputFileSequence::skipToItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, bool currentFileFirst) {
00526
00527 bool found = currentFileFirst && rootFile_ && rootFile_->setEntryAtItem(run, lumi, event);
00528 if(!found) {
00529
00530 if(currentFileFirst && rootFile_ && indexesIntoFiles_.size() == 1) {
00531 return false;
00532 }
00533
00534 typedef std::vector<boost::shared_ptr<IndexIntoFile> >::const_iterator Iter;
00535 for(Iter it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) {
00536 if(*it && (*it)->containsItem(run, lumi, event)) {
00537
00538 std::vector<FileCatalogItem>::const_iterator currentIter = fileIter_;
00539 fileIter_ = fileIterBegin_ + (it - indexesIntoFiles_.begin());
00540 if(fileIter_ != currentIter) {
00541 initFile(false);
00542 }
00543
00544 found = rootFile_->setEntryAtItem(run, lumi, event);
00545 assert (found);
00546 return true;
00547 }
00548 }
00549
00550 return skipToItemInNewFile(run, lumi, event);
00551 }
00552 return true;
00553 }
00554
00555 ProcessConfiguration const&
00556 RootInputFileSequence::processConfiguration() const {
00557 return input_.processConfiguration();
00558 }
00559
00560 int
00561 RootInputFileSequence::remainingEvents() const {
00562 return input_.remainingEvents();
00563 }
00564
00565 int
00566 RootInputFileSequence::remainingLuminosityBlocks() const {
00567 return input_.remainingLuminosityBlocks();
00568 }
00569
00570 ProductRegistry &
00571 RootInputFileSequence::productRegistryUpdate() const{
00572 return input_.productRegistryUpdate();
00573 }
00574
00575 boost::shared_ptr<ProductRegistry const>
00576 RootInputFileSequence::productRegistry() const{
00577 return input_.productRegistry();
00578 }
00579
00580 void
00581 RootInputFileSequence::dropUnwantedBranches_(std::vector<std::string> const& wantedBranches) {
00582 std::vector<std::string> rules;
00583 rules.reserve(wantedBranches.size() + 1);
00584 rules.emplace_back("drop *");
00585 for(std::vector<std::string>::const_iterator it = wantedBranches.begin(), itEnd = wantedBranches.end();
00586 it != itEnd; ++it) {
00587 rules.push_back("keep " + *it + "_*");
00588 }
00589 ParameterSet pset;
00590 pset.addUntrackedParameter("inputCommands", rules);
00591 productSelectorRules_ = ProductSelectorRules(pset, "inputCommands", "InputSource");
00592 }
00593
00594 EventPrincipal*
00595 RootInputFileSequence::readOneSequential(EventPrincipal& cache) {
00596 skipBadFiles_ = false;
00597 if(fileIter_ == fileIterEnd_ || !rootFile_) {
00598 if(fileIterEnd_ == fileIterBegin_) {
00599 throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequential(): no input files specified.\n";
00600 }
00601 fileIter_ = fileIterBegin_;
00602 initFile(false);
00603 rootFile_->setAtEventEntry(-1);
00604 }
00605 rootFile_->nextEventEntry();
00606 EventPrincipal* ep = rootFile_->readCurrentEvent(cache);
00607 if(ep == 0) {
00608 ++fileIter_;
00609 if(fileIter_ == fileIterEnd_) {
00610 return 0;
00611 }
00612 initFile(false);
00613 rootFile_->setAtEventEntry(-1);
00614 return readOneSequential(cache);
00615 }
00616 return ep;
00617 }
00618
00619 EventPrincipal*
00620 RootInputFileSequence::readOneSequentialWithID(EventPrincipal& cache, LuminosityBlockID const& id) {
00621 if(fileIterEnd_ == fileIterBegin_) {
00622 throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequentialWithID(): no input files specified.\n";
00623 }
00624 skipBadFiles_ = false;
00625 if(fileIter_ == fileIterEnd_ || !rootFile_ ||
00626 rootFile_->indexIntoFileIter().run() != id.run() ||
00627 rootFile_->indexIntoFileIter().lumi() != id.luminosityBlock()) {
00628 bool found = skipToItem(id.run(), id.luminosityBlock(), 0, false);
00629 if(!found) {
00630 return 0;
00631 }
00632 }
00633 bool nextFound = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
00634 EventPrincipal* ep = (nextFound ? rootFile_->readCurrentEvent(cache) : 0);
00635 if(ep == 0) {
00636 bool found = skipToItemInNewFile(id.run(), id.luminosityBlock(), 0);
00637 if(found) {
00638 return readOneSequentialWithID(cache, id);
00639 }
00640 }
00641 return ep;
00642 }
00643
00644 EventPrincipal*
00645 RootInputFileSequence::readOneSpecified(EventPrincipal& cache, EventID const& id) {
00646 skipBadFiles_ = false;
00647 bool found = skipToItem(id.run(), id.luminosityBlock(), id.event());
00648 if(!found) {
00649 throw Exception(errors::NotFound) <<
00650 "RootInputFileSequence::readOneSpecified(): Secondary Input file " <<
00651 fileIter_->fileName() <<
00652 " does not contain specified event:\n" << id << "\n";
00653 }
00654 EventPrincipal* ep = rootFile_->readCurrentEvent(cache);
00655 assert(ep != 0);
00656 return ep;
00657 }
00658
00659 EventPrincipal*
00660 RootInputFileSequence::readOneRandom(EventPrincipal& cache) {
00661 if(fileIterEnd_ == fileIterBegin_) {
00662 throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandom(): no input files specified.\n";
00663 }
00664 if(!flatDistribution_) {
00665 Service<RandomNumberGenerator> rng;
00666 CLHEP::HepRandomEngine& engine = rng->getEngine();
00667 flatDistribution_.reset(new CLHEP::RandFlat(engine));
00668 }
00669 skipBadFiles_ = false;
00670 unsigned int currentSeqNumber = fileIter_ - fileIterBegin_;
00671 while(eventsRemainingInFile_ == 0) {
00672 fileIter_ = fileIterBegin_ + flatDistribution_->fireInt(fileCatalogItems().size());
00673 unsigned int newSeqNumber = fileIter_ - fileIterBegin_;
00674 if(newSeqNumber != currentSeqNumber) {
00675 initFile(false);
00676 currentSeqNumber = newSeqNumber;
00677 }
00678 eventsRemainingInFile_ = rootFile_->eventTree().entries();
00679 if(eventsRemainingInFile_ == 0) {
00680 throw Exception(errors::NotFound) <<
00681 "RootInputFileSequence::readOneRandom(): Secondary Input file " << fileIter_->fileName() << " contains no events.\n";
00682 }
00683 rootFile_->setAtEventEntry(flatDistribution_->fireInt(eventsRemainingInFile_) - 1);
00684 }
00685 rootFile_->nextEventEntry();
00686
00687 EventPrincipal* ep = rootFile_->readCurrentEvent(cache);
00688 if(ep == 0) {
00689 rootFile_->setAtEventEntry(0);
00690 ep = rootFile_->readCurrentEvent(cache);
00691 assert(ep != 0);
00692 }
00693 --eventsRemainingInFile_;
00694 return ep;
00695 }
00696
00697
00698
00699 EventPrincipal*
00700 RootInputFileSequence::readOneRandomWithID(EventPrincipal& cache, LuminosityBlockID const& id) {
00701 if(fileIterEnd_ == fileIterBegin_) {
00702 throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandomWithID(): no input files specified.\n";
00703 }
00704 if(!flatDistribution_) {
00705 Service<RandomNumberGenerator> rng;
00706 CLHEP::HepRandomEngine& engine = rng->getEngine();
00707 flatDistribution_.reset(new CLHEP::RandFlat(engine));
00708 }
00709 skipBadFiles_ = false;
00710 if(fileIter_ == fileIterEnd_ || !rootFile_ ||
00711 rootFile_->indexIntoFileIter().run() != id.run() ||
00712 rootFile_->indexIntoFileIter().lumi() != id.luminosityBlock()) {
00713 bool found = skipToItem(id.run(), id.luminosityBlock(), 0);
00714 if(!found) {
00715 return 0;
00716 }
00717 int eventsInLumi = 0;
00718 while(rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock())) ++eventsInLumi;
00719 found = skipToItem(id.run(), id.luminosityBlock(), 0);
00720 assert(found);
00721 int eventInLumi = flatDistribution_->fireInt(eventsInLumi);
00722 for(int i = 0; i < eventInLumi; ++i) {
00723 bool found = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
00724 assert(found);
00725 }
00726 }
00727 bool nextFound = rootFile_->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock());
00728 EventPrincipal* ep = (nextFound ? rootFile_->readCurrentEvent(cache) : 0);
00729 if(ep == 0) {
00730 bool found = rootFile_->setEntryAtItem(id.run(), id.luminosityBlock(), 0);
00731 if(found) {
00732 return readOneRandomWithID(cache, id);
00733 }
00734 }
00735 return ep;
00736 }
00737
00738 void
00739 RootInputFileSequence::fillDescription(ParameterSetDescription & desc) {
00740 desc.addUntracked<unsigned int>("skipEvents", 0U)
00741 ->setComment("Skip the first 'skipEvents' events that otherwise would have been processed.");
00742 desc.addUntracked<bool>("noEventSort", true)
00743 ->setComment("True: Process runs, lumis and events in the order they appear in the file (but see notes 1 and 2).\n"
00744 "False: Process runs, lumis and events in each file in numerical order (run#, lumi#, event#) (but see note 3).\n"
00745 "Note 1: Events within the same lumi will always be processed contiguously.\n"
00746 "Note 2: Lumis within the same run will always be processed contiguously.\n"
00747 "Note 3: Any sorting occurs independently in each input file (no sorting across input files).");
00748 desc.addUntracked<bool>("skipBadFiles", false)
00749 ->setComment("True: Ignore any missing or unopenable input file.\n"
00750 "False: Throw exception if missing or unopenable input file.");
00751 desc.addUntracked<unsigned int>("cacheSize", roottree::defaultCacheSize)
00752 ->setComment("Size of ROOT TTree prefetch cache. Affects performance.");
00753 desc.addUntracked<int>("treeMaxVirtualSize", -1)
00754 ->setComment("Size of ROOT TTree TBasket cache. Affects performance.");
00755 desc.addUntracked<unsigned int>("setRunNumber", 0U)
00756 ->setComment("If non-zero, change number of first run to this number. Apply same offset to all runs. Allowed only for simulation.");
00757 desc.addUntracked<bool>("dropDescendantsOfDroppedBranches", true)
00758 ->setComment("If True, also drop on input any descendent of any branch dropped on input.");
00759 std::string defaultString("permissive");
00760 desc.addUntracked<std::string>("parametersMustMatch", defaultString)
00761 ->setComment("'strict': Values of tracked parameters must be unique across all input files.\n"
00762 "'permissive': Values of tracked parameters may differ across or within files.");
00763 desc.addUntracked<std::string>("branchesMustMatch", defaultString)
00764 ->setComment("'strict': Branches in each input file must match those in the first file.\n"
00765 "'permissive': Branches in each input file may be any subset of those in the first file.");
00766 desc.addUntracked<bool>("labelRawDataLikeMC", true)
00767 ->setComment("If True: replace module label for raw data to match MC. Also use 'LHC' as process.");
00768
00769 ProductSelectorRules::fillDescription(desc, "inputCommands");
00770 EventSkipperByID::fillDescription(desc);
00771 DuplicateChecker::fillDescription(desc);
00772 }
00773
00774 ProcessingController::ForwardState
00775 RootInputFileSequence::forwardState() const {
00776 if(rootFile_) {
00777 if(!rootFile_->wasLastEventJustRead()) {
00778 return ProcessingController::kEventsAheadInFile;
00779 }
00780 std::vector<FileCatalogItem>::const_iterator itr(fileIter_);
00781 if(itr != fileIterEnd_) ++itr;
00782 if(itr != fileIterEnd_) {
00783 return ProcessingController::kNextFileExists;
00784 }
00785 return ProcessingController::kAtLastEvent;
00786 }
00787 return ProcessingController::kUnknownForward;
00788 }
00789
00790 ProcessingController::ReverseState
00791 RootInputFileSequence::reverseState() const {
00792 if(rootFile_) {
00793 if(!rootFile_->wasFirstEventJustRead()) {
00794 return ProcessingController::kEventsBackwardsInFile;
00795 }
00796 if(fileIter_ != fileIterBegin_) {
00797 return ProcessingController::kPreviousFileExists;
00798 }
00799 return ProcessingController::kAtFirstEvent;
00800 }
00801 return ProcessingController::kUnknownReverse;
00802 }
00803 }