DuplicateChecker.cc

Go to the documentation of this file.

#include "IOPool/Input/src/DuplicateChecker.h"
#include "FWCore/ParameterSet/interface/ParameterSet.h"
#include "FWCore/Utilities/interface/Exception.h"
#include "FWCore/MessageLogger/interface/MessageLogger.h"
#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"

#include <cassert>
#include <algorithm>

namespace edm {

  DuplicateChecker::DuplicateChecker(ParameterSet const& pset)
      : dataType_(unknown), itIsKnownTheFileHasNoDuplicates_(false), disabled_(false) {
    // The default value provided as the second argument to the getUntrackedParameter function call
    // is not used when the ParameterSet has been validated and the parameters are not optional
    // in the description.  This is currently true when PoolSource is the primary input source.
    // The modules that use PoolSource as a SecSource have not defined their fillDescriptions function
    // yet, so the ParameterSet does not get validated yet.  As soon as all the modules with a SecSource
    // have defined descriptions, the defaults in the getUntrackedParameterSet function calls can
    // and should be deleted from the code.
    std::string duplicateCheckMode =
        pset.getUntrackedParameter<std::string>("duplicateCheckMode", std::string("checkAllFilesOpened"));

    if (duplicateCheckMode == std::string("noDuplicateCheck"))
      duplicateCheckMode_ = noDuplicateCheck;
    else if (duplicateCheckMode == std::string("checkEachFile"))
      duplicateCheckMode_ = checkEachFile;
    else if (duplicateCheckMode == std::string("checkEachRealDataFile"))
      duplicateCheckMode_ = checkEachRealDataFile;
    else if (duplicateCheckMode == std::string("checkAllFilesOpened"))
      duplicateCheckMode_ = checkAllFilesOpened;
    else {
      throw cms::Exception("Configuration")
          << "Illegal configuration parameter value passed to PoolSource for\n"
          << "the \"duplicateCheckMode\" parameter, legal values are:\n"
          << "\"noDuplicateCheck\", \"checkEachFile\", \"checkEachRealDataFile\", \"checkAllFilesOpened\"\n";
    }
  }

  void DuplicateChecker::disable() {
    disabled_ = true;
    dataType_ = unknown;
    relevantPreviousEvents_.clear();
    itIsKnownTheFileHasNoDuplicates_ = false;
  }

  void DuplicateChecker::inputFileOpened(bool realData,
                                         IndexIntoFile const& indexIntoFile,
                                         std::vector<std::shared_ptr<IndexIntoFile> > const& indexesIntoFiles,
                                         std::vector<std::shared_ptr<IndexIntoFile> >::size_type currentIndexIntoFile) {
    dataType_ = realData ? isRealData : isSimulation;
    if (checkDisabled())
      return;

    relevantPreviousEvents_.clear();
    itIsKnownTheFileHasNoDuplicates_ = false;

    if (duplicateCheckMode_ == checkAllFilesOpened) {
      // Compares the current IndexIntoFile to all the previous ones and saves any duplicates.
      // One unintended thing, it also saves the duplicate runs and lumis.
      for (std::vector<std::shared_ptr<IndexIntoFile> >::size_type i = 0; i < currentIndexIntoFile; ++i) {
        if (indexesIntoFiles[i].get() != nullptr) {
          indexIntoFile.set_intersection(*indexesIntoFiles[i], relevantPreviousEvents_);
        }
      }
    }
    if (relevantPreviousEvents_.empty()) {
      if (!indexIntoFile.containsDuplicateEvents()) {
        itIsKnownTheFileHasNoDuplicates_ = true;
      }
    }
  }

  void DuplicateChecker::inputFileClosed() {
    dataType_ = unknown;
    relevantPreviousEvents_.clear();
    itIsKnownTheFileHasNoDuplicates_ = false;
  }

  bool DuplicateChecker::isDuplicateAndCheckActive(
      int index, RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, std::string const& fileName) {
    if (itIsKnownTheFileHasNoDuplicates_)
      return false;
    if (checkDisabled())
      return false;

    IndexIntoFile::IndexRunLumiEventKey newEvent(index, run, lumi, event);
    bool duplicate = !relevantPreviousEvents_.insert(newEvent).second;

    if (duplicate) {
      if (duplicateCheckMode_ == checkAllFilesOpened) {
        LogWarning("DuplicateEvent") << "Duplicate Events found in entire set of input files.\n"
                                     << "Both events were from run " << run << " and luminosity block " << lumi
                                     << " with event number " << event << ".\n"
                                     << "The duplicate was from file " << fileName << ".\n"
                                     << "The duplicate will be skipped.\n";
      } else {
        LogWarning("DuplicateEvent") << "Duplicate Events found in file " << fileName << ".\n"
                                     << "Both events were from run " << run << " and luminosity block " << lumi
                                     << " with event number " << event << ".\n"
                                     << "The duplicate will be skipped.\n";
      }
      return true;
    }
    return false;
  }

  void DuplicateChecker::fillDescription(ParameterSetDescription& desc) {
    std::string defaultString("checkAllFilesOpened");
    desc.addUntracked<std::string>("duplicateCheckMode", defaultString)
        ->setComment(
            "'checkAllFilesOpened':   check across all input files\n"
            "'checkEachFile':         check each input file independently\n"
            "'checkEachRealDataFile': check each real data input file independently\n"
            "'noDuplicateCheck':      no duplicate checking\n");
  }
}  // namespace edm