ProductRegistry.cc

Go to the documentation of this file.

 
#include "DataFormats/Provenance/interface/ProductRegistry.h"
 
#include "DataFormats/Provenance/interface/ProductResolverIndexHelper.h"
 
#include "FWCore/Utilities/interface/Algorithms.h"
#include "FWCore/Utilities/interface/EDMException.h"
#include "FWCore/Reflection/interface/DictionaryTools.h"
#include "FWCore/Utilities/interface/TypeID.h"
#include "FWCore/Reflection/interface/TypeWithDict.h"
#include "FWCore/Utilities/interface/WrappedClassName.h"
 
#include "TDictAttributeMap.h"
 
#include <cassert>
#include <iterator>
#include <limits>
#include <set>
#include <sstream>
#include <ostream>
 
namespace edm {
 
  ProductRegistry::ProductRegistry() : productList_(), transient_() {}
 
  ProductRegistry::Transients::Transients()
      : frozen_(false),
        productProduced_(),
        anyProductProduced_(false),
        eventProductLookup_(new ProductResolverIndexHelper),
        lumiProductLookup_(new ProductResolverIndexHelper),
        runProductLookup_(new ProductResolverIndexHelper),
        eventNextIndexValue_(0),
        lumiNextIndexValue_(0),
        runNextIndexValue_(0),
 
        branchIDToIndex_() {
    for (bool& isProduced : productProduced_)
      isProduced = false;
  }
 
  void ProductRegistry::Transients::reset() {
    frozen_ = false;
    for (bool& isProduced : productProduced_)
      isProduced = false;
    anyProductProduced_ = false;
 
    // propagate_const<T> has no reset() function
    eventProductLookup_ = std::make_unique<ProductResolverIndexHelper>();
    lumiProductLookup_ = std::make_unique<ProductResolverIndexHelper>();
    runProductLookup_ = std::make_unique<ProductResolverIndexHelper>();
 
    eventNextIndexValue_ = 0;
    lumiNextIndexValue_ = 0;
    runNextIndexValue_ = 0;
 
    branchIDToIndex_.clear();
  }
 
  ProductRegistry::ProductRegistry(ProductList const& productList, bool toBeFrozen)
      : productList_(productList), transient_() {
    freezeIt(toBeFrozen);
  }
 
  void ProductRegistry::addProduct(BranchDescription const& productDesc, bool fromListener) {
    assert(productDesc.produced());
    throwIfFrozen();
    std::pair<ProductList::iterator, bool> ret =
        productList_.insert(std::make_pair(BranchKey(productDesc), productDesc));
    if (!ret.second) {
      auto const& previous = *productList_.find(BranchKey(productDesc));
      if (previous.second.produced()) {
        // Duplicate registration in current process
        throw Exception(errors::LogicError, "Duplicate Product Identifier")
            << "\nThe Framework requires a unique branch name for each product\n"
            << "which consists of four parts: a friendly class name, module label,\n"
            << "product instance name, and process name. A product has been\n"
            << "registered with a duplicate branch name. The most common way\n"
            << "to fix this error is to modify the product instance name in\n"
            << "one of the offending 'produces' function calls. Another fix\n"
            << "would be to delete one of them if they are for the same product.\n\n"
            << "    friendly class name = " << previous.second.friendlyClassName() << "\n"
            << "    module label = " << previous.second.moduleLabel() << "\n"
            << "    product instance name = " << previous.second.productInstanceName() << "\n"
            << "    process name = " << previous.second.processName() << "\n\n"
            << "The following additional information is not used as part of\n"
            << "the unique branch identifier.\n\n"
            << "    branch types = " << previous.second.branchType() << "  " << productDesc.branchType() << "\n"
            << "    class name = " << previous.second.fullClassName() << "\n\n"
            << "Note that if the four parts of the branch name are the same,\n"
            << "then this error will occur even if the branch types differ!\n\n";
      } else {
        // Duplicate registration in previous process
        throw Exception(errors::Configuration, "Duplicate Process Name.\n")
            << "The process name " << productDesc.processName() << " was previously used for products in the input.\n"
            << "This has caused branch name conflicts between input products and new products.\n"
            << "Please modify the configuration file to use a distinct process name.\n"
            << "Alternately, drop all input products using that process name and the\n"
            << "descendants of those products.\n";
      }
    }
    addCalled(productDesc, fromListener);
  }
 
  void ProductRegistry::addLabelAlias(BranchDescription const& productDesc,
                                      std::string const& labelAlias,
                                      std::string const& instanceAlias) {
    assert(productDesc.produced());
    assert(productDesc.branchID().isValid());
    throwIfFrozen();
    BranchDescription bd(productDesc, labelAlias, instanceAlias);
    std::pair<ProductList::iterator, bool> ret = productList_.insert(std::make_pair(BranchKey(bd), bd));
    assert(ret.second);
    transient_.aliasToOriginal_.emplace_back(labelAlias, productDesc.moduleLabel());
    addCalled(bd, false);
  }
 
  void ProductRegistry::copyProduct(BranchDescription const& productDesc) {
    assert(!productDesc.produced());
    throwIfFrozen();
    BranchKey k = BranchKey(productDesc);
    ProductList::iterator iter = productList_.find(k);
    if (iter == productList_.end()) {
      productList_.insert(std::make_pair(k, productDesc));
    } else {
      assert(combinable(iter->second, productDesc));
      iter->second.merge(productDesc);
    }
  }
 
  bool ProductRegistry::anyProducts(BranchType brType) const {
    throwIfNotFrozen();
    for (ProductList::const_iterator it = productList_.begin(), itEnd = productList_.end(); it != itEnd; ++it) {
      if (it->second.branchType() == brType) {
        return true;
      }
    }
    return false;
  }
 
  std::shared_ptr<ProductResolverIndexHelper const> ProductRegistry::productLookup(BranchType branchType) const {
    if (branchType == InEvent)
      return transient_.eventProductLookup();
    if (branchType == InLumi)
      return transient_.lumiProductLookup();
    return transient_.runProductLookup();
  }
 
  std::shared_ptr<ProductResolverIndexHelper> ProductRegistry::productLookup(BranchType branchType) {
    if (branchType == InEvent)
      return transient_.eventProductLookup();
    if (branchType == InLumi)
      return transient_.lumiProductLookup();
    return transient_.runProductLookup();
  }
 
  void ProductRegistry::setFrozen(bool initializeLookupInfo) {
    if (frozen())
      return;
    freezeIt();
    if (initializeLookupInfo) {
      initializeLookupTables(nullptr, nullptr, nullptr);
    }
    sort_all(transient_.aliasToOriginal_);
  }
 
  void ProductRegistry::setFrozen(std::set<TypeID> const& productTypesConsumed,
                                  std::set<TypeID> const& elementTypesConsumed,
                                  std::string const& processName) {
    if (frozen())
      return;
    freezeIt();
    initializeLookupTables(&productTypesConsumed, &elementTypesConsumed, &processName);
    sort_all(transient_.aliasToOriginal_);
  }
 
  void ProductRegistry::throwIfFrozen() const {
    if (frozen()) {
      throw cms::Exception("ProductRegistry", "throwIfFrozen")
          << "cannot modify the ProductRegistry because it is frozen\n";
    }
  }
 
  void ProductRegistry::throwIfNotFrozen() const {
    if (!frozen()) {
      throw cms::Exception("ProductRegistry", "throwIfNotFrozen")
          << "cannot read the ProductRegistry because it is not yet frozen\n";
    }
  }
 
  void ProductRegistry::addCalled(BranchDescription const&, bool) {}
 
  std::vector<std::string> ProductRegistry::allBranchNames() const {
    std::vector<std::string> result;
    result.reserve(productList().size());
 
    for (auto const& product : productList()) {
      result.push_back(product.second.branchName());
    }
    return result;
  }
 
  std::vector<BranchDescription const*> ProductRegistry::allBranchDescriptions() const {
    std::vector<BranchDescription const*> result;
    result.reserve(productList().size());
 
    for (auto const& product : productList()) {
      result.push_back(&product.second);
    }
    return result;
  }
 
  void ProductRegistry::updateFromInput(ProductList const& other) {
    for (auto const& product : other) {
      copyProduct(product.second);
    }
  }
 
  void ProductRegistry::updateFromInput(std::vector<BranchDescription> const& other) {
    for (BranchDescription const& branchDescription : other) {
      copyProduct(branchDescription);
    }
  }
 
  void ProductRegistry::setUnscheduledProducts(std::set<std::string> const& unscheduledLabels) {
    throwIfFrozen();
 
    bool hasAliases = false;
    std::vector<BranchID> onDemandIDs;
    for (auto& prod : productList_) {
      if (prod.second.produced() && prod.second.branchType() == InEvent &&
          unscheduledLabels.end() != unscheduledLabels.find(prod.second.moduleLabel())) {
        prod.second.setOnDemand(true);
        onDemandIDs.push_back(prod.second.branchID());
      }
      if (prod.second.produced() && prod.second.isAlias()) {
        hasAliases = true;
      }
    }
 
    // Need to loop over EDAliases to set their on-demand flag based on the pointed-to branch
    if (hasAliases) {
      std::sort(onDemandIDs.begin(), onDemandIDs.end());
      for (auto& prod : productList_) {
        if (prod.second.isAlias()) {
          if (std::binary_search(onDemandIDs.begin(), onDemandIDs.end(), prod.second.aliasForBranchID())) {
            prod.second.setOnDemand(true);
          }
        }
      }
    }
  }
 
  std::string ProductRegistry::merge(ProductRegistry const& other,
                                     std::string const& fileName,
                                     BranchDescription::MatchMode branchesMustMatch) {
    std::ostringstream differences;
 
    ProductRegistry::ProductList::iterator j = productList_.begin();
    ProductRegistry::ProductList::iterator s = productList_.end();
    ProductRegistry::ProductList::const_iterator i = other.productList().begin();
    ProductRegistry::ProductList::const_iterator e = other.productList().end();
 
    // Loop over entries in the main product registry.
    while (j != s || i != e) {
      if (j != s && j->second.produced()) {
        // Ignore branches just produced (i.e. not in input file).
        ++j;
      } else if (j == s || (i != e && i->first < j->first)) {
        if (i->second.present()) {
          differences << "Branch '" << i->second.branchName() << "' is in file '" << fileName << "'\n";
          differences << "    but not in previous files.\n";
        } else {
          productList_.insert(*i);
          transient_.branchIDToIndex_[i->second.branchID()] = getNextIndexValue(i->second.branchType());
          ++nextIndexValue(i->second.branchType());
        }
        ++i;
      } else if (i == e || (j != s && j->first < i->first)) {
        if (j->second.present() && branchesMustMatch == BranchDescription::Strict) {
          differences << "Branch '" << j->second.branchName() << "' is in previous files\n";
          differences << "    but not in file '" << fileName << "'.\n";
        }
        ++j;
      } else {
        std::string difs = match(j->second, i->second, fileName);
        if (difs.empty()) {
          j->second.merge(i->second);
        } else {
          differences << difs;
        }
        ++i;
        ++j;
      }
    }
    return differences.str();
  }
 
  void ProductRegistry::initializeLookupTables(std::set<TypeID> const* productTypesConsumed,
                                               std::set<TypeID> const* elementTypesConsumed,
                                               std::string const* processName) {
    std::map<TypeID, TypeID> containedTypeMap;
    std::map<TypeID, std::vector<TypeWithDict> > containedTypeToBaseTypesMap;
 
    std::vector<std::string> missingDictionaries;
    std::vector<std::string> branchNamesForMissing;
    std::vector<std::string> producedTypes;
 
    transient_.branchIDToIndex_.clear();
 
    for (auto const& product : productList_) {
      auto const& desc = product.second;
 
      checkForDuplicateProcessName(desc, processName);
 
      if (desc.produced() && !desc.transient()) {
        setProductProduced(desc.branchType());
      }
 
      //only do the following if the data is supposed to be available in the event
      if (desc.present()) {
        // Check dictionaries (we already checked for the produced ones earlier somewhere else).
        // We have to have the dictionaries to properly setup the lookup tables for support of
        // Views. Also we need them to determine which present products are declared to be
        // consumed in the case where the consumed type is a View<T>.
        if (!desc.produced()) {
          if (!checkDictionary(missingDictionaries, desc.className(), desc.unwrappedType())) {
            checkDictionaryOfWrappedType(missingDictionaries, desc.className());
            branchNamesForMissing.emplace_back(desc.branchName());
            producedTypes.emplace_back(desc.className() + std::string(" (read from input)"));
            continue;
          }
        }
        TypeID typeID(desc.unwrappedType().typeInfo());
 
        auto iterContainedType = containedTypeMap.find(typeID);
        bool alreadySawThisType = (iterContainedType != containedTypeMap.end());
 
        if (!desc.produced() && !alreadySawThisType) {
          if (!checkDictionary(missingDictionaries, desc.wrappedName(), desc.wrappedType())) {
            branchNamesForMissing.emplace_back(desc.branchName());
            producedTypes.emplace_back(desc.className() + std::string(" (read from input)"));
            continue;
          }
        }
 
        TypeID wrappedTypeID(desc.wrappedType().typeInfo());
 
        TypeID containedTypeID;
        if (alreadySawThisType) {
          containedTypeID = iterContainedType->second;
        } else {
          containedTypeID = productholderindexhelper::getContainedTypeFromWrapper(wrappedTypeID, typeID.className());
        }
        bool hasContainedType = (containedTypeID != TypeID(typeid(void)) && containedTypeID != TypeID());
 
        std::vector<TypeWithDict>* baseTypesOfContainedType = nullptr;
 
        if (!alreadySawThisType) {
          bool alreadyCheckedConstituents = desc.produced() && !desc.transient();
          if (!alreadyCheckedConstituents && !desc.transient()) {
            // This checks dictionaries of the wrapped class and all its constituent classes
            if (!checkClassDictionaries(missingDictionaries, desc.wrappedName(), desc.wrappedType())) {
              branchNamesForMissing.emplace_back(desc.branchName());
              producedTypes.emplace_back(desc.className() + std::string(" (read from input)"));
              continue;
            }
          }
 
          if (hasContainedType) {
            auto iterBaseTypes = containedTypeToBaseTypesMap.find(containedTypeID);
            if (iterBaseTypes == containedTypeToBaseTypesMap.end()) {
              std::vector<TypeWithDict> baseTypes;
              if (!public_base_classes(missingDictionaries, containedTypeID, baseTypes)) {
                branchNamesForMissing.emplace_back(desc.branchName());
                if (desc.produced()) {
                  producedTypes.emplace_back(desc.className() + std::string(" (produced in current process)"));
                } else {
                  producedTypes.emplace_back(desc.className() + std::string(" (read from input)"));
                }
                continue;
              }
              iterBaseTypes = containedTypeToBaseTypesMap.insert(std::make_pair(containedTypeID, baseTypes)).first;
            }
            baseTypesOfContainedType = &iterBaseTypes->second;
          }
 
          // Do this after the dictionary checks of constituents so the list of branch names for missing types
          // is complete
          containedTypeMap.emplace(typeID, containedTypeID);
        } else {
          if (hasContainedType) {
            auto iterBaseTypes = containedTypeToBaseTypesMap.find(containedTypeID);
            if (iterBaseTypes != containedTypeToBaseTypesMap.end()) {
              baseTypesOfContainedType = &iterBaseTypes->second;
            }
          }
        }
 
        if (productTypesConsumed != nullptr && !desc.produced()) {
          bool mainTypeConsumed = (productTypesConsumed->find(typeID) != productTypesConsumed->end());
          bool containedTypeConsumed =
              hasContainedType && (elementTypesConsumed->find(containedTypeID) != elementTypesConsumed->end());
          if (hasContainedType && !containedTypeConsumed && baseTypesOfContainedType != nullptr) {
            for (TypeWithDict const& baseType : *baseTypesOfContainedType) {
              if (elementTypesConsumed->find(TypeID(baseType.typeInfo())) != elementTypesConsumed->end()) {
                containedTypeConsumed = true;
                break;
              }
            }
          }
          if (!containedTypeConsumed) {
            if (mainTypeConsumed) {
              // The main type is consumed, but either
              // there is no contained type, or if there is,
              // neither it nor any of its base classes are consumed.
              // Set the contained type, if there is one, to void,
              if (hasContainedType) {
                containedTypeID = TypeID(typeid(void));
              }
            } else {
              // The main type is not consumed, and either
              // there is no contained type, or if there is,
              // neither it nor any of its base classes are consumed.
              // Don't insert anything in the lookup tables.
              continue;
            }
          }
        }
        ProductResolverIndex index = productLookup(desc.branchType())
                                         ->insert(typeID,
                                                  desc.moduleLabel().c_str(),
                                                  desc.productInstanceName().c_str(),
                                                  desc.processName().c_str(),
                                                  containedTypeID,
                                                  baseTypesOfContainedType);
 
        transient_.branchIDToIndex_[desc.branchID()] = index;
      }
    }
    if (!missingDictionaries.empty()) {
      std::string context("Calling ProductRegistry::initializeLookupTables");
      throwMissingDictionariesException(missingDictionaries, context, producedTypes, branchNamesForMissing);
    }
 
    productLookup(InEvent)->setFrozen();
    productLookup(InLumi)->setFrozen();
    productLookup(InRun)->setFrozen();
 
    transient_.eventNextIndexValue_ = productLookup(InEvent)->nextIndexValue();
    transient_.lumiNextIndexValue_ = productLookup(InLumi)->nextIndexValue();
    transient_.runNextIndexValue_ = productLookup(InRun)->nextIndexValue();
 
    for (auto const& product : productList_) {
      auto const& desc = product.second;
      if (transient_.branchIDToIndex_.find(desc.branchID()) == transient_.branchIDToIndex_.end()) {
        transient_.branchIDToIndex_[desc.branchID()] = getNextIndexValue(desc.branchType());
        ++nextIndexValue(desc.branchType());
      }
    }
    checkDictionariesOfConsumedTypes(
        productTypesConsumed, elementTypesConsumed, containedTypeMap, containedTypeToBaseTypesMap);
  }
 
  void ProductRegistry::checkDictionariesOfConsumedTypes(
      std::set<TypeID> const* productTypesConsumed,
      std::set<TypeID> const* elementTypesConsumed,
      std::map<TypeID, TypeID> const& containedTypeMap,
      std::map<TypeID, std::vector<TypeWithDict> >& containedTypeToBaseTypesMap) {
    std::vector<std::string> missingDictionaries;
    std::set<std::string> consumedTypesWithMissingDictionaries;
 
    if (productTypesConsumed) {
      // Check dictionaries for all classes declared to be consumed
      for (auto const& consumedTypeID : *productTypesConsumed) {
        // We use the containedTypeMap to see which types have already
        // had their dictionaries checked. We do not waste time rechecking
        // those dictionaries.
        if (containedTypeMap.find(consumedTypeID) == containedTypeMap.end()) {
          std::string wrappedName = wrappedClassName(consumedTypeID.className());
          TypeWithDict wrappedType = TypeWithDict::byName(wrappedName);
          if (!checkDictionary(missingDictionaries, wrappedName, wrappedType)) {
            checkDictionary(missingDictionaries, consumedTypeID);
            consumedTypesWithMissingDictionaries.emplace(consumedTypeID.className());
            continue;
          }
          bool transient = false;
          TDictAttributeMap* wp = wrappedType.getClass()->GetAttributeMap();
          if (wp && wp->HasKey("persistent") && !strcmp(wp->GetPropertyAsString("persistent"), "false")) {
            transient = true;
          }
          if (transient) {
            if (!checkDictionary(missingDictionaries, consumedTypeID)) {
              consumedTypesWithMissingDictionaries.emplace(consumedTypeID.className());
            }
 
            TypeID containedTypeID = productholderindexhelper::getContainedTypeFromWrapper(
                TypeID(wrappedType.typeInfo()), consumedTypeID.className());
            bool hasContainedType = (containedTypeID != TypeID(typeid(void)) && containedTypeID != TypeID());
            if (hasContainedType) {
              if (containedTypeToBaseTypesMap.find(containedTypeID) == containedTypeToBaseTypesMap.end()) {
                std::vector<TypeWithDict> bases;
                // Run this to check for missing dictionaries, bases is not really used
                if (!public_base_classes(missingDictionaries, containedTypeID, bases)) {
                  consumedTypesWithMissingDictionaries.emplace(consumedTypeID.className());
                }
                containedTypeToBaseTypesMap.insert(std::make_pair(containedTypeID, bases));
              }
            }
          } else {
            if (!checkClassDictionaries(missingDictionaries, wrappedName, wrappedType)) {
              consumedTypesWithMissingDictionaries.emplace(consumedTypeID.className());
            }
          }
        }
      }
      if (!missingDictionaries.empty()) {
        std::string context(
            "Calling ProductRegistry::initializeLookupTables, checking dictionaries for consumed products");
        throwMissingDictionariesException(missingDictionaries, context, consumedTypesWithMissingDictionaries, false);
      }
    }
 
    if (elementTypesConsumed) {
      missingDictionaries.clear();
      consumedTypesWithMissingDictionaries.clear();
      for (auto const& consumedTypeID : *elementTypesConsumed) {
        if (containedTypeToBaseTypesMap.find(consumedTypeID) == containedTypeToBaseTypesMap.end()) {
          std::vector<TypeWithDict> bases;
          // Run this to check for missing dictionaries, bases is not really used
          if (!public_base_classes(missingDictionaries, consumedTypeID, bases)) {
            consumedTypesWithMissingDictionaries.emplace(consumedTypeID.className());
          }
        }
      }
      if (!missingDictionaries.empty()) {
        std::string context(
            "Calling ProductRegistry::initializeLookupTables, checking dictionaries for elements of products consumed "
            "using View");
        throwMissingDictionariesException(missingDictionaries, context, consumedTypesWithMissingDictionaries, true);
      }
    }
  }
 
  void ProductRegistry::checkForDuplicateProcessName(BranchDescription const& desc,
                                                     std::string const* processName) const {
    if (processName && !desc.produced() && (*processName == desc.processName())) {
      throw Exception(errors::Configuration, "Duplicate Process Name.\n")
          << "The process name " << *processName << " was previously used for products in the input.\n"
          << "Please modify the configuration file to use a distinct process name.\n"
          << "Alternately, drop all input products using that process name and the\n"
          << "descendants of those products.\n";
    }
  }
 
  ProductResolverIndex ProductRegistry::indexFrom(BranchID const& iID) const {
    std::map<BranchID, ProductResolverIndex>::const_iterator itFind = transient_.branchIDToIndex_.find(iID);
    if (itFind == transient_.branchIDToIndex_.end()) {
      return ProductResolverIndexInvalid;
    }
    return itFind->second;
  }
 
  void ProductRegistry::print(std::ostream& os) const {
    for (auto const& product : productList_) {
      os << product.second << "\n-----\n";
    }
  }
 
  ProductResolverIndex const& ProductRegistry::getNextIndexValue(BranchType branchType) const {
    if (branchType == InEvent)
      return transient_.eventNextIndexValue_;
    if (branchType == InLumi)
      return transient_.lumiNextIndexValue_;
    return transient_.runNextIndexValue_;
  }
 
  ProductResolverIndex& ProductRegistry::nextIndexValue(BranchType branchType) {
    if (branchType == InEvent)
      return transient_.eventNextIndexValue_;
    if (branchType == InLumi)
      return transient_.lumiNextIndexValue_;
    return transient_.runNextIndexValue_;
  }
}  // namespace edm