egammaTools::EgammaDNNHelper Class Reference

#include <EgammaDNNHelper.h>

Public Member Functions
	EgammaDNNHelper (const DNNConfiguration &, const ModelSelector &sel, const std::vector< std::string > &availableVars)
std::vector< std::pair< uint, std::vector< float > > >	evaluate (const std::vector< std::map< std::string, float >> &candidates, const std::vector< tensorflow::Session *> &sessions) const
std::pair< uint, std::vector< float > >	getScaledInputs (const std::map< std::string, float > &variables) const
std::vector< tensorflow::Session * >	getSessions () const

Private Member Functions
void	initScalerFiles (const std::vector< std::string > &availableVars)
void	initTensorFlowGraphs ()

Private Attributes
const DNNConfiguration	cfg_
std::vector< std::vector< ScalerConfiguration > >	featuresMap_
std::vector< std::unique_ptr< const tensorflow::GraphDef > >	graphDefs_
const ModelSelector	modelSelector_
std::vector< uint >	nInputs_
uint	nModels_

Detailed Description

Definition at line 41 of file EgammaDNNHelper.h.

Constructor & Destructor Documentation

EgammaDNNHelper()

EgammaDNNHelper::EgammaDNNHelper	(	const DNNConfiguration &	cfg,
		const ModelSelector &	sel,
		const std::vector< std::string > &	availableVars
	)

Definition at line 9 of file EgammaDNNHelper.cc.

References initScalerFiles(), and initTensorFlowGraphs().

    : cfg_(cfg), modelSelector_(modelSelector), nModels_(cfg_.modelsFiles.size()), graphDefs_(cfg_.modelsFiles.size()) {
  initTensorFlowGraphs();
  initScalerFiles(availableVars);
}

Member Function Documentation

evaluate()

std::vector< std::pair< uint, std::vector< float > > > EgammaDNNHelper::evaluate	(	const std::vector< std::map< std::string, float >> &	candidates,
		const std::vector< tensorflow::Session *> &	sessions
	)		const

Definition at line 100 of file EgammaDNNHelper.cc.

References a, b, HLT_2022v12_cff::candidates, cfg_, dqmiodumpmetadata::counts, data, dqmMemoryStats::float, getScaledInputs(), mps_fire::i, PixelMapPlotter::inputs, egammaTools::DNNConfiguration::inputTensorName, dqmdumpme::k, LogDebug, visualization-live-secondInstance_cfg::m, nInputs_, nModels_, convertSQLitetoXML_cfg::output, egammaTools::DNNConfiguration::outputDim, PatBasicFWLiteJetAnalyzer_Selector_cfg::outputs, egammaTools::DNNConfiguration::outputTensorName, alignCSCRings::r, mps_fire::result, tensorflow::run(), jetUpdater_cfi::sort, HcalDetIdTransform::transform(), and parallelization::uint.

Referenced by ElectronDNNEstimator::evaluate(), and PhotonDNNEstimator::evaluate().

                                                         {
  /*
    Evaluate the PFID DNN for all the electrons/photons. 
    nModels_ are defined depending on modelIndex  --> we need to build N input tensors to evaluate
    the DNNs with batching.
    
    1) Get all the variable for each candidate  vector<map<string:float>>
    2) Scale the input and select the variables for each model
    2) Prepare the input tensors for the  models
    3) Run the models and get the output for each candidate
    4) Sort the output by candidate index
    5) Return the DNN outputs along with the model index used on it

    */
  size_t nCandidates = candidates.size();
  std::vector<std::vector<uint>> indexMap(nModels_);  // for each model; the list of candidate index is saved
  std::vector<std::vector<float>> inputsVectors(nCandidates);
  std::vector<uint> counts(nModels_);

  LogDebug("EgammaDNNHelper") << "Working on " << nCandidates << " candidates";

  uint icand = 0;
  for (auto& candidate : candidates) {
    LogDebug("EgammaDNNHelper") << "Working on candidate: " << icand;
    const auto& [model_index, inputs] = getScaledInputs(candidate);
    counts[model_index] += 1;
    indexMap[model_index].push_back(icand);
    inputsVectors[icand] = inputs;
    icand++;
  }

  // Prepare one input tensors for each model
  std::vector<tensorflow::Tensor> input_tensors(nModels_);
  // Pointers for filling efficiently the input tensors
  std::vector<float*> input_tensors_pointer(nModels_);
  for (size_t i = 0; i < nModels_; i++) {
    LogDebug("EgammaDNNHelper") << "Initializing TF input " << i << " with rows:" << counts[i]
                                << " and cols:" << nInputs_[i];
    input_tensors[i] = tensorflow::Tensor{tensorflow::DT_FLOAT, {counts[i], nInputs_[i]}};
    input_tensors_pointer[i] = input_tensors[i].flat<float>().data();
  }

  // Filling the input tensors
  for (size_t m = 0; m < nModels_; m++) {
    LogDebug("EgammaDNNHelper") << "Loading TF input tensor for model: " << m;
    float* T = input_tensors_pointer[m];
    for (size_t cand_index : indexMap[m]) {
      for (size_t k = 0; k < nInputs_[m]; k++, T++) {  //Note the input tensor pointer incremented
        *T = inputsVectors[cand_index][k];
      }
    }
  }

  // Define the output and run
  // The initial output is [(cand_index,(model_index, outputs)),.. ]
  std::vector<std::pair<uint, std::pair<uint, std::vector<float>>>> outputs;
  // Run all the models
  for (size_t m = 0; m < nModels_; m++) {
    if (counts[m] == 0)
      continue;  //Skip model witout inputs
    std::vector<tensorflow::Tensor> output;
    LogDebug("EgammaDNNHelper") << "Run model: " << m << " with " << counts[m] << "objects";
    tensorflow::run(sessions[m], {{cfg_.inputTensorName, input_tensors[m]}}, {cfg_.outputTensorName}, &output);
    // Get the output and save the ElectronDNNEstimator::outputDim numbers along with the ele index
    const auto& r = output[0].tensor<float, 2>();
    // Iterate on the list of elements in the batch --> many electrons
    LogDebug("EgammaDNNHelper") << "Model " << m << " has " << cfg_.outputDim[m] << " nodes!";
    for (uint b = 0; b < counts[m]; b++) {
      //auto outputDim=cfg_.outputDim;
      std::vector<float> result(cfg_.outputDim[m]);
      for (size_t k = 0; k < cfg_.outputDim[m]; k++) {
        result[k] = r(b, k);
        LogDebug("EgammaDNNHelper") << "For Object " << b + 1 << " : Node " << k + 1 << " score = " << r(b, k);
      }
      // Get the original index of the electorn in the original order
      const auto cand_index = indexMap[m][b];
      outputs.push_back(std::make_pair(cand_index, std::make_pair(m, result)));
    }
  }
  // Now we have just to re-order the outputs
  std::sort(outputs.begin(), outputs.end());
  std::vector<std::pair<uint, std::vector<float>>> final_outputs(outputs.size());
  std::transform(outputs.begin(), outputs.end(), final_outputs.begin(), [](auto a) { return a.second; });

  return final_outputs;
}

getScaledInputs()

std::pair< uint, std::vector< float > > EgammaDNNHelper::getScaledInputs

(

const std::map< std::string, float > &

variables

)

const

Definition at line 76 of file EgammaDNNHelper.cc.

References featuresMap_, PixelMapPlotter::inputs, modelSelector_, pfMETCorrectionType0_cfi::par1, pfMETCorrectionType0_cfi::par2, and L1TEGammaDiff_cfi::variables.

Referenced by evaluate().

                                                     {
  // Call the modelSelector function passing the variables map to return
  // the modelIndex to be used for the current candidate
  const auto modelIndex = modelSelector_(variables);
  std::vector<float> inputs;
  // Loop on the list of requested variables and scaling values for the specific modelIndex
  // Different type of scaling are available: 0=no scaling, 1=standard scaler, 2=minmax
  for (auto& [varName, type, par1, par2] : featuresMap_[modelIndex]) {
    if (type == 1)  // Standard scaling
      inputs.push_back((variables.at(varName) - par1) / par2);
    else if (type == 2)  // MinMax
      inputs.push_back((variables.at(varName) - par1) / (par2 - par1));
    else if (type == 3)  //2*((X_train - minValues)/(MaxMinusMin)) -1.0
      inputs.push_back(2 * (variables.at(varName) - par1) / (par2 - par1) - 1.);
    else {
      inputs.push_back(variables.at(varName));  // Do nothing on the variable
    }
    //Protection for mismatch between requested variables and the available ones
    // have been added when the scaler config are loaded --> here we know that the variables are available
  }
  return std::make_pair(modelIndex, inputs);
}

getSessions()

std::vector< tensorflow::Session * > EgammaDNNHelper::getSessions

(

)

const

Definition at line 28 of file EgammaDNNHelper.cc.

References tensorflow::createSession(), graphDefs_, LogDebug, and nModels_.

Referenced by ElectronDNNEstimator::getSessions(), and PhotonDNNEstimator::getSessions().

                                                                 {
  std::vector<tensorflow::Session*> sessions;
  LogDebug("EgammaDNNHelper") << "Starting " << nModels_ << " TF sessions";
  for (const auto& graphDef : graphDefs_) {
    sessions.push_back(tensorflow::createSession(graphDef.get()));
  }
  LogDebug("EgammaDNNHelper") << "TF sessions started";
  return sessions;
}

initScalerFiles()

void EgammaDNNHelper::initScalerFiles ( const std::vector< std::string > &  availableVars )

private

Definition at line 38 of file EgammaDNNHelper.cc.

References cfg_, mps_fire::end, Exception, lowptgsfeleseed::features(), featuresMap_, spr::find(), edm::FileInPath::fullPath(), match(), nInputs_, pfMETCorrectionType0_cfi::par1, pfMETCorrectionType0_cfi::par2, egammaTools::DNNConfiguration::scalersFiles, AlCaHLTBitMon_QueryRunRegistry::string, parallelization::uint, and egammaTools::ScalerConfiguration::varName.

Referenced by EgammaDNNHelper().

                                                                               {
  for (const auto& scaler_file : cfg_.scalersFiles) {
    // Parse scaler configuration
    std::vector<ScalerConfiguration> features;
    std::ifstream inputfile_scaler{edm::FileInPath(scaler_file).fullPath()};
    int ninputs = 0;
    if (inputfile_scaler.fail()) {
      throw cms::Exception("MissingFile") << "Scaler file for PFid DNN not found";
    } else {
      // Now read mean, scale factors for each variable
      float par1, par2;
      std::string varName, type_str;
      uint type;
      while (inputfile_scaler >> varName >> type_str >> par1 >> par2) {
        if (type_str == "stdscale")
          type = 1;
        else if (type_str == "minmax")
          type = 2;
        else if (type_str == "custom1")  // 2*((X_train - minValues)/(MaxMinusMin)) -1.0
          type = 3;
        else
          type = 0;
        features.push_back(ScalerConfiguration{.varName = varName, .type = type, .par1 = par1, .par2 = par2});
        // Protection for mismatch between requested variables and the available ones
        auto match = std::find(availableVars.begin(), availableVars.end(), varName);
        if (match == std::end(availableVars)) {
          throw cms::Exception("MissingVariable")
              << "Requested variable (" << varName << ") not available between DNN inputs";
        }
        ninputs += 1;
      }
    }
    inputfile_scaler.close();
    featuresMap_.push_back(features);
    nInputs_.push_back(ninputs);
  }
}

initTensorFlowGraphs()

void EgammaDNNHelper::initTensorFlowGraphs ( )

private

Definition at line 17 of file EgammaDNNHelper.cc.

References cfg_, contentValuesFiles::fullPath, graphDefs_, mps_fire::i, tensorflow::loadGraphDef(), LogDebug, egammaTools::DNNConfiguration::modelsFiles, and nModels_.

Referenced by EgammaDNNHelper().

                                           {
  // load the graph definition
  LogDebug("EgammaDNNHelper") << "Loading " << nModels_ << " graphs";
  size_t i = 0;
  for (const auto& model_file : cfg_.modelsFiles) {
    graphDefs_[i] =
        std::unique_ptr<tensorflow::GraphDef>(tensorflow::loadGraphDef(edm::FileInPath(model_file).fullPath()));
    i++;
  }
}

Member Data Documentation

cfg_

const DNNConfiguration egammaTools::EgammaDNNHelper::cfg_

private

Definition at line 60 of file EgammaDNNHelper.h.

Referenced by evaluate(), initScalerFiles(), and initTensorFlowGraphs().

featuresMap_

std::vector<std::vector<ScalerConfiguration> > egammaTools::EgammaDNNHelper::featuresMap_

private

Definition at line 70 of file EgammaDNNHelper.h.

Referenced by getScaledInputs(), and initScalerFiles().

graphDefs_

std::vector<std::unique_ptr<const tensorflow::GraphDef> > egammaTools::EgammaDNNHelper::graphDefs_

private

Definition at line 67 of file EgammaDNNHelper.h.

Referenced by getSessions(), and initTensorFlowGraphs().

modelSelector_

const ModelSelector egammaTools::EgammaDNNHelper::modelSelector_

private

Definition at line 61 of file EgammaDNNHelper.h.

Referenced by getScaledInputs().

nInputs_

std::vector<uint> egammaTools::EgammaDNNHelper::nInputs_

private

Definition at line 65 of file EgammaDNNHelper.h.

Referenced by evaluate(), and initScalerFiles().

nModels_

uint egammaTools::EgammaDNNHelper::nModels_

private

Definition at line 63 of file EgammaDNNHelper.h.

Referenced by evaluate(), getSessions(), and initTensorFlowGraphs().