CMS 3D CMS Logo

List of all members | Public Member Functions | Private Member Functions | Private Attributes
egammaTools::EgammaDNNHelper Class Reference

#include <EgammaDNNHelper.h>

Public Member Functions

 EgammaDNNHelper (const DNNConfiguration &, const ModelSelector &sel, const std::vector< std::string > &availableVars)
 
std::vector< std::pair< uint, std::vector< float > > > evaluate (const std::vector< std::map< std::string, float >> &candidates, const std::vector< tensorflow::Session *> &sessions) const
 
std::pair< uint, std::vector< float > > getScaledInputs (const std::map< std::string, float > &variables) const
 
std::vector< tensorflow::Session * > getSessions () const
 

Private Member Functions

void initScalerFiles (const std::vector< std::string > &availableVars)
 
void initTensorFlowGraphs ()
 

Private Attributes

const DNNConfiguration cfg_
 
std::vector< std::vector< ScalerConfiguration > > featuresMap_
 
std::vector< std::unique_ptr< const tensorflow::GraphDef > > graphDefs_
 
const ModelSelector modelSelector_
 
std::vector< uint > nInputs_
 
uint nModels_
 

Detailed Description

Definition at line 41 of file EgammaDNNHelper.h.

Constructor & Destructor Documentation

◆ EgammaDNNHelper()

EgammaDNNHelper::EgammaDNNHelper ( const DNNConfiguration cfg,
const ModelSelector sel,
const std::vector< std::string > &  availableVars 
)

Definition at line 9 of file EgammaDNNHelper.cc.

References initScalerFiles(), and initTensorFlowGraphs().

14  initScalerFiles(availableVars);
15 }
std::vector< std::unique_ptr< const tensorflow::GraphDef > > graphDefs_
std::vector< std::string > modelsFiles
void initScalerFiles(const std::vector< std::string > &availableVars)
const ModelSelector modelSelector_
const DNNConfiguration cfg_

Member Function Documentation

◆ evaluate()

std::vector< std::pair< uint, std::vector< float > > > EgammaDNNHelper::evaluate ( const std::vector< std::map< std::string, float >> &  candidates,
const std::vector< tensorflow::Session *> &  sessions 
) const

Definition at line 101 of file EgammaDNNHelper.cc.

References a, b, HLT_2024v12_cff::candidates, cfg_, dqmiodumpmetadata::counts, data, nano_mu_digi_cff::float, getScaledInputs(), mps_fire::i, PixelMapPlotter::inputs, egammaTools::DNNConfiguration::inputTensorName, dqmdumpme::k, LogDebug, visualization-live-secondInstance_cfg::m, nInputs_, nModels_, convertSQLitetoXML_cfg::output, egammaTools::DNNConfiguration::outputDim, PatBasicFWLiteJetAnalyzer_Selector_cfg::outputs, egammaTools::DNNConfiguration::outputTensorName, mps_fire::result, tensorflow::run(), jetUpdater_cfi::sort, HcalDetIdTransform::transform(), and parallelization::uint.

Referenced by ElectronDNNEstimator::evaluate(), and PhotonDNNEstimator::evaluate().

103  {
104  /*
105  Evaluate the PFID DNN for all the electrons/photons.
106  nModels_ are defined depending on modelIndex --> we need to build N input tensors to evaluate
107  the DNNs with batching.
108 
109  1) Get all the variable for each candidate vector<map<string:float>>
110  2) Scale the input and select the variables for each model
111  2) Prepare the input tensors for the models
112  3) Run the models and get the output for each candidate
113  4) Sort the output by candidate index
114  5) Return the DNN outputs along with the model index used on it
115 
116  */
117  size_t nCandidates = candidates.size();
118  std::vector<std::vector<uint>> indexMap(nModels_); // for each model; the list of candidate index is saved
119  std::vector<std::vector<float>> inputsVectors(nCandidates);
120  std::vector<uint> counts(nModels_);
121 
122  LogDebug("EgammaDNNHelper") << "Working on " << nCandidates << " candidates";
123 
124  uint icand = 0;
125  for (auto& candidate : candidates) {
126  LogDebug("EgammaDNNHelper") << "Working on candidate: " << icand;
127  const auto& [model_index, inputs] = getScaledInputs(candidate);
128  counts[model_index] += 1;
129  indexMap[model_index].push_back(icand);
130  inputsVectors[icand] = inputs;
131  icand++;
132  }
133 
134  // Prepare one input tensors for each model
135  std::vector<tensorflow::Tensor> input_tensors(nModels_);
136  // Pointers for filling efficiently the input tensors
137  std::vector<float*> input_tensors_pointer(nModels_);
138  for (size_t i = 0; i < nModels_; i++) {
139  LogDebug("EgammaDNNHelper") << "Initializing TF input " << i << " with rows:" << counts[i]
140  << " and cols:" << nInputs_[i];
141  input_tensors[i] = tensorflow::Tensor{tensorflow::DT_FLOAT, {counts[i], nInputs_[i]}};
142  input_tensors_pointer[i] = input_tensors[i].flat<float>().data();
143  }
144 
145  // Filling the input tensors
146  for (size_t m = 0; m < nModels_; m++) {
147  LogDebug("EgammaDNNHelper") << "Loading TF input tensor for model: " << m;
148  float* T = input_tensors_pointer[m];
149  for (size_t cand_index : indexMap[m]) {
150  for (size_t k = 0; k < nInputs_[m]; k++, T++) { //Note the input tensor pointer incremented
151  *T = inputsVectors[cand_index][k];
152  }
153  }
154  }
155 
156  // Define the output and run
157  // The initial output is [(cand_index,(model_index, outputs)),.. ]
158  std::vector<std::pair<uint, std::pair<uint, std::vector<float>>>> outputs;
159  // Run all the models
160  for (size_t m = 0; m < nModels_; m++) {
161  if (counts[m] == 0)
162  continue; //Skip model witout inputs
163  std::vector<tensorflow::Tensor> output;
164  LogDebug("EgammaDNNHelper") << "Run model: " << m << " with " << counts[m] << "objects";
165  tensorflow::run(sessions[m], {{cfg_.inputTensorName, input_tensors[m]}}, {cfg_.outputTensorName}, &output);
166  // Get the output and save the ElectronDNNEstimator::outputDim numbers along with the ele index
167  const auto& r = output[0].tensor<float, 2>();
168  // Iterate on the list of elements in the batch --> many electrons
169  LogDebug("EgammaDNNHelper") << "Model " << m << " has " << cfg_.outputDim[m] << " nodes!";
170  for (uint b = 0; b < counts[m]; b++) {
171  //auto outputDim=cfg_.outputDim;
172  std::vector<float> result(cfg_.outputDim[m]);
173  for (size_t k = 0; k < cfg_.outputDim[m]; k++) {
174  result[k] = r(b, k);
175  LogDebug("EgammaDNNHelper") << "For Object " << b + 1 << " : Node " << k + 1 << " score = " << r(b, k);
176  }
177  // Get the original index of the electorn in the original order
178  const auto cand_index = indexMap[m][b];
179  outputs.push_back(std::make_pair(cand_index, std::make_pair(m, result)));
180  }
181  }
182  // Now we have just to re-order the outputs
183  std::sort(outputs.begin(), outputs.end());
184  std::vector<std::pair<uint, std::vector<float>>> final_outputs(outputs.size());
185  std::transform(outputs.begin(), outputs.end(), final_outputs.begin(), [](auto a) { return a.second; });
186 
187  return final_outputs;
188 }
std::pair< uint, std::vector< float > > getScaledInputs(const std::map< std::string, float > &variables) const
std::vector< unsigned int > outputDim
std::vector< uint > nInputs_
void run(Session *session, const NamedTensorList &inputs, const std::vector< std::string > &outputNames, std::vector< Tensor > *outputs, const thread::ThreadPoolOptions &threadPoolOptions)
Definition: TensorFlow.cc:259
double b
Definition: hdecay.h:120
char data[epos_bytes_allocation]
Definition: EPOS_Wrapper.h:80
double a
Definition: hdecay.h:121
Definition: output.py:1
long double T
const DNNConfiguration cfg_
#define LogDebug(id)
unsigned transform(const HcalDetId &id, unsigned transformCode)

◆ getScaledInputs()

std::pair< uint, std::vector< float > > EgammaDNNHelper::getScaledInputs ( const std::map< std::string, float > &  variables) const

Definition at line 77 of file EgammaDNNHelper.cc.

References featuresMap_, PixelMapPlotter::inputs, modelSelector_, pfMETCorrectionType0_cfi::par1, pfMETCorrectionType0_cfi::par2, and hcalRecHitTable_cff::variables.

Referenced by evaluate().

78  {
79  // Call the modelSelector function passing the variables map to return
80  // the modelIndex to be used for the current candidate
81  const auto modelIndex = modelSelector_(variables);
82  std::vector<float> inputs;
83  // Loop on the list of requested variables and scaling values for the specific modelIndex
84  // Different type of scaling are available: 0=no scaling, 1=standard scaler, 2=minmax
85  for (auto& [varName, type, par1, par2] : featuresMap_[modelIndex]) {
86  if (type == 1) // Standard scaling
87  inputs.push_back((variables.at(varName) - par1) / par2);
88  else if (type == 2) // MinMax
89  inputs.push_back((variables.at(varName) - par1) / (par2 - par1));
90  else if (type == 3) //2*((X_train - minValues)/(MaxMinusMin)) -1.0
91  inputs.push_back(2 * (variables.at(varName) - par1) / (par2 - par1) - 1.);
92  else {
93  inputs.push_back(variables.at(varName)); // Do nothing on the variable
94  }
95  //Protection for mismatch between requested variables and the available ones
96  // have been added when the scaler config are loaded --> here we know that the variables are available
97  }
98  return std::make_pair(modelIndex, inputs);
99 }
std::vector< std::vector< ScalerConfiguration > > featuresMap_
const ModelSelector modelSelector_

◆ getSessions()

std::vector< tensorflow::Session * > EgammaDNNHelper::getSessions ( ) const

Definition at line 28 of file EgammaDNNHelper.cc.

References tensorflow::createSession(), graphDefs_, LogDebug, and nModels_.

Referenced by ElectronDNNEstimator::getSessions(), and PhotonDNNEstimator::getSessions().

28  {
29  std::vector<tensorflow::Session*> sessions;
30  LogDebug("EgammaDNNHelper") << "Starting " << nModels_ << " TF sessions";
31  sessions.reserve(graphDefs_.size());
32  for (const auto& graphDef : graphDefs_) {
33  sessions.push_back(tensorflow::createSession(graphDef.get()));
34  }
35  LogDebug("EgammaDNNHelper") << "TF sessions started";
36  return sessions;
37 }
std::vector< std::unique_ptr< const tensorflow::GraphDef > > graphDefs_
Session * createSession()
Definition: TensorFlow.cc:137
#define LogDebug(id)

◆ initScalerFiles()

void EgammaDNNHelper::initScalerFiles ( const std::vector< std::string > &  availableVars)
private

Definition at line 39 of file EgammaDNNHelper.cc.

References cfg_, Exception, lowptgsfeleseed::features(), featuresMap_, spr::find(), edm::FileInPath::fullPath(), nInputs_, pfMETCorrectionType0_cfi::par1, pfMETCorrectionType0_cfi::par2, egammaTools::DNNConfiguration::scalersFiles, AlCaHLTBitMon_QueryRunRegistry::string, parallelization::uint, and egammaTools::ScalerConfiguration::varName.

Referenced by EgammaDNNHelper().

39  {
40  for (const auto& scaler_file : cfg_.scalersFiles) {
41  // Parse scaler configuration
42  std::vector<ScalerConfiguration> features;
43  std::ifstream inputfile_scaler{edm::FileInPath(scaler_file).fullPath()};
44  int ninputs = 0;
45  if (inputfile_scaler.fail()) {
46  throw cms::Exception("MissingFile") << "Scaler file for PFid DNN not found";
47  } else {
48  // Now read mean, scale factors for each variable
49  float par1, par2;
50  std::string varName, type_str;
51  uint type;
52  while (inputfile_scaler >> varName >> type_str >> par1 >> par2) {
53  if (type_str == "stdscale")
54  type = 1;
55  else if (type_str == "minmax")
56  type = 2;
57  else if (type_str == "custom1") // 2*((X_train - minValues)/(MaxMinusMin)) -1.0
58  type = 3;
59  else
60  type = 0;
61  features.push_back(ScalerConfiguration{.varName = varName, .type = type, .par1 = par1, .par2 = par2});
62  // Protection for mismatch between requested variables and the available ones
63  auto match = std::find(availableVars.begin(), availableVars.end(), varName);
64  if (match == std::end(availableVars)) {
65  throw cms::Exception("MissingVariable")
66  << "Requested variable (" << varName << ") not available between DNN inputs";
67  }
68  ninputs += 1;
69  }
70  }
71  inputfile_scaler.close();
72  featuresMap_.push_back(features);
73  nInputs_.push_back(ninputs);
74  }
75 }
std::string fullPath() const
Definition: FileInPath.cc:161
std::vector< uint > nInputs_
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:19
std::vector< float > features(const reco::PreId &ecal, const reco::PreId &hcal, double rho, const reco::BeamSpot &spot, noZS::EcalClusterLazyTools &ecalTools)
std::vector< std::vector< ScalerConfiguration > > featuresMap_
std::vector< std::string > scalersFiles
std::pair< typename Association::data_type::first_type, double > match(Reference key, Association association, bool bestMatchByMaxValue)
Generic matching function.
Definition: Utils.h:10
const DNNConfiguration cfg_

◆ initTensorFlowGraphs()

void EgammaDNNHelper::initTensorFlowGraphs ( )
private

Definition at line 17 of file EgammaDNNHelper.cc.

References cfg_, contentValuesFiles::fullPath, graphDefs_, mps_fire::i, tensorflow::loadGraphDef(), LogDebug, egammaTools::DNNConfiguration::modelsFiles, and nModels_.

Referenced by EgammaDNNHelper().

17  {
18  // load the graph definition
19  LogDebug("EgammaDNNHelper") << "Loading " << nModels_ << " graphs";
20  size_t i = 0;
21  for (const auto& model_file : cfg_.modelsFiles) {
22  graphDefs_[i] =
23  std::unique_ptr<tensorflow::GraphDef>(tensorflow::loadGraphDef(edm::FileInPath(model_file).fullPath()));
24  i++;
25  }
26 }
std::vector< std::unique_ptr< const tensorflow::GraphDef > > graphDefs_
GraphDef * loadGraphDef(const std::string &pbFile)
Definition: TensorFlow.cc:120
std::vector< std::string > modelsFiles
const DNNConfiguration cfg_
#define LogDebug(id)

Member Data Documentation

◆ cfg_

const DNNConfiguration egammaTools::EgammaDNNHelper::cfg_
private

Definition at line 60 of file EgammaDNNHelper.h.

Referenced by evaluate(), initScalerFiles(), and initTensorFlowGraphs().

◆ featuresMap_

std::vector<std::vector<ScalerConfiguration> > egammaTools::EgammaDNNHelper::featuresMap_
private

Definition at line 70 of file EgammaDNNHelper.h.

Referenced by getScaledInputs(), and initScalerFiles().

◆ graphDefs_

std::vector<std::unique_ptr<const tensorflow::GraphDef> > egammaTools::EgammaDNNHelper::graphDefs_
private

Definition at line 67 of file EgammaDNNHelper.h.

Referenced by getSessions(), and initTensorFlowGraphs().

◆ modelSelector_

const ModelSelector egammaTools::EgammaDNNHelper::modelSelector_
private

Definition at line 61 of file EgammaDNNHelper.h.

Referenced by getScaledInputs().

◆ nInputs_

std::vector<uint> egammaTools::EgammaDNNHelper::nInputs_
private

Definition at line 65 of file EgammaDNNHelper.h.

Referenced by evaluate(), and initScalerFiles().

◆ nModels_

uint egammaTools::EgammaDNNHelper::nModels_
private

Definition at line 63 of file EgammaDNNHelper.h.

Referenced by evaluate(), getSessions(), and initTensorFlowGraphs().