#include <TritonClient.h>

Inheritance diagram for TritonClient:

Classes
struct	ServerSideStats

Public Member Functions
unsigned	batchSize () const

void	reset () override

bool	setBatchSize (unsigned bsize)

	TritonClient (const edm::ParameterSet &params, const std::string &debugName)

bool	verbose () const

Public Member Functions inherited from SonicClient< TritonInputMap, TritonOutputMap >
	SonicClient (const edm::ParameterSet &params, const std::string &debugName, const std::string &clientName)

Public Member Functions inherited from SonicClientBase
const std::string &	clientName () const

const std::string &	debugName () const

virtual void	dispatch ()

virtual void	dispatch (edm::WaitingTaskWithArenaHolder holder)

SonicMode	mode () const

	SonicClientBase (const edm::ParameterSet &params, const std::string &debugName, const std::string &clientName)

virtual	~SonicClientBase ()=default

Public Member Functions inherited from SonicClientTypes< TritonInputMap, TritonOutputMap >
Input &	input ()

const Output &	output () const

virtual	~SonicClientTypes ()=default

Static Public Member Functions
static void	fillPSetDescription (edm::ParameterSetDescription &iDesc)

Static Public Member Functions inherited from SonicClientBase
static void	fillBasePSetDescription (edm::ParameterSetDescription &desc, bool allowRetry=true)

Protected Member Functions
void	evaluate () override

bool	getResults (std::shared_ptr< nvidia::inferenceserver::client::InferResult > results)

inference::ModelStatistics	getServerSideStatus () const

void	reportServerSideStats (const ServerSideStats &stats) const

ServerSideStats	summarizeServerStats (const inference::ModelStatistics &start_status, const inference::ModelStatistics &end_status) const

Protected Member Functions inherited from SonicClientBase
void	finish (bool success, std::exception_ptr eptr=std::exception_ptr{})

void	setMode (SonicMode mode)

void	start ()

void	start (edm::WaitingTaskWithArenaHolder holder)

Protected Attributes
unsigned	batchSize_

std::unique_ptr< nvidia::inferenceserver::client::InferenceServerGrpcClient >	client_

std::vector< nvidia::inferenceserver::client::InferInput * >	inputsTriton_

unsigned	maxBatchSize_

bool	noBatch_

nvidia::inferenceserver::client::InferOptions	options_

std::vector< const nvidia::inferenceserver::client::InferRequestedOutput * >	outputsTriton_

bool	verbose_

Protected Attributes inherited from SonicClientBase
unsigned	allowedTries_

std::string	clientName_

std::string	debugName_

std::unique_ptr< SonicDispatcher >	dispatcher_

std::string	fullDebugName_

std::optional< edm::WaitingTaskWithArenaHolder >	holder_

SonicMode	mode_

std::chrono::time_point< std::chrono::high_resolution_clock >	t0_

unsigned	tries_

Protected Attributes inherited from SonicClientTypes< TritonInputMap, TritonOutputMap >
Input	input_

Output	output_

Additional Inherited Members
Public Types inherited from SonicClientTypes< TritonInputMap, TritonOutputMap >
typedef TritonInputMap	Input

typedef TritonOutputMap	Output

Detailed Description

Definition at line 18 of file TritonClient.h.

Constructor & Destructor Documentation

◆ TritonClient()

TritonClient::TritonClient	(	const edm::ParameterSet &	params,
		const std::string &	debugName
	)

Definition at line 26 of file TritonClient.cc.

     : SonicClient(params, debugName, "TritonClient"),
       verbose_(params.getUntrackedParameter<bool>("verbose")),
       options_(params.getParameter<std::string>("modelName")) {
   //get appropriate server for this model
   edm::Service<TritonService> ts;
   const auto& [url, isFallbackCPU] =
       ts->serverAddress(options_.model_name_, params.getUntrackedParameter<std::string>("preferredServer"));
   if (verbose_)
     edm::LogInfo(fullDebugName_) << "Using server: " << url;
   //enforce sync mode for fallback CPU server to avoid contention
   //todo: could enforce async mode otherwise (unless mode was specified by user?)
   if (isFallbackCPU)
     setMode(SonicMode::Sync);
  
   //connect to the server
   //TODO: add SSL options
   triton_utils::throwIfError(nic::InferenceServerGrpcClient::Create(&client_, url, false),
                              "TritonClient(): unable to create inference context");
  
   //set options
   options_.model_version_ = params.getParameter<std::string>("modelVersion");
   //convert seconds to microseconds
   options_.client_timeout_ = params.getUntrackedParameter<unsigned>("timeout") * 1e6;
  
   //config needed for batch size
   inference::ModelConfigResponse modelConfigResponse;
   triton_utils::throwIfError(client_->ModelConfig(&modelConfigResponse, options_.model_name_, options_.model_version_),
                              "TritonClient(): unable to get model config");
   inference::ModelConfig modelConfig(modelConfigResponse.config());
  
   //check batch size limitations (after i/o setup)
   //triton uses max batch size = 0 to denote a model that does not support batching
   //but for models that do support batching, a given event may set batch size 0 to indicate no valid input is present
   //so set the local max to 1 and keep track of "no batch" case
   maxBatchSize_ = modelConfig.max_batch_size();
   noBatch_ = maxBatchSize_ == 0;
   maxBatchSize_ = std::max(1u, maxBatchSize_);
  
   //get model info
   inference::ModelMetadataResponse modelMetadata;
   triton_utils::throwIfError(client_->ModelMetadata(&modelMetadata, options_.model_name_, options_.model_version_),
                              "TritonClient(): unable to get model metadata");
  
   //get input and output (which know their sizes)
   const auto& nicInputs = modelMetadata.inputs();
   const auto& nicOutputs = modelMetadata.outputs();
  
   //report all model errors at once
   std::stringstream msg;
   std::string msg_str;
  
   //currently no use case is foreseen for a model with zero inputs or outputs
   if (nicInputs.empty())
     msg << "Model on server appears malformed (zero inputs)\n";
  
   if (nicOutputs.empty())
     msg << "Model on server appears malformed (zero outputs)\n";
  
   //stop if errors
   msg_str = msg.str();
   if (!msg_str.empty())
     throw cms::Exception("ModelErrors") << msg_str;
  
   //setup input map
   std::stringstream io_msg;
   if (verbose_)
     io_msg << "Model inputs: "
            << "\n";
   inputsTriton_.reserve(nicInputs.size());
   for (const auto& nicInput : nicInputs) {
     const auto& iname = nicInput.name();
     auto [curr_itr, success] = input_.emplace(
         std::piecewise_construct, std::forward_as_tuple(iname), std::forward_as_tuple(iname, nicInput, noBatch_));
     auto& curr_input = curr_itr->second;
     inputsTriton_.push_back(curr_input.data());
     if (verbose_) {
       io_msg << "  " << iname << " (" << curr_input.dname() << ", " << curr_input.byteSize()
              << " b) : " << triton_utils::printColl(curr_input.shape()) << "\n";
     }
   }
  
   //allow selecting only some outputs from server
   const auto& v_outputs = params.getUntrackedParameter<std::vector<std::string>>("outputs");
   std::unordered_set s_outputs(v_outputs.begin(), v_outputs.end());
  
   //setup output map
   if (verbose_)
     io_msg << "Model outputs: "
            << "\n";
   outputsTriton_.reserve(nicOutputs.size());
   for (const auto& nicOutput : nicOutputs) {
     const auto& oname = nicOutput.name();
     if (!s_outputs.empty() and s_outputs.find(oname) == s_outputs.end())
       continue;
     auto [curr_itr, success] = output_.emplace(
         std::piecewise_construct, std::forward_as_tuple(oname), std::forward_as_tuple(oname, nicOutput, noBatch_));
     auto& curr_output = curr_itr->second;
     outputsTriton_.push_back(curr_output.data());
     if (verbose_) {
       io_msg << "  " << oname << " (" << curr_output.dname() << ", " << curr_output.byteSize()
              << " b) : " << triton_utils::printColl(curr_output.shape()) << "\n";
     }
     if (!s_outputs.empty())
       s_outputs.erase(oname);
   }
  
   //check if any requested outputs were not available
   if (!s_outputs.empty())
     throw cms::Exception("MissingOutput")
         << "Some requested outputs were not available on the server: " << triton_utils::printColl(s_outputs);
  
   //propagate batch size to inputs and outputs
   setBatchSize(1);
  
   //print model info
   std::stringstream model_msg;
   if (verbose_) {
     model_msg << "Model name: " << options_.model_name_ << "\n"
               << "Model version: " << options_.model_version_ << "\n"
               << "Model max batch size: " << (noBatch_ ? 0 : maxBatchSize_) << "\n";
     edm::LogInfo(fullDebugName_) << model_msg.str() << io_msg.str();
   }
 }

References client_, SonicClientBase::fullDebugName_, SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, inputsTriton_, SiStripPI::max, maxBatchSize_, mps_check::msg, noBatch_, heppy_report::oname, options_, SonicClientTypes< TritonInputMap, TritonOutputMap >::output_, outputsTriton_, CalibrationSummaryClient_cfi::params, triton_utils::printColl(), TritonService::serverAddress(), setBatchSize(), SonicClientBase::setMode(), AlCaHLTBitMon_QueryRunRegistry::string, summarizeEdmComparisonLogfiles::success, Sync, triton_utils::throwIfError(), relmon_authenticated_wget::url, and verbose_.

Member Function Documentation

◆ batchSize()

unsigned TritonClient::batchSize ( ) const

inline

Definition at line 35 of file TritonClient.h.

35 { return batchSize_; }

References batchSize_.

◆ evaluate()

void TritonClient::evaluate ( )

overrideprotectedvirtual

Implements SonicClientBase.

Definition at line 197 of file TritonClient.cc.

                             {
   //in case there is nothing to process
   if (batchSize_ == 0) {
     finish(true);
     return;
   }
  
   // Get the status of the server prior to the request being made.
   const auto& start_status = getServerSideStatus();
  
   if (mode_ == SonicMode::Async) {
     //non-blocking call
     auto t1 = std::chrono::high_resolution_clock::now();
     bool status = triton_utils::warnIfError(
         client_->AsyncInfer(
             [t1, start_status, this](nic::InferResult* results) {
               //get results
               std::shared_ptr<nic::InferResult> results_ptr(results);
               bool status = triton_utils::warnIfError(results_ptr->RequestStatus(), "evaluate(): unable to get result");
               if (!status) {
                 finish(false);
                 return;
               }
               auto t2 = std::chrono::high_resolution_clock::now();
  
               if (!debugName_.empty())
                 edm::LogInfo(fullDebugName_)
                     << "Remote time: " << std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1).count();
  
               const auto& end_status = getServerSideStatus();
  
               if (verbose()) {
                 const auto& stats = summarizeServerStats(start_status, end_status);
                 reportServerSideStats(stats);
               }
  
               //check result
               status = getResults(results_ptr);
  
               //finish
               finish(status);
             },
             options_,
             inputsTriton_,
             outputsTriton_),
         "evaluate(): unable to launch async run");
  
     //if AsyncRun failed, finish() wasn't called
     if (!status)
       finish(false);
   } else {
     //blocking call
     auto t1 = std::chrono::high_resolution_clock::now();
     nic::InferResult* results;
     bool status = triton_utils::warnIfError(client_->Infer(&results, options_, inputsTriton_, outputsTriton_),
                                             "evaluate(): unable to run and/or get result");
     if (!status) {
       finish(false);
       return;
     }
  
     auto t2 = std::chrono::high_resolution_clock::now();
     if (!debugName_.empty())
       edm::LogInfo(fullDebugName_) << "Remote time: "
                                    << std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1).count();
  
     const auto& end_status = getServerSideStatus();
  
     if (verbose()) {
       const auto& stats = summarizeServerStats(start_status, end_status);
       reportServerSideStats(stats);
     }
  
     std::shared_ptr<nic::InferResult> results_ptr(results);
     status = getResults(results_ptr);
  
     finish(status);
   }
 }

References Async, batchSize_, client_, SonicClientBase::debugName_, SonicClientBase::finish(), SonicClientBase::fullDebugName_, getResults(), getServerSideStatus(), if(), inputsTriton_, SonicClientBase::mode_, submitPVValidationJobs::now, options_, outputsTriton_, reportServerSideStats(), bookConverter::results, dqmMemoryStats::stats, mps_update::status, summarizeServerStats(), RandomServiceHelper::t1, RandomServiceHelper::t2, verbose(), and triton_utils::warnIfError().

◆ fillPSetDescription()

void TritonClient::fillPSetDescription ( edm::ParameterSetDescription & iDesc )

static

Definition at line 348 of file TritonClient.cc.

                                                                         {
   edm::ParameterSetDescription descClient;
   fillBasePSetDescription(descClient);
   descClient.add<std::string>("modelName");
   descClient.add<std::string>("modelVersion", "");
   descClient.add<edm::FileInPath>("modelConfigPath");
   //server parameters should not affect the physics results
   descClient.addUntracked<std::string>("preferredServer", "");
   descClient.addUntracked<unsigned>("timeout");
   descClient.addUntracked<bool>("verbose", false);
   descClient.addUntracked<std::vector<std::string>>("outputs", {});
   iDesc.add<edm::ParameterSetDescription>("Client", descClient);
 }

References edm::ParameterSetDescription::add(), edm::ParameterSetDescription::addUntracked(), SonicClientBase::fillBasePSetDescription(), and AlCaHLTBitMon_QueryRunRegistry::string.

◆ getResults()

bool TritonClient::getResults ( std::shared_ptr< nvidia::inferenceserver::client::InferResult > results )

protected

Definition at line 178 of file TritonClient.cc.

                                                                    {
   for (auto& [oname, output] : output_) {
     //set shape here before output becomes const
     if (output.variableDims()) {
       std::vector<int64_t> tmp_shape;
       bool status = triton_utils::warnIfError(results->Shape(oname, &tmp_shape),
                                               "getResults(): unable to get output shape for " + oname);
       if (!status)
         return status;
       output.setShape(tmp_shape, false);
     }
     //extend lifetime
     output.setResult(results);
   }
  
   return true;
 }

References heppy_report::oname, SonicClientTypes< TritonInputMap, TritonOutputMap >::output(), SonicClientTypes< TritonInputMap, TritonOutputMap >::output_, bookConverter::results, mps_update::status, and triton_utils::warnIfError().

Referenced by evaluate().

◆ getServerSideStatus()

inference::ModelStatistics TritonClient::getServerSideStatus ( ) const

protected

Definition at line 335 of file TritonClient.cc.

                                                                  {
   if (verbose_) {
     inference::ModelStatisticsResponse resp;
     bool success = triton_utils::warnIfError(
         client_->ModelInferenceStatistics(&resp, options_.model_name_, options_.model_version_),
         "getServerSideStatus(): unable to get model statistics");
     if (success)
       return *(resp.model_stats().begin());
   }
   return inference::ModelStatistics{};
 }

References client_, options_, summarizeEdmComparisonLogfiles::success, verbose_, and triton_utils::warnIfError().

Referenced by evaluate().

◆ reportServerSideStats()

void TritonClient::reportServerSideStats ( const ServerSideStats & stats ) const

protected

Definition at line 277 of file TritonClient.cc.

                                                                                        {
   std::stringstream msg;
  
   // https://github.com/triton-inference-server/server/blob/v2.3.0/src/clients/c++/perf_client/inference_profiler.cc
   const uint64_t count = stats.success_count_;
   msg << "  Inference count: " << stats.inference_count_ << "\n";
   msg << "  Execution count: " << stats.execution_count_ << "\n";
   msg << "  Successful request count: " << count << "\n";
  
   if (count > 0) {
     auto get_avg_us = [count](uint64_t tval) {
       constexpr uint64_t us_to_ns = 1000;
       return tval / us_to_ns / count;
     };
  
     const uint64_t cumm_avg_us = get_avg_us(stats.cumm_time_ns_);
     const uint64_t queue_avg_us = get_avg_us(stats.queue_time_ns_);
     const uint64_t compute_input_avg_us = get_avg_us(stats.compute_input_time_ns_);
     const uint64_t compute_infer_avg_us = get_avg_us(stats.compute_infer_time_ns_);
     const uint64_t compute_output_avg_us = get_avg_us(stats.compute_output_time_ns_);
     const uint64_t compute_avg_us = compute_input_avg_us + compute_infer_avg_us + compute_output_avg_us;
     const uint64_t overhead =
         (cumm_avg_us > queue_avg_us + compute_avg_us) ? (cumm_avg_us - queue_avg_us - compute_avg_us) : 0;
  
     msg << "  Avg request latency: " << cumm_avg_us << " usec"
         << "\n"
         << "  (overhead " << overhead << " usec + "
         << "queue " << queue_avg_us << " usec + "
         << "compute input " << compute_input_avg_us << " usec + "
         << "compute infer " << compute_infer_avg_us << " usec + "
         << "compute output " << compute_output_avg_us << " usec)" << std::endl;
   }
  
   if (!debugName_.empty())
     edm::LogInfo(fullDebugName_) << msg.str();
 }

References submitPVResolutionJobs::count, SonicClientBase::debugName_, SonicClientBase::fullDebugName_, mps_check::msg, and dqmMemoryStats::stats.

Referenced by evaluate().

◆ reset()

void TritonClient::reset ( void )

overridevirtual

Reimplemented from SonicClientBase.

Definition at line 169 of file TritonClient.cc.

                          {
   for (auto& element : input_) {
     element.second.reset();
   }
   for (auto& element : output_) {
     element.second.reset();
   }
 }

References SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, and SonicClientTypes< TritonInputMap, TritonOutputMap >::output_.

◆ setBatchSize()

bool TritonClient::setBatchSize ( unsigned bsize )

Definition at line 151 of file TritonClient.cc.

                                               {
   if (bsize > maxBatchSize_) {
     edm::LogWarning(fullDebugName_) << "Requested batch size " << bsize << " exceeds server-specified max batch size "
                                     << maxBatchSize_ << ". Batch size will remain as" << batchSize_;
     return false;
   } else {
     batchSize_ = bsize;
     //set for input and output
     for (auto& element : input_) {
       element.second.setBatchSize(bsize);
     }
     for (auto& element : output_) {
       element.second.setBatchSize(bsize);
     }
     return true;
   }
 }

References batchSize_, SonicClientBase::fullDebugName_, SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, maxBatchSize_, and SonicClientTypes< TritonInputMap, TritonOutputMap >::output_.

Referenced by TritonClient().

◆ summarizeServerStats()

TritonClient::ServerSideStats TritonClient::summarizeServerStats	(	const inference::ModelStatistics &	start_status,
		const inference::ModelStatistics &	end_status
	)		const

protected

Definition at line 314 of file TritonClient.cc.

                                                                                                                    {
   TritonClient::ServerSideStats server_stats;
  
   server_stats.inference_count_ = end_status.inference_count() - start_status.inference_count();
   server_stats.execution_count_ = end_status.execution_count() - start_status.execution_count();
   server_stats.success_count_ =
       end_status.inference_stats().success().count() - start_status.inference_stats().success().count();
   server_stats.cumm_time_ns_ =
       end_status.inference_stats().success().ns() - start_status.inference_stats().success().ns();
   server_stats.queue_time_ns_ = end_status.inference_stats().queue().ns() - start_status.inference_stats().queue().ns();
   server_stats.compute_input_time_ns_ =
       end_status.inference_stats().compute_input().ns() - start_status.inference_stats().compute_input().ns();
   server_stats.compute_infer_time_ns_ =
       end_status.inference_stats().compute_infer().ns() - start_status.inference_stats().compute_infer().ns();
   server_stats.compute_output_time_ns_ =
       end_status.inference_stats().compute_output().ns() - start_status.inference_stats().compute_output().ns();
  
   return server_stats;
 }

References TritonClient::ServerSideStats::compute_infer_time_ns_, TritonClient::ServerSideStats::compute_input_time_ns_, TritonClient::ServerSideStats::compute_output_time_ns_, TritonClient::ServerSideStats::cumm_time_ns_, TritonClient::ServerSideStats::execution_count_, TritonClient::ServerSideStats::inference_count_, TritonClient::ServerSideStats::queue_time_ns_, and TritonClient::ServerSideStats::success_count_.

Referenced by evaluate().

◆ verbose()

bool TritonClient::verbose ( ) const

inline

Definition at line 36 of file TritonClient.h.

36 { return verbose_; }

References verbose_.

Referenced by evaluate().

Member Data Documentation

◆ batchSize_

unsigned TritonClient::batchSize_

protected

Definition at line 57 of file TritonClient.h.

Referenced by batchSize(), evaluate(), and setBatchSize().

◆ client_

std::unique_ptr<nvidia::inferenceserver::client::InferenceServerGrpcClient> TritonClient::client_

protected

Definition at line 65 of file TritonClient.h.

Referenced by evaluate(), getServerSideStatus(), and TritonClient().

◆ inputsTriton_

std::vector<nvidia::inferenceserver::client::InferInput*> TritonClient::inputsTriton_

protected

Definition at line 62 of file TritonClient.h.

Referenced by evaluate(), and TritonClient().

◆ maxBatchSize_

unsigned TritonClient::maxBatchSize_

protected

Definition at line 56 of file TritonClient.h.

Referenced by setBatchSize(), and TritonClient().

◆ noBatch_

bool TritonClient::noBatch_

protected

Definition at line 58 of file TritonClient.h.

Referenced by TritonClient().

◆ options_

nvidia::inferenceserver::client::InferOptions TritonClient::options_

protected

Definition at line 67 of file TritonClient.h.

Referenced by batchmanager.BatchManager::CheckBatchScript(), evaluate(), getServerSideStatus(), batchmanager.BatchManager::ManageOutputDir(), valtools.webpage::parseArgs(), batchmanager.BatchManager::ParseOptions(), batchmanager.BatchManager::SubmitJob(), batchmanager.BatchManager::SubmitJobs(), and TritonClient().

◆ outputsTriton_

std::vector<const nvidia::inferenceserver::client::InferRequestedOutput*> TritonClient::outputsTriton_

protected

Definition at line 63 of file TritonClient.h.

Referenced by evaluate(), and TritonClient().

◆ verbose_

bool TritonClient::verbose_

protected

Definition at line 59 of file TritonClient.h.

Referenced by getServerSideStatus(), TritonClient(), and verbose().

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ TritonClient()

Member Function Documentation

◆ batchSize()

◆ evaluate()

◆ fillPSetDescription()

◆ getResults()

◆ getServerSideStatus()

◆ reportServerSideStats()

◆ reset()

◆ setBatchSize()

◆ summarizeServerStats()

◆ verbose()

Member Data Documentation

◆ batchSize_

◆ client_

◆ inputsTriton_

◆ maxBatchSize_

◆ noBatch_

◆ options_

◆ outputsTriton_

◆ verbose_