TritonClient Class Reference

#include <TritonClient.h>

Inheritance diagram for TritonClient:

Classes
struct	ServerSideStats

Public Member Functions
TritonBatchMode	batchMode () const
unsigned	batchSize () const
void	reset () override
void	resetBatchMode ()
TritonServerType	serverType () const
void	setBatchMode (TritonBatchMode batchMode)
bool	setBatchSize (unsigned bsize)
void	setUseSharedMemory (bool useShm)
	TritonClient (const edm::ParameterSet &params, const std::string &debugName)
bool	useSharedMemory () const
bool	verbose () const
	~TritonClient () override
Public Member Functions inherited from SonicClient< TritonInputMap, TritonOutputMap >
	SonicClient (const edm::ParameterSet &params, const std::string &debugName, const std::string &clientName)
Public Member Functions inherited from SonicClientBase
const std::string &	clientName () const
const std::string &	debugName () const
virtual void	dispatch (edm::WaitingTaskWithArenaHolder holder)
virtual void	dispatch ()
SonicMode	mode () const
	SonicClientBase (const edm::ParameterSet &params, const std::string &debugName, const std::string &clientName)
virtual	~SonicClientBase ()=default
Public Member Functions inherited from SonicClientTypes< TritonInputMap, TritonOutputMap >
Input &	input ()
const Output &	output () const
virtual	~SonicClientTypes ()=default

Static Public Member Functions
static void	fillPSetDescription (edm::ParameterSetDescription &iDesc)
Static Public Member Functions inherited from SonicClientBase
static void	fillBasePSetDescription (edm::ParameterSetDescription &desc, bool allowRetry=true)

Protected Member Functions
void	evaluate () override
void	getResults (const std::vector< std::shared_ptr< triton::client::InferResult >> &results)
inference::ModelStatistics	getServerSideStatus () const
template<typename F >
bool	handle_exception (F &&call)
unsigned	nEntries () const
bool	noOuterDim () const
unsigned	outerDim () const
void	reportServerSideStats (const ServerSideStats &stats) const
ServerSideStats	summarizeServerStats (const inference::ModelStatistics &start_status, const inference::ModelStatistics &end_status) const
Protected Member Functions inherited from SonicClientBase
void	finish (bool success, std::exception_ptr eptr=std::exception_ptr{})
void	setMode (SonicMode mode)
void	start (edm::WaitingTaskWithArenaHolder holder)
void	start ()

Protected Attributes
TritonBatchMode	batchMode_
std::unique_ptr< triton::client::InferenceServerGrpcClient >	client_
grpc_compression_algorithm	compressionAlgo_
triton::client::Headers	headers_
bool	manualBatchMode_
unsigned	maxOuterDim_
unsigned	nEntries_
bool	noOuterDim_
std::vector< triton::client::InferOptions >	options_
unsigned	outerDim_
TritonServerType	serverType_
bool	useSharedMemory_
bool	verbose_
Protected Attributes inherited from SonicClientBase
unsigned	allowedTries_
std::string	clientName_
std::string	debugName_
std::unique_ptr< SonicDispatcher >	dispatcher_
std::string	fullDebugName_
std::optional< edm::WaitingTaskWithArenaHolder >	holder_
SonicMode	mode_
unsigned	tries_
bool	verbose_
Protected Attributes inherited from SonicClientTypes< TritonInputMap, TritonOutputMap >
Input	input_
Output	output_

Private Member Functions
void	addEntry (unsigned entry)
auto	client ()
void	resizeEntries (unsigned entry)

Private Attributes
friend	TritonInputData
friend	TritonOutputData

Additional Inherited Members
Public Types inherited from SonicClientTypes< TritonInputMap, TritonOutputMap >
typedef TritonInputMap	Input
typedef TritonOutputMap	Output

Detailed Description

Definition at line 21 of file TritonClient.h.

Constructor & Destructor Documentation

TritonClient()

TritonClient::TritonClient	(	const edm::ParameterSet &	params,
		const std::string &	debugName
	)

Definition at line 55 of file TritonClient.cc.

References a, client_, SonicClientBase::fullDebugName_, edm::FileInPath::fullPath(), SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, submitPVResolutionJobs::key, LocalCPU, SiStripPI::max, maxOuterDim_, mps_check::msg, noOuterDim_, heppy_report::oname, options_, SonicClientTypes< TritonInputMap, TritonOutputMap >::output_, submitPVValidationJobs::params, TritonService::pid(), triton_utils::printColl(), Rectangular, contentValuesFiles::server, TritonService::serverInfo(), serverType_, setBatchSize(), SonicClientBase::setMode(), AlCaHLTBitMon_QueryRunRegistry::string, summarizeEdmComparisonLogfiles::success, Sync, TRITON_THROW_IF_ERROR, heppy_batch::val, and verbose_.

    : SonicClient(params, debugName, "TritonClient"),
      batchMode_(TritonBatchMode::Rectangular),
      manualBatchMode_(false),
      verbose_(params.getUntrackedParameter<bool>("verbose")),
      useSharedMemory_(params.getUntrackedParameter<bool>("useSharedMemory")),
      compressionAlgo_(getCompressionAlgo(params.getUntrackedParameter<std::string>("compression"))) {
  options_.emplace_back(params.getParameter<std::string>("modelName"));
  //get appropriate server for this model
  edm::Service<TritonService> ts;
  const auto& server =
      ts->serverInfo(options_[0].model_name_, params.getUntrackedParameter<std::string>("preferredServer"));
  serverType_ = server.type;
  if (verbose_)
    edm::LogInfo(fullDebugName_) << "Using server: " << server.url;
  //enforce sync mode for fallback CPU server to avoid contention
  //todo: could enforce async mode otherwise (unless mode was specified by user?)
  if (serverType_ == TritonServerType::LocalCPU)
    setMode(SonicMode::Sync);

  //connect to the server
  TRITON_THROW_IF_ERROR(
      tc::InferenceServerGrpcClient::Create(&client_, server.url, false, server.useSsl, server.sslOptions),
      "TritonClient(): unable to create inference context");

  //set options
  options_[0].model_version_ = params.getParameter<std::string>("modelVersion");
  //convert seconds to microseconds
  options_[0].client_timeout_ = params.getUntrackedParameter<unsigned>("timeout") * 1e6;

  //get fixed parameters from local config
  inference::ModelConfig localModelConfig;
  {
    const std::string& localModelConfigPath(params.getParameter<edm::FileInPath>("modelConfigPath").fullPath());
    int fileDescriptor = open(localModelConfigPath.c_str(), O_RDONLY);
    if (fileDescriptor < 0)
      throw TritonException("LocalFailure")
          << "TritonClient(): unable to open local model config: " << localModelConfigPath;
    google::protobuf::io::FileInputStream localModelConfigInput(fileDescriptor);
    localModelConfigInput.SetCloseOnDelete(true);
    if (!google::protobuf::TextFormat::Parse(&localModelConfigInput, &localModelConfig))
      throw TritonException("LocalFailure")
          << "TritonClient(): unable to parse local model config: " << localModelConfigPath;
  }

  //check batch size limitations (after i/o setup)
  //triton uses max batch size = 0 to denote a model that does not support native batching (using the outer dimension)
  //but for models that do support batching (native or otherwise), a given event may set batch size 0 to indicate no valid input is present
  //so set the local max to 1 and keep track of "no outer dim" case
  maxOuterDim_ = localModelConfig.max_batch_size();
  noOuterDim_ = maxOuterDim_ == 0;
  maxOuterDim_ = std::max(1u, maxOuterDim_);
  //propagate batch size
  setBatchSize(1);

  //compare model checksums to remote config to enforce versioning
  inference::ModelConfigResponse modelConfigResponse;
  TRITON_THROW_IF_ERROR(client_->ModelConfig(&modelConfigResponse, options_[0].model_name_, options_[0].model_version_),
                        "TritonClient(): unable to get model config");
  inference::ModelConfig remoteModelConfig(modelConfigResponse.config());

  std::map<std::string, std::array<std::string, 2>> checksums;
  size_t fileCounter = 0;
  for (const auto& modelConfig : {localModelConfig, remoteModelConfig}) {
    const auto& agents = modelConfig.model_repository_agents().agents();
    auto agent = std::find_if(agents.begin(), agents.end(), [](auto const& a) { return a.name() == "checksum"; });
    if (agent != agents.end()) {
      const auto& params = agent->parameters();
      for (const auto& [key, val] : params) {
        // only check the requested version
        if (key.compare(0, options_[0].model_version_.size() + 1, options_[0].model_version_ + "/") == 0)
          checksums[key][fileCounter] = val;
      }
    }
    ++fileCounter;
  }
  std::vector<std::string> incorrect;
  for (const auto& [key, val] : checksums) {
    if (checksums[key][0] != checksums[key][1])
      incorrect.push_back(key);
  }
  if (!incorrect.empty())
    throw TritonException("ModelVersioning") << "The following files have incorrect checksums on the remote server: "
                                             << triton_utils::printColl(incorrect, ", ");

  //get model info
  inference::ModelMetadataResponse modelMetadata;
  TRITON_THROW_IF_ERROR(client_->ModelMetadata(&modelMetadata, options_[0].model_name_, options_[0].model_version_),
                        "TritonClient(): unable to get model metadata");

  //get input and output (which know their sizes)
  const auto& nicInputs = modelMetadata.inputs();
  const auto& nicOutputs = modelMetadata.outputs();

  //report all model errors at once
  std::stringstream msg;
  std::string msg_str;

  //currently no use case is foreseen for a model with zero inputs or outputs
  if (nicInputs.empty())
    msg << "Model on server appears malformed (zero inputs)\n";

  if (nicOutputs.empty())
    msg << "Model on server appears malformed (zero outputs)\n";

  //stop if errors
  msg_str = msg.str();
  if (!msg_str.empty())
    throw cms::Exception("ModelErrors") << msg_str;

  //setup input map
  std::stringstream io_msg;
  if (verbose_)
    io_msg << "Model inputs: "
           << "\n";
  for (const auto& nicInput : nicInputs) {
    const auto& iname = nicInput.name();
    auto [curr_itr, success] = input_.emplace(std::piecewise_construct,
                                              std::forward_as_tuple(iname),
                                              std::forward_as_tuple(iname, nicInput, this, ts->pid()));
    auto& curr_input = curr_itr->second;
    if (verbose_) {
      io_msg << "  " << iname << " (" << curr_input.dname() << ", " << curr_input.byteSize()
             << " b) : " << triton_utils::printColl(curr_input.shape()) << "\n";
    }
  }

  //allow selecting only some outputs from server
  const auto& v_outputs = params.getUntrackedParameter<std::vector<std::string>>("outputs");
  std::unordered_set s_outputs(v_outputs.begin(), v_outputs.end());

  //setup output map
  if (verbose_)
    io_msg << "Model outputs: "
           << "\n";
  for (const auto& nicOutput : nicOutputs) {
    const auto& oname = nicOutput.name();
    if (!s_outputs.empty() and s_outputs.find(oname) == s_outputs.end())
      continue;
    auto [curr_itr, success] = output_.emplace(std::piecewise_construct,
                                               std::forward_as_tuple(oname),
                                               std::forward_as_tuple(oname, nicOutput, this, ts->pid()));
    auto& curr_output = curr_itr->second;
    if (verbose_) {
      io_msg << "  " << oname << " (" << curr_output.dname() << ", " << curr_output.byteSize()
             << " b) : " << triton_utils::printColl(curr_output.shape()) << "\n";
    }
    if (!s_outputs.empty())
      s_outputs.erase(oname);
  }

  //check if any requested outputs were not available
  if (!s_outputs.empty())
    throw cms::Exception("MissingOutput")
        << "Some requested outputs were not available on the server: " << triton_utils::printColl(s_outputs);

  //print model info
  std::stringstream model_msg;
  if (verbose_) {
    model_msg << "Model name: " << options_[0].model_name_ << "\n"
              << "Model version: " << options_[0].model_version_ << "\n"
              << "Model max outer dim: " << (noOuterDim_ ? 0 : maxOuterDim_) << "\n";
    edm::LogInfo(fullDebugName_) << model_msg.str() << io_msg.str();
  }
}

~TritonClient()

TritonClient::~TritonClient ( )

override

Definition at line 221 of file TritonClient.cc.

References SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, and SonicClientTypes< TritonInputMap, TritonOutputMap >::output_.

                            {
  //by default: members of this class destroyed before members of base class
  //in shared memory case, TritonMemResource (member of TritonData) unregisters from client_ in its destructor
  //but input/output objects are member of base class, so destroyed after client_ (member of this class)
  //therefore, clear the maps here
  input_.clear();
  output_.clear();
}

Member Function Documentation

addEntry()

void TritonClient::addEntry ( unsigned  entry )

private

Definition at line 281 of file TritonClient.cc.

References batchMode_, mps_splice::entry, SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, outerDim_, SonicClientTypes< TritonInputMap, TritonOutputMap >::output_, and Ragged.

Referenced by resizeEntries().

                                          {
  for (auto& element : input_) {
    element.second.addEntryImpl(entry);
  }
  for (auto& element : output_) {
    element.second.addEntryImpl(entry);
  }
  if (entry > 0) {
    batchMode_ = TritonBatchMode::Ragged;
    outerDim_ = 1;
  }
}

batchMode()

TritonBatchMode TritonClient::batchMode ( ) const

inline

Definition at line 42 of file TritonClient.h.

References batchMode_.

Referenced by setBatchMode().

{ return batchMode_; }

batchSize()

unsigned TritonClient::batchSize

(

)

const

Definition at line 246 of file TritonClient.cc.

References batchMode_, nEntries(), outerDim_, and Rectangular.

Referenced by TritonData< IO >::allocate(), evaluate(), TritonData< IO >::fromServer(), setBatchMode(), and TritonData< IO >::toServer().

{ return batchMode_ == TritonBatchMode::Rectangular ? outerDim_ : nEntries(); }

client()

auto TritonClient::client ( )

inlineprivate

Definition at line 93 of file TritonClient.h.

References client_.

Referenced by TritonData< IO >::client().

{ return client_.get(); }

evaluate()

void TritonClient::evaluate ( )

overrideprotectedvirtual

Implements SonicClientBase.

Definition at line 348 of file TritonClient.cc.

References Async, batchSize(), client_, compressionAlgo_, SonicClientBase::finish(), getResults(), getServerSideStatus(), handle_exception(), headers_, mps_fire::i, ALPAKA_ACCELERATOR_NAMESPACE::caPixelDoublets::if(), SonicClientTypes< TritonInputMap, TritonOutputMap >::input(), SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, SonicClientBase::mode_, nEntries(), heppy_report::oname, options_, SonicClientTypes< TritonInputMap, TritonOutputMap >::output_, reportServerSideStats(), mysort::results, dqmMemoryStats::stats, summarizeEdmComparisonLogfiles::success, summarizeServerStats(), TRITON_THROW_IF_ERROR, and verbose().

                            {
  //in case there is nothing to process
  if (batchSize() == 0) {
    //call getResults on an empty vector
    std::vector<std::shared_ptr<tc::InferResult>> empty_results;
    getResults(empty_results);
    finish(true);
    return;
  }

  //set up input pointers for triton (generalized for multi-request ragged batching case)
  //one vector<InferInput*> per request
  unsigned nEntriesVal = nEntries();
  std::vector<std::vector<triton::client::InferInput*>> inputsTriton(nEntriesVal);
  for (auto& inputTriton : inputsTriton) {
    inputTriton.reserve(input_.size());
  }
  for (auto& [iname, input] : input_) {
    for (unsigned i = 0; i < nEntriesVal; ++i) {
      inputsTriton[i].push_back(input.data(i));
    }
  }

  //set up output pointers similarly
  std::vector<std::vector<const triton::client::InferRequestedOutput*>> outputsTriton(nEntriesVal);
  for (auto& outputTriton : outputsTriton) {
    outputTriton.reserve(output_.size());
  }
  for (auto& [oname, output] : output_) {
    for (unsigned i = 0; i < nEntriesVal; ++i) {
      outputsTriton[i].push_back(output.data(i));
    }
  }

  //set up shared memory for output
  auto success = handle_exception([&]() {
    for (auto& element : output_) {
      element.second.prepare();
    }
  });
  if (!success)
    return;

  // Get the status of the server prior to the request being made.
  inference::ModelStatistics start_status;
  success = handle_exception([&]() {
    if (verbose())
      start_status = getServerSideStatus();
  });
  if (!success)
    return;

  if (mode_ == SonicMode::Async) {
    //non-blocking call
    success = handle_exception([&]() {
      TRITON_THROW_IF_ERROR(
          client_->AsyncInferMulti(
              [start_status, this](std::vector<tc::InferResult*> resultsTmp) {
                //immediately convert to shared_ptr
                const auto& results = convertToShared(resultsTmp);
                //check results
                for (auto ptr : results) {
                  auto success = handle_exception(
                      [&]() { TRITON_THROW_IF_ERROR(ptr->RequestStatus(), "evaluate(): unable to get result(s)"); });
                  if (!success)
                    return;
                }

                if (verbose()) {
                  inference::ModelStatistics end_status;
                  auto success = handle_exception([&]() { end_status = getServerSideStatus(); });
                  if (!success)
                    return;

                  const auto& stats = summarizeServerStats(start_status, end_status);
                  reportServerSideStats(stats);
                }

                //check result
                auto success = handle_exception([&]() { getResults(results); });
                if (!success)
                  return;

                //finish
                finish(true);
              },
              options_,
              inputsTriton,
              outputsTriton,
              headers_,
              compressionAlgo_),
          "evaluate(): unable to launch async run");
    });
    if (!success)
      return;
  } else {
    //blocking call
    std::vector<tc::InferResult*> resultsTmp;
    success = handle_exception([&]() {
      TRITON_THROW_IF_ERROR(
          client_->InferMulti(&resultsTmp, options_, inputsTriton, outputsTriton, headers_, compressionAlgo_),
          "evaluate(): unable to run and/or get result");
    });
    //immediately convert to shared_ptr
    const auto& results = convertToShared(resultsTmp);
    if (!success)
      return;

    if (verbose()) {
      inference::ModelStatistics end_status;
      success = handle_exception([&]() { end_status = getServerSideStatus(); });
      if (!success)
        return;

      const auto& stats = summarizeServerStats(start_status, end_status);
      reportServerSideStats(stats);
    }

    success = handle_exception([&]() { getResults(results); });
    if (!success)
      return;

    finish(true);
  }
}

fillPSetDescription()

void TritonClient::fillPSetDescription ( edm::ParameterSetDescription &  iDesc )

static

Definition at line 543 of file TritonClient.cc.

References edm::ParameterSetDescription::add(), edm::ParameterSetDescription::addUntracked(), SonicClientBase::fillBasePSetDescription(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by DeepTauIdSonicProducer::fillDescriptions(), DeepMETSonicProducer::fillDescriptions(), ParticleNetSonicJetTagsProducer::fillDescriptions(), SCEnergyCorrectorDRNProducer::fillDescriptions(), and DRNCorrectionProducerT< T >::fillDescriptions().

                                                                        {
  edm::ParameterSetDescription descClient;
  fillBasePSetDescription(descClient);
  descClient.add<std::string>("modelName");
  descClient.add<std::string>("modelVersion", "");
  descClient.add<edm::FileInPath>("modelConfigPath");
  //server parameters should not affect the physics results
  descClient.addUntracked<std::string>("preferredServer", "");
  descClient.addUntracked<unsigned>("timeout");
  descClient.addUntracked<bool>("useSharedMemory", true);
  descClient.addUntracked<std::string>("compression", "");
  descClient.addUntracked<std::vector<std::string>>("outputs", {});
  iDesc.add<edm::ParameterSetDescription>("Client", descClient);
}

getResults()

void TritonClient::getResults ( const std::vector< std::shared_ptr< triton::client::InferResult >> &  results )

protected

Definition at line 325 of file TritonClient.cc.

References mps_fire::i, noOuterDim_, heppy_report::oname, SonicClientTypes< TritonInputMap, TritonOutputMap >::output_, mps_fire::result, mysort::results, and TRITON_THROW_IF_ERROR.

Referenced by evaluate().

                                                                                    {
  for (unsigned i = 0; i < results.size(); ++i) {
    const auto& result = results[i];
    for (auto& [oname, output] : output_) {
      //set shape here before output becomes const
      if (output.variableDims()) {
        std::vector<int64_t> tmp_shape;
        TRITON_THROW_IF_ERROR(result->Shape(oname, &tmp_shape),
                              "getResults(): unable to get output shape for " + oname);
        if (!noOuterDim_)
          tmp_shape.erase(tmp_shape.begin());
        output.setShape(tmp_shape, i);
      }
      //extend lifetime
      output.setResult(result, i);
      //compute size after getting all result entries
      if (i == results.size() - 1)
        output.computeSizes();
    }
  }
}

getServerSideStatus()

inference::ModelStatistics TritonClient::getServerSideStatus ( ) const

protected

Definition at line 532 of file TritonClient.cc.

References client_, options_, TRITON_THROW_IF_ERROR, and verbose_.

Referenced by evaluate().

                                                                 {
  if (verbose_) {
    inference::ModelStatisticsResponse resp;
    TRITON_THROW_IF_ERROR(client_->ModelInferenceStatistics(&resp, options_[0].model_name_, options_[0].model_version_),
                          "getServerSideStatus(): unable to get model statistics");
    return *(resp.model_stats().begin());
  }
  return inference::ModelStatistics{};
}

handle_exception()

template<typename F >

bool TritonClient::handle_exception ( F &&  call )

protected

Definition at line 306 of file TritonClient.cc.

References CMS_SA_ALLOW, MillePedeFileConverter_cfg::e, and SonicClientBase::finish().

Referenced by evaluate().

                                            {
  //caught exceptions will be propagated to edm::WaitingTaskWithArenaHolder
  CMS_SA_ALLOW try {
    call();
    return true;
  }
  //TritonExceptions are intended/expected to be recoverable, i.e. retries should be allowed
  catch (TritonException& e) {
    e.convertToWarning();
    finish(false);
    return false;
  }
  //other exceptions are not: execution should stop if they are encountered
  catch (...) {
    finish(false, std::current_exception());
    return false;
  }
}

nEntries()

unsigned TritonClient::nEntries ( ) const

protected

Definition at line 244 of file TritonClient.cc.

References SonicClientTypes< TritonInputMap, TritonOutputMap >::input_.

Referenced by batchSize(), evaluate(), and resizeEntries().

{ return !input_.empty() ? input_.begin()->second.entries_.size() : 0; }

noOuterDim()

bool TritonClient::noOuterDim ( ) const

inlineprotected

Definition at line 57 of file TritonClient.h.

References noOuterDim_.

Referenced by TritonData< IO >::fullLoc(), and TritonData< IO >::toServer().

{ return noOuterDim_; }

outerDim()

unsigned TritonClient::outerDim ( ) const

inlineprotected

Definition at line 58 of file TritonClient.h.

References outerDim_.

Referenced by TritonData< IO >::computeSizes(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

{ return outerDim_; }

reportServerSideStats()

void TritonClient::reportServerSideStats ( const ServerSideStats &  stats ) const

protected

Definition at line 474 of file TritonClient.cc.

References ALPAKA_ACCELERATOR_NAMESPACE::brokenline::constexpr(), submitPVResolutionJobs::count, SonicClientBase::debugName_, SonicClientBase::fullDebugName_, mps_check::msg, and dqmMemoryStats::stats.

Referenced by evaluate().

                                                                                       {
  std::stringstream msg;

  // https://github.com/triton-inference-server/server/blob/v2.3.0/src/clients/c++/perf_client/inference_profiler.cc
  const uint64_t count = stats.success_count_;
  msg << "  Inference count: " << stats.inference_count_ << "\n";
  msg << "  Execution count: " << stats.execution_count_ << "\n";
  msg << "  Successful request count: " << count << "\n";

  if (count > 0) {
    auto get_avg_us = [count](uint64_t tval) {
      constexpr uint64_t us_to_ns = 1000;
      return tval / us_to_ns / count;
    };

    const uint64_t cumm_avg_us = get_avg_us(stats.cumm_time_ns_);
    const uint64_t queue_avg_us = get_avg_us(stats.queue_time_ns_);
    const uint64_t compute_input_avg_us = get_avg_us(stats.compute_input_time_ns_);
    const uint64_t compute_infer_avg_us = get_avg_us(stats.compute_infer_time_ns_);
    const uint64_t compute_output_avg_us = get_avg_us(stats.compute_output_time_ns_);
    const uint64_t compute_avg_us = compute_input_avg_us + compute_infer_avg_us + compute_output_avg_us;
    const uint64_t overhead =
        (cumm_avg_us > queue_avg_us + compute_avg_us) ? (cumm_avg_us - queue_avg_us - compute_avg_us) : 0;

    msg << "  Avg request latency: " << cumm_avg_us << " usec"
        << "\n"
        << "  (overhead " << overhead << " usec + "
        << "queue " << queue_avg_us << " usec + "
        << "compute input " << compute_input_avg_us << " usec + "
        << "compute infer " << compute_infer_avg_us << " usec + "
        << "compute output " << compute_output_avg_us << " usec)" << std::endl;
  }

  if (!debugName_.empty())
    edm::LogInfo(fullDebugName_) << msg.str();
}

reset()

void TritonClient::reset ( void  )

overridevirtual

Reimplemented from SonicClientBase.

Definition at line 294 of file TritonClient.cc.

References batchMode_, SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, manualBatchMode_, SonicClientTypes< TritonInputMap, TritonOutputMap >::output_, and Rectangular.

                         {
  if (!manualBatchMode_)
    batchMode_ = TritonBatchMode::Rectangular;
  for (auto& element : input_) {
    element.second.reset();
  }
  for (auto& element : output_) {
    element.second.reset();
  }
}

resetBatchMode()

void TritonClient::resetBatchMode

(

)

Definition at line 239 of file TritonClient.cc.

References batchMode_, manualBatchMode_, and Rectangular.

                                  {
  batchMode_ = TritonBatchMode::Rectangular;
  manualBatchMode_ = false;
}

resizeEntries()

void TritonClient::resizeEntries ( unsigned  entry )

private

Definition at line 267 of file TritonClient.cc.

References addEntry(), mps_splice::entry, SonicClientTypes< TritonInputMap, TritonOutputMap >::input_, nEntries(), and SonicClientTypes< TritonInputMap, TritonOutputMap >::output_.

Referenced by setBatchSize().

                                               {
  if (entry > nEntries())
    //addEntry(entry) extends the vector to size entry+1
    addEntry(entry - 1);
  else if (entry < nEntries()) {
    for (auto& element : input_) {
      element.second.entries_.resize(entry);
    }
    for (auto& element : output_) {
      element.second.entries_.resize(entry);
    }
  }
}

serverType()

TritonServerType TritonClient::serverType ( ) const

inline

Definition at line 50 of file TritonClient.h.

References serverType_.

Referenced by TritonData< IO >::updateMem().

{ return serverType_; }

setBatchMode()

void TritonClient::setBatchMode

(

TritonBatchMode

batchMode

)

Definition at line 230 of file TritonClient.cc.

References batchMode(), batchMode_, batchSize(), manualBatchMode_, and setBatchSize().

                                                         {
  unsigned oldBatchSize = batchSize();
  batchMode_ = batchMode;
  manualBatchMode_ = true;
  //this allows calling setBatchSize() and setBatchMode() in either order consistently to change back and forth
  //includes handling of change from ragged to rectangular if multiple entries already created
  setBatchSize(oldBatchSize);
}

setBatchSize()

bool TritonClient::setBatchSize

(

unsigned

bsize

)

Definition at line 248 of file TritonClient.cc.

References batchMode_, SonicClientBase::fullDebugName_, maxOuterDim_, SiStripPI::min, outerDim_, Rectangular, and resizeEntries().

Referenced by setBatchMode(), and TritonClient().

                                              {
  if (batchMode_ == TritonBatchMode::Rectangular) {
    if (bsize > maxOuterDim_) {
      edm::LogWarning(fullDebugName_) << "Requested batch size " << bsize << " exceeds server-specified max batch size "
                                      << maxOuterDim_ << ". Batch size will remain as " << outerDim_;
      return false;
    } else {
      outerDim_ = bsize;
      //take min to allow resizing to 0
      resizeEntries(std::min(outerDim_, 1u));
      return true;
    }
  } else {
    resizeEntries(bsize);
    outerDim_ = 1;
    return true;
  }
}

setUseSharedMemory()

void TritonClient::setUseSharedMemory ( bool  useShm )

inline

Definition at line 45 of file TritonClient.h.

References useSharedMemory_.

{ useSharedMemory_ = useShm; }

summarizeServerStats()

TritonClient::ServerSideStats TritonClient::summarizeServerStats ( const inference::ModelStatistics &  start_status, const inference::ModelStatistics &  end_status  ) const

protected

Definition at line 511 of file TritonClient.cc.

References TritonClient::ServerSideStats::compute_infer_time_ns_, TritonClient::ServerSideStats::compute_input_time_ns_, TritonClient::ServerSideStats::compute_output_time_ns_, TritonClient::ServerSideStats::cumm_time_ns_, TritonClient::ServerSideStats::execution_count_, TritonClient::ServerSideStats::inference_count_, TritonClient::ServerSideStats::queue_time_ns_, and TritonClient::ServerSideStats::success_count_.

Referenced by evaluate().

                                                                                                                   {
  TritonClient::ServerSideStats server_stats;

  server_stats.inference_count_ = end_status.inference_count() - start_status.inference_count();
  server_stats.execution_count_ = end_status.execution_count() - start_status.execution_count();
  server_stats.success_count_ =
      end_status.inference_stats().success().count() - start_status.inference_stats().success().count();
  server_stats.cumm_time_ns_ =
      end_status.inference_stats().success().ns() - start_status.inference_stats().success().ns();
  server_stats.queue_time_ns_ = end_status.inference_stats().queue().ns() - start_status.inference_stats().queue().ns();
  server_stats.compute_input_time_ns_ =
      end_status.inference_stats().compute_input().ns() - start_status.inference_stats().compute_input().ns();
  server_stats.compute_infer_time_ns_ =
      end_status.inference_stats().compute_infer().ns() - start_status.inference_stats().compute_infer().ns();
  server_stats.compute_output_time_ns_ =
      end_status.inference_stats().compute_output().ns() - start_status.inference_stats().compute_output().ns();

  return server_stats;
}

useSharedMemory()

bool TritonClient::useSharedMemory ( ) const

inline

Definition at line 44 of file TritonClient.h.

References useSharedMemory_.

{ return useSharedMemory_; }

verbose()

bool TritonClient::verbose ( ) const

inline

Definition at line 43 of file TritonClient.h.

References verbose_.

Referenced by evaluate().

{ return verbose_; }

Member Data Documentation

batchMode_

TritonBatchMode TritonClient::batchMode_

protected

Definition at line 76 of file TritonClient.h.

Referenced by addEntry(), batchMode(), batchSize(), reset(), resetBatchMode(), setBatchMode(), and setBatchSize().

client_

std::unique_ptr<triton::client::InferenceServerGrpcClient> TritonClient::client_

protected

Definition at line 84 of file TritonClient.h.

Referenced by client(), evaluate(), getServerSideStatus(), and TritonClient().

compressionAlgo_

grpc_compression_algorithm TritonClient::compressionAlgo_

protected

Definition at line 81 of file TritonClient.h.

Referenced by evaluate().

headers_

triton::client::Headers TritonClient::headers_

protected

Definition at line 82 of file TritonClient.h.

Referenced by evaluate().

manualBatchMode_

bool TritonClient::manualBatchMode_

protected

Definition at line 77 of file TritonClient.h.

Referenced by reset(), resetBatchMode(), and setBatchMode().

maxOuterDim_

unsigned TritonClient::maxOuterDim_

protected

Definition at line 72 of file TritonClient.h.

Referenced by setBatchSize(), and TritonClient().

nEntries_

unsigned TritonClient::nEntries_

protected

Definition at line 75 of file TritonClient.h.

noOuterDim_

bool TritonClient::noOuterDim_

protected

Definition at line 74 of file TritonClient.h.

Referenced by getResults(), noOuterDim(), and TritonClient().

options_

std::vector<triton::client::InferOptions> TritonClient::options_

protected

Definition at line 86 of file TritonClient.h.

Referenced by batchmanager.BatchManager::CheckBatchScript(), evaluate(), getServerSideStatus(), batchmanager.BatchManager::ManageOutputDir(), batchmanager.BatchManager::ParseOptions(), batchmanager.BatchManager::SubmitJob(), batchmanager.BatchManager::SubmitJobs(), and TritonClient().

outerDim_

unsigned TritonClient::outerDim_

protected

Definition at line 73 of file TritonClient.h.

Referenced by addEntry(), batchSize(), outerDim(), and setBatchSize().

serverType_

TritonServerType TritonClient::serverType_

protected

Definition at line 80 of file TritonClient.h.

Referenced by serverType(), and TritonClient().

TritonInputData

friend TritonClient::TritonInputData

private

Definition at line 89 of file TritonClient.h.

TritonOutputData

friend TritonClient::TritonOutputData

private

Definition at line 90 of file TritonClient.h.

useSharedMemory_

bool TritonClient::useSharedMemory_

protected

Definition at line 79 of file TritonClient.h.

Referenced by setUseSharedMemory(), and useSharedMemory().

verbose_

bool TritonClient::verbose_

protected

Definition at line 78 of file TritonClient.h.

Referenced by getServerSideStatus(), TritonClient(), and verbose().