#include <TritonData.h>

Public Types
using	Result = nvidia::inferenceserver::client::InferResult

using	ShapeType = std::vector< int64_t >

using	ShapeView = edm::Span< ShapeType::const_iterator >

using	TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Public Member Functions
unsigned	batchSize () const

int64_t	byteSize () const

const std::string &	dname () const

template<typename DT >
TritonOutput< DT >	fromServer () const

bool	setShape (const ShapeType &newShape)

bool	setShape (unsigned loc, int64_t val)

const ShapeView &	shape () const

int64_t	sizeDims () const

int64_t	sizeShape () const

template<typename DT >
void	toServer (std::shared_ptr< TritonInput< DT >> ptr)

	TritonData (const std::string &name, const TensorMetadata &model_info, bool noBatch)

bool	variableDims () const

Private Member Functions
bool	anyNeg (const ShapeView &vec) const

void	createObject (IO **ioptr) const

IO *	data ()

int64_t	dimProduct (const ShapeView &vec) const

void	reset ()

void	setBatchSize (unsigned bsize)

void	setResult (std::shared_ptr< Result > result)

bool	setShape (const ShapeType &newShape, bool canThrow)

bool	setShape (unsigned loc, int64_t val, bool canThrow)

Private Attributes
unsigned	batchSize_

int64_t	byteSize_

std::shared_ptr< IO >	data_

const ShapeType	dims_

std::string	dname_

inference::DataType	dtype_

ShapeType	fullShape_

std::any	holder_

std::string	name_

bool	noBatch_

int64_t	productDims_

std::shared_ptr< Result >	result_

ShapeView	shape_

bool	variableDims_

Friends
class	TritonClient

Detailed Description

template<typename IO>
class TritonData< IO >

Definition at line 29 of file TritonData.h.

Member Typedef Documentation

◆ Result

template<typename IO>

using TritonData< IO >::Result = nvidia::inferenceserver::client::InferResult

Definition at line 31 of file TritonData.h.

◆ ShapeType

template<typename IO>

using TritonData< IO >::ShapeType = std::vector<int64_t>

Definition at line 33 of file TritonData.h.

◆ ShapeView

template<typename IO>

using TritonData< IO >::ShapeView = edm::Span<ShapeType::const_iterator>

Definition at line 34 of file TritonData.h.

◆ TensorMetadata

template<typename IO>

using TritonData< IO >::TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Definition at line 32 of file TritonData.h.

Constructor & Destructor Documentation

◆ TritonData()

template<typename IO>

TritonData< IO >::TritonData	(	const std::string &	name,
		const TensorMetadata &	model_info,
		bool	noBatch
	)

Definition at line 25 of file TritonData.cc.

     : name_(name),
       dims_(model_info.shape().begin(), model_info.shape().end()),
       noBatch_(noBatch),
       batchSize_(0),
       fullShape_(dims_),
       shape_(fullShape_.begin() + (noBatch_ ? 0 : 1), fullShape_.end()),
       variableDims_(anyNeg(shape_)),
       productDims_(variableDims_ ? -1 : dimProduct(shape_)),
       dname_(model_info.datatype()),
       dtype_(ni::ProtocolStringToDataType(dname_)),
       byteSize_(ni::GetDataTypeByteSize(dtype_)) {
   //create input or output object
   IO* iotmp;
   createObject(&iotmp);
   data_.reset(iotmp);
 }

References TritonData< IO >::createObject(), and TritonData< IO >::data_.

Member Function Documentation

◆ anyNeg()

template<typename IO>

bool TritonData< IO >::anyNeg ( const ShapeView & vec ) const

inlineprivate

Definition at line 73 of file TritonData.h.

                                           {
     return std::any_of(vec.begin(), vec.end(), [](int64_t i) { return i < 0; });
   }

References edm::Span< T >::begin(), edm::Span< T >::end(), and mps_fire::i.

◆ batchSize()

template<typename IO>

unsigned TritonData< IO >::batchSize ( ) const

inline

Definition at line 53 of file TritonData.h.

53 { return batchSize_; }

References TritonData< IO >::batchSize_.

◆ byteSize()

template<typename IO>

int64_t TritonData< IO >::byteSize ( ) const

inline

Definition at line 51 of file TritonData.h.

51 { return byteSize_; }

References TritonData< IO >::byteSize_.

◆ createObject()

template<typename IO>

void TritonData< IO >::createObject ( IO ** ioptr ) const

private

Referenced by TritonData< IO >::TritonData().

◆ data()

template<typename IO>

IO* TritonData< IO >::data ( )

inlineprivate

Definition at line 70 of file TritonData.h.

70 { return data_.get(); }

References TritonData< IO >::data_.

◆ dimProduct()

template<typename IO>

int64_t TritonData< IO >::dimProduct ( const ShapeView & vec ) const

inlineprivate

Definition at line 76 of file TritonData.h.

                                                  {
     return std::accumulate(vec.begin(), vec.end(), 1, std::multiplies<int64_t>());
   }

References edm::Span< T >::begin(), and edm::Span< T >::end().

Referenced by TritonData< IO >::sizeShape().

◆ dname()

template<typename IO>

const std::string& TritonData< IO >::dname ( ) const

inline

Definition at line 52 of file TritonData.h.

52 { return dname_; }

References TritonData< IO >::dname_.

◆ fromServer()

template<typename DT >

TritonOutput< DT > TritonOutputData::fromServer ( ) const

Definition at line 136 of file TritonData.cc.

                                                     {
   if (!result_) {
     throw cms::Exception("TritonDataError") << name_ << " output(): missing result";
   }
  
   if (byteSize_ != sizeof(DT)) {
     throw cms::Exception("TritonDataError") << name_ << " output(): inconsistent byte size " << sizeof(DT)
                                             << " (should be " << byteSize_ << " for " << dname_ << ")";
   }
  
   uint64_t nOutput = sizeShape();
   TritonOutput<DT> dataOut;
   const uint8_t* r0;
   size_t contentByteSize;
   size_t expectedContentByteSize = nOutput * byteSize_ * batchSize_;
   triton_utils::throwIfError(result_->RawData(name_, &r0, &contentByteSize), "output(): unable to get raw");
   if (contentByteSize != expectedContentByteSize) {
     throw cms::Exception("TritonDataError") << name_ << " output(): unexpected content byte size " << contentByteSize
                                             << " (expected " << expectedContentByteSize << ")";
   }
  
   const DT* r1 = reinterpret_cast<const DT*>(r0);
   dataOut.reserve(batchSize_);
   for (unsigned i0 = 0; i0 < batchSize_; ++i0) {
     auto offset = i0 * nOutput;
     dataOut.emplace_back(r1 + offset, r1 + offset + nOutput);
   }
  
   return dataOut;
 }

References TritonData< IO >::batchSize_, TritonData< IO >::byteSize_, TritonData< IO >::dname_, GeomDetEnumerators::DT, Exception, TritonData< IO >::name_, hltrates_dqm_sourceclient-live_cfg::offset, diffTwoXMLs::r1, TritonData< IO >::result_, TritonData< IO >::sizeShape(), and triton_utils::throwIfError().

◆ reset()

void TritonOutputData::reset ( void )

private

Definition at line 168 of file TritonData.cc.

                             {
   data_->Reset();
   holder_.reset();
 }

References TritonData< IO >::data_, and TritonData< IO >::holder_.

◆ setBatchSize()

template<typename IO >

void TritonData< IO >::setBatchSize ( unsigned bsize )

private

Definition at line 98 of file TritonData.cc.

                                                 {
   batchSize_ = bsize;
   if (!noBatch_)
     fullShape_[0] = batchSize_;
 }

◆ setResult()

template<typename IO>

void TritonData< IO >::setResult ( std::shared_ptr< Result > result )

inlineprivate

Definition at line 69 of file TritonData.h.

69 { result_ = result; }

References mps_fire::result, and TritonData< IO >::result_.

◆ setShape() [1/4]

template<typename IO>

bool TritonData< IO >::setShape ( const ShapeType & newShape )

inline

Definition at line 40 of file TritonData.h.

40 { return setShape(newShape, true); }

References TritonData< IO >::setShape().

Referenced by TritonData< IO >::setShape().

◆ setShape() [2/4]

template<typename IO>

bool TritonData< IO >::setShape	(	const ShapeType &	newShape,
		bool	canThrow
	)

private

◆ setShape() [3/4]

template<typename IO>

bool TritonData< IO >::setShape	(	unsigned	loc,
		int64_t	val
	)

inline

Definition at line 41 of file TritonData.h.

41 { return setShape(loc, val, true); }

References TritonData< IO >::setShape(), and heppy_batch::val.

Referenced by TritonData< IO >::setShape().

◆ setShape() [4/4]

template<typename IO >

bool TritonData< IO >::setShape	(	unsigned	loc,
		int64_t	val,
		bool	canThrow
	)

private

Definition at line 64 of file TritonData.cc.

                                                                       {
   std::stringstream msg;
   unsigned full_loc = loc + (noBatch_ ? 0 : 1);
  
   //check boundary
   if (full_loc >= fullShape_.size()) {
     msg << name_ << " setShape(): dimension " << full_loc << " out of bounds (" << fullShape_.size() << ")";
     if (canThrow)
       throw cms::Exception("TritonDataError") << msg.str();
     else {
       edm::LogWarning("TritonDataWarning") << msg.str();
       return false;
     }
   }
  
   if (val != fullShape_[full_loc]) {
     if (dims_[full_loc] == -1) {
       fullShape_[full_loc] = val;
       return true;
     } else {
       msg << name_ << " setShape(): attempt to change value of non-variable shape dimension " << loc;
       if (canThrow)
         throw cms::Exception("TritonDataError") << msg.str();
       else {
         edm::LogWarning("TritonDataError") << msg.str();
         return false;
       }
     }
   }
  
   return true;
 }

References Exception, mps_check::msg, and heppy_batch::val.

◆ shape()

template<typename IO>

const ShapeView& TritonData< IO >::shape ( void ) const

inline

Definition at line 50 of file TritonData.h.

50 { return shape_; }

References TritonData< IO >::shape_.

◆ sizeDims()

template<typename IO>

int64_t TritonData< IO >::sizeDims ( ) const

inline

Definition at line 57 of file TritonData.h.

57 { return productDims_; }

References TritonData< IO >::productDims_.

Referenced by TritonData< IO >::sizeShape().

◆ sizeShape()

template<typename IO>

int64_t TritonData< IO >::sizeShape ( ) const

inline

Definition at line 59 of file TritonData.h.

59 { return variableDims_ ? dimProduct(shape_) : sizeDims(); }

References TritonData< IO >::dimProduct(), TritonData< IO >::shape_, TritonData< IO >::sizeDims(), and TritonData< IO >::variableDims_.

Referenced by TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

◆ toServer()

template<typename DT >

void TritonInputData::toServer ( std::shared_ptr< TritonInput< DT >> ptr )

Definition at line 107 of file TritonData.cc.

                                                                  {
   const auto& data_in = *ptr;
  
   //check batch size
   if (data_in.size() != batchSize_) {
     throw cms::Exception("TritonDataError") << name_ << " input(): input vector has size " << data_in.size()
                                             << " but specified batch size is " << batchSize_;
   }
  
   //shape must be specified for variable dims or if batch size changes
   data_->SetShape(fullShape_);
  
   if (byteSize_ != sizeof(DT))
     throw cms::Exception("TritonDataError") << name_ << " input(): inconsistent byte size " << sizeof(DT)
                                             << " (should be " << byteSize_ << " for " << dname_ << ")";
  
   int64_t nInput = sizeShape();
   for (unsigned i0 = 0; i0 < batchSize_; ++i0) {
     const DT* arr = data_in[i0].data();
     triton_utils::throwIfError(data_->AppendRaw(reinterpret_cast<const uint8_t*>(arr), nInput * byteSize_),
                                name_ + " input(): unable to set data for batch entry " + std::to_string(i0));
   }
  
   //keep input data in scope
   holder_ = std::move(ptr);
 }

References TritonData< IO >::batchSize_, TritonData< IO >::byteSize_, TritonData< IO >::data_, TritonData< IO >::dname_, GeomDetEnumerators::DT, Exception, TritonData< IO >::fullShape_, TritonData< IO >::holder_, eostools::move(), TritonData< IO >::name_, TritonData< IO >::sizeShape(), and triton_utils::throwIfError().

◆ variableDims()

template<typename IO>

bool TritonData< IO >::variableDims ( ) const

inline

Definition at line 56 of file TritonData.h.

56 { return variableDims_; }

References TritonData< IO >::variableDims_.

Friends And Related Function Documentation

◆ TritonClient

template<typename IO>

friend class TritonClient

friend

Definition at line 62 of file TritonData.h.

Member Data Documentation

◆ batchSize_

template<typename IO>

unsigned TritonData< IO >::batchSize_

private

Definition at line 86 of file TritonData.h.

Referenced by TritonData< IO >::batchSize(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

◆ byteSize_

template<typename IO>

int64_t TritonData< IO >::byteSize_

private

Definition at line 93 of file TritonData.h.

Referenced by TritonData< IO >::byteSize(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

◆ data_

template<typename IO>

std::shared_ptr<IO> TritonData< IO >::data_

private

Definition at line 83 of file TritonData.h.

Referenced by TritonData< IO >::data(), TritonData< IO >::reset(), TritonData< IO >::toServer(), and TritonData< IO >::TritonData().

◆ dims_

template<typename IO>

const ShapeType TritonData< IO >::dims_

private

Definition at line 84 of file TritonData.h.

◆ dname_

template<typename IO>

std::string TritonData< IO >::dname_

private

Definition at line 91 of file TritonData.h.

Referenced by TritonData< IO >::dname(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

◆ dtype_

template<typename IO>

inference::DataType TritonData< IO >::dtype_

private

Definition at line 92 of file TritonData.h.

◆ fullShape_

template<typename IO>

ShapeType TritonData< IO >::fullShape_

private

Definition at line 87 of file TritonData.h.

Referenced by TritonData< IO >::toServer().

◆ holder_

template<typename IO>

std::any TritonData< IO >::holder_

private

Definition at line 94 of file TritonData.h.

Referenced by TritonData< IO >::reset(), and TritonData< IO >::toServer().

◆ name_

template<typename IO>

std::string TritonData< IO >::name_

private

Definition at line 82 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

◆ noBatch_

template<typename IO>

bool TritonData< IO >::noBatch_

private

Definition at line 85 of file TritonData.h.

◆ productDims_

template<typename IO>

int64_t TritonData< IO >::productDims_

private

Definition at line 90 of file TritonData.h.

Referenced by TritonData< IO >::sizeDims().

◆ result_

template<typename IO>

std::shared_ptr<Result> TritonData< IO >::result_

private

Definition at line 95 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), and TritonData< IO >::setResult().

◆ shape_

template<typename IO>

ShapeView TritonData< IO >::shape_

private

Definition at line 88 of file TritonData.h.

Referenced by TritonData< IO >::shape(), and TritonData< IO >::sizeShape().

◆ variableDims_

template<typename IO>

bool TritonData< IO >::variableDims_

private

Definition at line 89 of file TritonData.h.

Referenced by TritonData< IO >::sizeShape(), and TritonData< IO >::variableDims().

Public Types

Public Member Functions

Private Member Functions

Private Attributes

Friends

Detailed Description

template<typename IO> class TritonData< IO >

Member Typedef Documentation

◆ Result

◆ ShapeType

◆ ShapeView

◆ TensorMetadata

Constructor & Destructor Documentation

◆ TritonData()

Member Function Documentation

◆ anyNeg()

◆ batchSize()

◆ byteSize()

◆ createObject()

◆ data()

◆ dimProduct()

◆ dname()

◆ fromServer()

◆ reset()

◆ setBatchSize()

◆ setResult()

◆ setShape() [1/4]

◆ setShape() [2/4]

◆ setShape() [3/4]

◆ setShape() [4/4]

◆ shape()

◆ sizeDims()

◆ sizeShape()

◆ toServer()

◆ variableDims()

Friends And Related Function Documentation

◆ TritonClient

Member Data Documentation

◆ batchSize_

◆ byteSize_

◆ data_

◆ dims_

◆ dname_

◆ dtype_

◆ fullShape_

◆ holder_

◆ name_

◆ noBatch_

◆ productDims_

◆ result_

◆ shape_

◆ variableDims_

template<typename IO>
class TritonData< IO >