#include <TritonData.h>

Public Types
using	Result = triton::client::InferResult

using	ShapeType = std::vector< int64_t >

using	ShapeView = edm::Span< ShapeType::const_iterator >

using	TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Public Member Functions
template<typename DT >
TritonInputContainer< DT >	allocate (bool reserve=true)

unsigned	batchSize () const

int64_t	byteSize () const

const std::string &	dname () const

template<typename DT >
TritonOutput< DT >	fromServer () const

void	prepare ()

void	setShape (const ShapeType &newShape)

void	setShape (unsigned loc, int64_t val)

const ShapeView &	shape () const

int64_t	sizeDims () const

int64_t	sizeShape () const

template<typename DT >
void	toServer (TritonInputContainer< DT > ptr)

	TritonData (const std::string &name, const TensorMetadata &model_info, TritonClient *client, const std::string &pid)

bool	variableDims () const

Private Member Functions
bool	anyNeg (const ShapeView &vec) const

triton::client::InferenceServerGrpcClient *	client ()

void	computeSizes ()

void	createObject (IO **ioptr)

IO *	data ()

int64_t	dimProduct (const ShapeView &vec) const

unsigned	fullLoc (unsigned loc) const

void	reset ()

void	resetSizes ()

void	setBatchSize (unsigned bsize)

void	setResult (std::shared_ptr< Result > result)

unsigned	uid () const

void	updateMem (size_t size)

std::string	xput () const

Private Attributes
unsigned	batchSize_

int64_t	byteSize_

size_t	byteSizePerBatch_

TritonClient *	client_

std::shared_ptr< IO >	data_

const ShapeType	dims_

std::string	dname_

bool	done_ {}

inference::DataType	dtype_

ShapeType	fullShape_

std::shared_ptr< void >	holder_

std::shared_ptr < TritonMemResource< IO > >	memResource_

std::string	name_

bool	noBatch_

int64_t	productDims_

std::shared_ptr< Result >	result_

ShapeView	shape_

std::string	shmName_

size_t	sizeShape_

size_t	totalByteSize_

bool	useShm_

bool	variableDims_

Friends
class	TritonClient

class	TritonCpuShmResource< IO >

class	TritonHeapResource< IO >

class	TritonMemResource< IO >

Detailed Description

template<typename IO>
class TritonData< IO >

Definition at line 45 of file TritonData.h.

Member Typedef Documentation

template<typename IO>

using TritonData< IO >::Result = triton::client::InferResult

Definition at line 47 of file TritonData.h.

template<typename IO>

using TritonData< IO >::ShapeType = std::vector<int64_t>

Definition at line 49 of file TritonData.h.

template<typename IO>

using TritonData< IO >::ShapeView = edm::Span<ShapeType::const_iterator>

Definition at line 50 of file TritonData.h.

template<typename IO>

using TritonData< IO >::TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Definition at line 48 of file TritonData.h.

Constructor & Destructor Documentation

template<typename IO>

TritonData< IO >::TritonData	(	const std::string &	name,
		const TensorMetadata &	model_info,
		TritonClient *	client,
		const std::string &	pid
	)

Definition at line 19 of file TritonData.cc.

References TritonData< IO >::createObject(), and TritonData< IO >::data_.

     : name_(name),
       client_(client),
       useShm_(client_->useSharedMemory()),
       //ensure unique name for shared memory region
       shmName_(useShm_ ? pid + "_" + xput() + std::to_string(uid()) : ""),
       dims_(model_info.shape().begin(), model_info.shape().end()),
       noBatch_(client_->noBatch()),
       batchSize_(0),
       fullShape_(dims_),
       shape_(fullShape_.begin() + (noBatch_ ? 0 : 1), fullShape_.end()),
       variableDims_(anyNeg(shape_)),
       productDims_(variableDims_ ? -1 : dimProduct(shape_)),
       dname_(model_info.datatype()),
       dtype_(ni::ProtocolStringToDataType(dname_)),
       byteSize_(ni::GetDataTypeByteSize(dtype_)),
       totalByteSize_(0) {
   //create input or output object
   IO* iotmp;
   createObject(&iotmp);
   data_.reset(iotmp);
 }

Member Function Documentation

template<typename DT >

template TritonInputContainer< int64_t > TritonInputData::allocate ( bool reserve = true )

Definition at line 150 of file TritonData.cc.

References TritonData< IO >::anyNeg(), TritonData< IO >::batchSize_, TritonData< IO >::computeSizes(), TritonData< IO >::shape_, and TritonData< IO >::sizeShape_.

                                                                {
   //automatically creates a vector for each batch entry (if batch size known)
   auto ptr = std::make_shared<TritonInput<DT>>(batchSize_);
   if (reserve and !anyNeg(shape_)) {
     computeSizes();
     for (auto& vec : *ptr) {
       vec.reserve(sizeShape_);
     }
   }
   return ptr;
 }

template<typename IO>

bool TritonData< IO >::anyNeg ( const ShapeView & vec ) const

inlineprivate

Definition at line 101 of file TritonData.h.

References edm::Span< T >::begin(), edm::Span< T >::end(), and mps_fire::i.

Referenced by TritonData< IO >::allocate().

                                           {
     return std::any_of(vec.begin(), vec.end(), [](int64_t i) { return i < 0; });
   }

template<typename IO>

unsigned TritonData< IO >::batchSize ( ) const

inline

Definition at line 72 of file TritonData.h.

References TritonData< IO >::batchSize_.

72 { return batchSize_; }

TritonData::batchSize_

unsigned batchSize_

Definition: TritonData.h:123

template<typename IO>

int64_t TritonData< IO >::byteSize ( ) const

inline

Definition at line 70 of file TritonData.h.

References TritonData< IO >::byteSize_.

70 { return byteSize_; }

TritonData::byteSize_

int64_t byteSize_

Definition: TritonData.h:130

template<typename IO >

tc::InferenceServerGrpcClient * TritonData< IO >::client ( )

private

Definition at line 68 of file TritonData.cc.

                                                     {
   return client_->client();
 }

template<typename IO >

void TritonData< IO >::computeSizes ( )

private

Definition at line 106 of file TritonData.cc.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::prepare(), and TritonData< IO >::toServer().

                                   {
   sizeShape_ = sizeShape();
   byteSizePerBatch_ = byteSize_ * sizeShape_;
   totalByteSize_ = byteSizePerBatch_ * batchSize_;
 }

template<typename IO>

void TritonData< IO >::createObject ( IO ** ioptr )

private

Referenced by TritonData< IO >::TritonData().

template<typename IO>

IO* TritonData< IO >::data ( )

inlineprivate

Definition at line 94 of file TritonData.h.

References TritonData< IO >::data_.

Referenced by TritonData< IO >::toServer().

94 { return data_.get(); }

TritonData::data_

std::shared_ptr< IO > data_

Definition: TritonData.h:117

template<typename IO>

int64_t TritonData< IO >::dimProduct ( const ShapeView & vec ) const

inlineprivate

Definition at line 104 of file TritonData.h.

References edm::Span< T >::begin(), and edm::Span< T >::end().

Referenced by TritonData< IO >::sizeShape().

                                                  {
     return std::accumulate(vec.begin(), vec.end(), 1, std::multiplies<int64_t>());
   }

template<typename IO>

const std::string& TritonData< IO >::dname ( ) const

inline

Definition at line 71 of file TritonData.h.

References TritonData< IO >::dname_.

71 { return dname_; }

TritonData::dname_

std::string dname_

Definition: TritonData.h:128

template<typename DT >

template TritonOutput< float > TritonOutputData::fromServer ( ) const

Definition at line 206 of file TritonData.cc.

References TritonData< IO >::batchSize_, TritonData< IO >::byteSize_, TritonData< IO >::dname_, TritonData< IO >::done_, GeomDetEnumerators::DT, Exception, TritonData< IO >::memResource_, TritonData< IO >::name_, hltrates_dqm_sourceclient-live_cfg::offset, diffTwoXMLs::r1, TritonData< IO >::result_, and TritonData< IO >::sizeShape_.

                                                     {
   //shouldn't be called twice
   if (done_)
     throw cms::Exception("TritonDataError") << name_ << " fromServer() was already called for this event";
 
   if (!result_) {
     throw cms::Exception("TritonDataError") << name_ << " fromServer(): missing result";
   }
 
   if (byteSize_ != sizeof(DT)) {
     throw cms::Exception("TritonDataError") << name_ << " fromServer(): inconsistent byte size " << sizeof(DT)
                                             << " (should be " << byteSize_ << " for " << dname_ << ")";
   }
 
   const uint8_t* r0 = memResource_->copyOutput();
   const DT* r1 = reinterpret_cast<const DT*>(r0);
 
   TritonOutput<DT> dataOut;
   dataOut.reserve(batchSize_);
   for (unsigned i0 = 0; i0 < batchSize_; ++i0) {
     auto offset = i0 * sizeShape_;
     dataOut.emplace_back(r1 + offset, r1 + offset + sizeShape_);
   }
 
   done_ = true;
   return dataOut;
 }

template<typename IO>

unsigned TritonData< IO >::fullLoc ( unsigned loc ) const

inlineprivate

Definition at line 90 of file TritonData.h.

References TritonData< IO >::noBatch_.

Referenced by TritonData< IO >::reset().

90 { return loc + (noBatch_ ? 0 : 1); }

TritonData::noBatch_

bool noBatch_

Definition: TritonData.h:122

void TritonOutputData::prepare ( )

Definition at line 198 of file TritonData.cc.

References TritonData< IO >::computeSizes(), TritonData< IO >::memResource_, TritonData< IO >::totalByteSize_, and TritonData< IO >::updateMem().

                                {
   computeSizes();
   updateMem(totalByteSize_);
   memResource_->set();
 }

void TritonOutputData::reset ( void )

private

Definition at line 235 of file TritonData.cc.

References TritonData< IO >::data_, TritonData< IO >::dims_, TritonData< IO >::done_, TritonData< IO >::fullLoc(), TritonData< IO >::fullShape_, TritonData< IO >::holder_, mps_fire::i, TritonData< IO >::resetSizes(), TritonData< IO >::shape_, edm::Span< T >::size(), and TritonData< IO >::variableDims_.

                             {
   done_ = false;
   holder_.reset();
   data_->Reset();
   //reset shape
   if (variableDims_) {
     for (unsigned i = 0; i < shape_.size(); ++i) {
       unsigned locFull = fullLoc(i);
       fullShape_[locFull] = dims_[locFull];
     }
   }
   resetSizes();
 }

template<typename IO >

void TritonData< IO >::resetSizes ( )

private

Definition at line 112 of file TritonData.cc.

Referenced by TritonData< IO >::reset().

                                 {
   sizeShape_ = 0;
   byteSizePerBatch_ = 0;
   totalByteSize_ = 0;
 }

template<typename IO >

void TritonData< IO >::setBatchSize ( unsigned bsize )

private

Definition at line 99 of file TritonData.cc.

                                                 {
   batchSize_ = bsize;
   if (!noBatch_)
     fullShape_[0] = batchSize_;
 }

template<typename IO>

void TritonData< IO >::setResult ( std::shared_ptr< Result > result )

inlineprivate

Definition at line 93 of file TritonData.h.

References mps_fire::result, and TritonData< IO >::result_.

93 { result_ = result; }

mps_fire.result

tuple result

Definition: mps_fire.py:311

TritonData::result_

std::shared_ptr< Result > result_

Definition: TritonData.h:139

template<typename IO>

void TritonData< IO >::setShape ( const ShapeType & newShape )

template<typename IO >

void TritonData< IO >::setShape	(	unsigned	loc,
		int64_t	val
	)

Definition at line 81 of file TritonData.cc.

References Exception, and hgcalPerformanceValidation::val.

                                                        {
   unsigned locFull = fullLoc(loc);
 
   //check boundary
   if (locFull >= fullShape_.size())
     throw cms::Exception("TritonDataError")
         << name_ << " setShape(): dimension " << locFull << " out of bounds (" << fullShape_.size() << ")";
 
   if (val != fullShape_[locFull]) {
     if (dims_[locFull] == -1)
       fullShape_[locFull] = val;
     else
       throw cms::Exception("TritonDataError")
           << name_ << " setShape(): attempt to change value of non-variable shape dimension " << loc;
   }
 }

template<typename IO>

const ShapeView& TritonData< IO >::shape ( void ) const

inline

Definition at line 69 of file TritonData.h.

References TritonData< IO >::shape_.

69 { return shape_; }

TritonData::shape_

ShapeView shape_

Definition: TritonData.h:125

template<typename IO>

int64_t TritonData< IO >::sizeDims ( ) const

inline

Definition at line 76 of file TritonData.h.

References TritonData< IO >::productDims_.

Referenced by TritonData< IO >::sizeShape().

76 { return productDims_; }

TritonData::productDims_

int64_t productDims_

Definition: TritonData.h:127

template<typename IO>

int64_t TritonData< IO >::sizeShape ( ) const

inline

Definition at line 78 of file TritonData.h.

References TritonData< IO >::dimProduct(), TritonData< IO >::shape_, TritonData< IO >::sizeDims(), and TritonData< IO >::variableDims_.

78 { return variableDims_ ? dimProduct(shape_) : sizeDims(); }

TritonData::dimProduct

int64_t dimProduct(const ShapeView &vec) const

Definition: TritonData.h:104

TritonData::shape_

ShapeView shape_

Definition: TritonData.h:125

TritonData::variableDims_

bool variableDims_

Definition: TritonData.h:126

TritonData::sizeDims

int64_t sizeDims() const

Definition: TritonData.h:76

template<typename DT >

void TritonInputData::toServer ( TritonInputContainer< DT > ptr )

Definition at line 164 of file TritonData.cc.

References TritonData< IO >::batchSize_, TritonData< IO >::byteSize_, TritonData< IO >::byteSizePerBatch_, TritonData< IO >::computeSizes(), TritonData< IO >::data(), TritonData< IO >::data_, TritonData< IO >::dname_, TritonData< IO >::done_, GeomDetEnumerators::DT, Exception, TritonData< IO >::fullShape_, TritonData< IO >::holder_, TritonData< IO >::memResource_, TritonData< IO >::name_, TritonData< IO >::totalByteSize_, and TritonData< IO >::updateMem().

                                                            {
   //shouldn't be called twice
   if (done_)
     throw cms::Exception("TritonDataError") << name_ << " toServer() was already called for this event";
 
   const auto& data_in = *ptr;
 
   //check batch size
   if (data_in.size() != batchSize_) {
     throw cms::Exception("TritonDataError") << name_ << " toServer(): input vector has size " << data_in.size()
                                             << " but specified batch size is " << batchSize_;
   }
 
   //shape must be specified for variable dims or if batch size changes
   data_->SetShape(fullShape_);
 
   if (byteSize_ != sizeof(DT))
     throw cms::Exception("TritonDataError") << name_ << " toServer(): inconsistent byte size " << sizeof(DT)
                                             << " (should be " << byteSize_ << " for " << dname_ << ")";
 
   computeSizes();
   updateMem(totalByteSize_);
   for (unsigned i0 = 0; i0 < batchSize_; ++i0) {
     memResource_->copyInput(data_in[i0].data(), i0 * byteSizePerBatch_);
   }
   memResource_->set();
 
   //keep input data in scope
   holder_ = ptr;
   done_ = true;
 }

template<typename IO>

unsigned TritonData< IO >::uid ( ) const

inlineprivate

Definition at line 109 of file TritonData.h.

                        {
     static std::atomic<unsigned> uid{0};
     return ++uid;
   }

template<typename IO >

void TritonData< IO >::updateMem ( size_t size )

private

Definition at line 121 of file TritonData.cc.

References LocalCPU, LocalGPU, or, and findQualityFiles::size.

Referenced by TritonData< IO >::prepare(), and TritonData< IO >::toServer().

                                           {
   if (!memResource_ or size > memResource_->size()) {
     if (useShm_ and client_->serverType() == TritonServerType::LocalCPU) {
       //avoid unnecessarily throwing in destructor
       if (memResource_)
         memResource_->close();
       //need to destroy before constructing new instance because shared memory key will be reused
       memResource_.reset();
       memResource_ = std::make_shared<TritonCpuShmResource<IO>>(this, shmName_, size);
     }
 #ifdef TRITON_ENABLE_GPU
     else if (useShm_ and client_->serverType() == TritonServerType::LocalGPU) {
       //avoid unnecessarily throwing in destructor
       if (memResource_)
         memResource_->close();
       //need to destroy before constructing new instance because shared memory key will be reused
       memResource_.reset();
       memResource_ = std::make_shared<TritonGpuShmResource<IO>>(this, shmName_, size);
     }
 #endif
     //for remote/heap, size increases don't matter
     else if (!memResource_)
       memResource_ = std::make_shared<TritonHeapResource<IO>>(this, shmName_, size);
   }
 }

template<typename IO>

bool TritonData< IO >::variableDims ( ) const

inline

Definition at line 75 of file TritonData.h.

References TritonData< IO >::variableDims_.

75 { return variableDims_; }

TritonData::variableDims_

bool variableDims_

Definition: TritonData.h:126

std::string TritonOutputData::xput ( ) const

private

Definition at line 58 of file TritonData.cc.

                                       {
   return "input";
 }

Friends And Related Function Documentation

template<typename IO>

friend class TritonClient

friend

Definition at line 81 of file TritonData.h.

template<typename IO>

friend class TritonCpuShmResource< IO >

friend

Definition at line 84 of file TritonData.h.

template<typename IO>

friend class TritonHeapResource< IO >

friend

Definition at line 83 of file TritonData.h.

template<typename IO>

friend class TritonMemResource< IO >

friend

Definition at line 82 of file TritonData.h.

Member Data Documentation

template<typename IO>

unsigned TritonData< IO >::batchSize_

private

Definition at line 123 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::batchSize(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

template<typename IO>

int64_t TritonData< IO >::byteSize_

private

Definition at line 130 of file TritonData.h.

Referenced by TritonData< IO >::byteSize(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

template<typename IO>

size_t TritonData< IO >::byteSizePerBatch_

private

Definition at line 132 of file TritonData.h.

Referenced by TritonData< IO >::toServer().

template<typename IO>

TritonClient* TritonData< IO >::client_

private

Definition at line 118 of file TritonData.h.

template<typename IO>

std::shared_ptr<IO> TritonData< IO >::data_

private

Definition at line 117 of file TritonData.h.

Referenced by TritonData< IO >::data(), TritonData< IO >::reset(), TritonData< IO >::toServer(), and TritonData< IO >::TritonData().

template<typename IO>

const ShapeType TritonData< IO >::dims_

private

Definition at line 121 of file TritonData.h.

Referenced by TritonData< IO >::reset().

template<typename IO>

std::string TritonData< IO >::dname_

private

Definition at line 128 of file TritonData.h.

Referenced by TritonData< IO >::dname(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

template<typename IO>

bool TritonData< IO >::done_ {}

mutableprivate

Definition at line 141 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), TritonData< IO >::reset(), and TritonData< IO >::toServer().

template<typename IO>

inference::DataType TritonData< IO >::dtype_

private

Definition at line 129 of file TritonData.h.

template<typename IO>

ShapeType TritonData< IO >::fullShape_

private

Definition at line 124 of file TritonData.h.

Referenced by TritonData< IO >::reset(), and TritonData< IO >::toServer().

template<typename IO>

std::shared_ptr<void> TritonData< IO >::holder_

private

Definition at line 137 of file TritonData.h.

Referenced by TritonData< IO >::reset(), and TritonData< IO >::toServer().

template<typename IO>

std::shared_ptr<TritonMemResource<IO> > TritonData< IO >::memResource_

private

Definition at line 138 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), TritonData< IO >::prepare(), and TritonData< IO >::toServer().

template<typename IO>

std::string TritonData< IO >::name_

private

Definition at line 116 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

template<typename IO>

bool TritonData< IO >::noBatch_

private

Definition at line 122 of file TritonData.h.

Referenced by TritonData< IO >::fullLoc().

template<typename IO>

int64_t TritonData< IO >::productDims_

private

Definition at line 127 of file TritonData.h.

Referenced by TritonData< IO >::sizeDims().

template<typename IO>

std::shared_ptr<Result> TritonData< IO >::result_

private

Definition at line 139 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), and TritonData< IO >::setResult().

template<typename IO>

ShapeView TritonData< IO >::shape_

private

Definition at line 125 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::reset(), TritonData< IO >::shape(), and TritonData< IO >::sizeShape().

template<typename IO>

std::string TritonData< IO >::shmName_

private

Definition at line 120 of file TritonData.h.

template<typename IO>

size_t TritonData< IO >::sizeShape_

private

Definition at line 131 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), and TritonData< IO >::fromServer().

template<typename IO>

size_t TritonData< IO >::totalByteSize_

private

Definition at line 133 of file TritonData.h.

Referenced by TritonData< IO >::prepare(), and TritonData< IO >::toServer().

template<typename IO>

bool TritonData< IO >::useShm_

private

Definition at line 119 of file TritonData.h.

template<typename IO>

bool TritonData< IO >::variableDims_

private

Definition at line 126 of file TritonData.h.

Referenced by TritonData< IO >::reset(), TritonData< IO >::sizeShape(), and TritonData< IO >::variableDims().

Public Types

Public Member Functions

Private Member Functions

Private Attributes

Friends

Detailed Description

template<typename IO> class TritonData< IO >

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Friends And Related Function Documentation

Member Data Documentation

template<typename IO>
class TritonData< IO >