TritonData< IO > Class Template Reference

#include <TritonData.h>

Public Types
using	Result = triton::client::InferResult
using	ShapeType = std::vector< int64_t >
using	ShapeView = edm::Span< ShapeType::const_iterator >
using	TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Public Member Functions
template<typename DT >
TritonInputContainer< DT >	allocate (bool reserve=true)
unsigned	batchSize () const
int64_t	byteSize () const
const std::string &	dname () const
template<typename DT >
TritonOutput< DT >	fromServer () const
void	prepare ()
void	setShape (const ShapeType &newShape)
void	setShape (unsigned loc, int64_t val)
const ShapeView &	shape () const
int64_t	sizeDims () const
int64_t	sizeShape () const
template<typename DT >
void	toServer (TritonInputContainer< DT > ptr)
	TritonData (const std::string &name, const TensorMetadata &model_info, TritonClient *client, const std::string &pid)
bool	variableDims () const

Private Member Functions
bool	anyNeg (const ShapeView &vec) const
triton::client::InferenceServerGrpcClient *	client ()
void	computeSizes ()
void	createObject (IO **ioptr)
IO *	data ()
int64_t	dimProduct (const ShapeView &vec) const
unsigned	fullLoc (unsigned loc) const
void	reset ()
void	resetSizes ()
void	setBatchSize (unsigned bsize)
void	setResult (std::shared_ptr< Result > result)
unsigned	uid () const
void	updateMem (size_t size)
std::string	xput () const

Private Attributes
unsigned	batchSize_
int64_t	byteSize_
size_t	byteSizePerBatch_
TritonClient *	client_
std::shared_ptr< IO >	data_
const ShapeType	dims_
std::string	dname_
bool	done_ {}
inference::DataType	dtype_
ShapeType	fullShape_
std::shared_ptr< void >	holder_
std::shared_ptr< TritonMemResource< IO > >	memResource_
std::string	name_
bool	noBatch_
int64_t	productDims_
std::shared_ptr< Result >	result_
ShapeView	shape_
std::string	shmName_
size_t	sizeShape_
size_t	totalByteSize_
bool	useShm_
bool	variableDims_

Friends
class	TritonClient
class	TritonCpuShmResource< IO >
class	TritonHeapResource< IO >
class	TritonMemResource< IO >

Detailed Description

template<typename IO>
class TritonData< IO >

Definition at line 45 of file TritonData.h.

Member Typedef Documentation

Result

template<typename IO>

using TritonData< IO >::Result = triton::client::InferResult

Definition at line 47 of file TritonData.h.

ShapeType

template<typename IO>

using TritonData< IO >::ShapeType = std::vector<int64_t>

Definition at line 49 of file TritonData.h.

ShapeView

template<typename IO>

using TritonData< IO >::ShapeView = edm::Span<ShapeType::const_iterator>

Definition at line 50 of file TritonData.h.

TensorMetadata

template<typename IO>

using TritonData< IO >::TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Definition at line 48 of file TritonData.h.

Constructor & Destructor Documentation

TritonData()

template<typename IO>

TritonData< IO >::TritonData	(	const std::string &	name,
		const TensorMetadata &	model_info,
		TritonClient *	client,
		const std::string &	pid
	)

Definition at line 19 of file TritonData.cc.

    : name_(name),
      client_(client),
      useShm_(client_->useSharedMemory()),
      //ensure unique name for shared memory region
      shmName_(useShm_ ? pid + "_" + xput() + std::to_string(uid()) : ""),
      dims_(model_info.shape().begin(), model_info.shape().end()),
      noBatch_(client_->noBatch()),
      batchSize_(0),
      fullShape_(dims_),
      shape_(fullShape_.begin() + (noBatch_ ? 0 : 1), fullShape_.end()),
      variableDims_(anyNeg(shape_)),
      productDims_(variableDims_ ? -1 : dimProduct(shape_)),
      dname_(model_info.datatype()),
      dtype_(ni::ProtocolStringToDataType(dname_)),
      byteSize_(ni::GetDataTypeByteSize(dtype_)),
      totalByteSize_(0) {
  //create input or output object
  IO* iotmp;
  createObject(&iotmp);
  data_.reset(iotmp);
}

References TritonData< IO >::createObject(), and TritonData< IO >::data_.

Member Function Documentation

allocate()

template<typename DT >

TritonInputContainer< DT > TritonInputData::allocate

(

bool

reserve = true

)

Definition at line 150 of file TritonData.cc.

                                                               {
  //automatically creates a vector for each batch entry (if batch size known)
  auto ptr = std::make_shared<TritonInput<DT>>(batchSize_);
  if (reserve and !anyNeg(shape_)) {
    computeSizes();
    for (auto& vec : *ptr) {
      vec.reserve(sizeShape_);
    }
  }
  return ptr;
}

References TritonData< IO >::anyNeg(), TritonData< IO >::batchSize_, TritonData< IO >::computeSizes(), TritonData< IO >::shape_, and TritonData< IO >::sizeShape_.

anyNeg()

template<typename IO>

bool TritonData< IO >::anyNeg ( const ShapeView &  vec ) const

inlineprivate

Definition at line 101 of file TritonData.h.

                                          {
    return std::any_of(vec.begin(), vec.end(), [](int64_t i) { return i < 0; });
  }

References edm::Span< T >::begin(), edm::Span< T >::end(), and mps_fire::i.

Referenced by TritonData< IO >::allocate().

batchSize()

template<typename IO>

unsigned TritonData< IO >::batchSize ( ) const

inline

Definition at line 72 of file TritonData.h.

{ return batchSize_; }

References TritonData< IO >::batchSize_.

byteSize()

template<typename IO>

int64_t TritonData< IO >::byteSize ( ) const

inline

Definition at line 70 of file TritonData.h.

{ return byteSize_; }

References TritonData< IO >::byteSize_.

client()

template<typename IO >

tc::InferenceServerGrpcClient * TritonData< IO >::client ( )

private

Definition at line 68 of file TritonData.cc.

                                                    {
  return client_->client();
}

computeSizes()

template<typename IO >

void TritonData< IO >::computeSizes ( )

private

Definition at line 106 of file TritonData.cc.

                                  {
  sizeShape_ = sizeShape();
  byteSizePerBatch_ = byteSize_ * sizeShape_;
  totalByteSize_ = byteSizePerBatch_ * batchSize_;
}

Referenced by TritonData< IO >::allocate(), TritonData< IO >::prepare(), and TritonData< IO >::toServer().

createObject()

template<typename IO>

void TritonData< IO >::createObject ( IO **  ioptr )

private

Referenced by TritonData< IO >::TritonData().

data()

template<typename IO>

IO* TritonData< IO >::data ( )

inlineprivate

Definition at line 94 of file TritonData.h.

{ return data_.get(); }

References TritonData< IO >::data_.

Referenced by TritonData< IO >::toServer().

dimProduct()

template<typename IO>

int64_t TritonData< IO >::dimProduct ( const ShapeView &  vec ) const

inlineprivate

Definition at line 104 of file TritonData.h.

                                                 {
    return std::accumulate(vec.begin(), vec.end(), 1, std::multiplies<int64_t>());
  }

References edm::Span< T >::begin(), and edm::Span< T >::end().

Referenced by TritonData< IO >::sizeShape().

dname()

template<typename IO>

const std::string& TritonData< IO >::dname ( ) const

inline

Definition at line 71 of file TritonData.h.

{ return dname_; }

References TritonData< IO >::dname_.

fromServer()

template<typename DT >

TritonOutput< DT > TritonOutputData::fromServer

(

)

const

Definition at line 206 of file TritonData.cc.

                                                    {
  //shouldn't be called twice
  if (done_)
    throw cms::Exception("TritonDataError") << name_ << " fromServer() was already called for this event";
 
  if (!result_) {
    throw cms::Exception("TritonDataError") << name_ << " fromServer(): missing result";
  }
 
  if (byteSize_ != sizeof(DT)) {
    throw cms::Exception("TritonDataError") << name_ << " fromServer(): inconsistent byte size " << sizeof(DT)
                                            << " (should be " << byteSize_ << " for " << dname_ << ")";
  }
 
  const uint8_t* r0 = memResource_->copyOutput();
  const DT* r1 = reinterpret_cast<const DT*>(r0);
 
  TritonOutput<DT> dataOut;
  dataOut.reserve(batchSize_);
  for (unsigned i0 = 0; i0 < batchSize_; ++i0) {
    auto offset = i0 * sizeShape_;
    dataOut.emplace_back(r1 + offset, r1 + offset + sizeShape_);
  }
 
  done_ = true;
  return dataOut;
}

References TritonData< IO >::batchSize_, TritonData< IO >::byteSize_, TritonData< IO >::dname_, TritonData< IO >::done_, GeomDetEnumerators::DT, Exception, TritonData< IO >::memResource_, TritonData< IO >::name_, hltrates_dqm_sourceclient-live_cfg::offset, diffTwoXMLs::r1, TritonData< IO >::result_, and TritonData< IO >::sizeShape_.

fullLoc()

template<typename IO>

unsigned TritonData< IO >::fullLoc ( unsigned  loc ) const

inlineprivate

Definition at line 90 of file TritonData.h.

{ return loc + (noBatch_ ? 0 : 1); }

References TritonData< IO >::noBatch_.

Referenced by TritonData< IO >::reset().

prepare()

void TritonOutputData::prepare

(

)

Definition at line 198 of file TritonData.cc.

                               {
  computeSizes();
  updateMem(totalByteSize_);
  memResource_->set();
}

References TritonData< IO >::computeSizes(), TritonData< IO >::memResource_, TritonData< IO >::totalByteSize_, and TritonData< IO >::updateMem().

reset()

void TritonOutputData::reset ( void  )

private

Definition at line 235 of file TritonData.cc.

                            {
  done_ = false;
  holder_.reset();
  data_->Reset();
  //reset shape
  if (variableDims_) {
    for (unsigned i = 0; i < shape_.size(); ++i) {
      unsigned locFull = fullLoc(i);
      fullShape_[locFull] = dims_[locFull];
    }
  }
  resetSizes();
}

References TritonData< IO >::data_, TritonData< IO >::dims_, TritonData< IO >::done_, TritonData< IO >::fullLoc(), TritonData< IO >::fullShape_, TritonData< IO >::holder_, mps_fire::i, TritonData< IO >::resetSizes(), TritonData< IO >::shape_, edm::Span< T >::size(), and TritonData< IO >::variableDims_.

resetSizes()

template<typename IO >

void TritonData< IO >::resetSizes ( )

private

Definition at line 112 of file TritonData.cc.

                                {
  sizeShape_ = 0;
  byteSizePerBatch_ = 0;
  totalByteSize_ = 0;
}

Referenced by TritonData< IO >::reset().

setBatchSize()

template<typename IO >

void TritonData< IO >::setBatchSize ( unsigned  bsize )

private

Definition at line 99 of file TritonData.cc.

                                                {
  batchSize_ = bsize;
  if (!noBatch_)
    fullShape_[0] = batchSize_;
}

setResult()

template<typename IO>

void TritonData< IO >::setResult ( std::shared_ptr< Result >  result )

inlineprivate

Definition at line 93 of file TritonData.h.

{ result_ = result; }

References mps_fire::result, and TritonData< IO >::result_.

setShape() [1/2]

template<typename IO>

void TritonData< IO >::setShape

(

const ShapeType &

newShape

)

setShape() [2/2]

template<typename IO >

void TritonData< IO >::setShape	(	unsigned	loc,
		int64_t	val
	)

Definition at line 81 of file TritonData.cc.

                                                       {
  unsigned locFull = fullLoc(loc);
 
  //check boundary
  if (locFull >= fullShape_.size())
    throw cms::Exception("TritonDataError")
        << name_ << " setShape(): dimension " << locFull << " out of bounds (" << fullShape_.size() << ")";
 
  if (val != fullShape_[locFull]) {
    if (dims_[locFull] == -1)
      fullShape_[locFull] = val;
    else
      throw cms::Exception("TritonDataError")
          << name_ << " setShape(): attempt to change value of non-variable shape dimension " << loc;
  }
}

References Exception, and heppy_batch::val.

shape()

template<typename IO>

const ShapeView& TritonData< IO >::shape ( void  ) const

inline

Definition at line 69 of file TritonData.h.

{ return shape_; }

References TritonData< IO >::shape_.

sizeDims()

template<typename IO>

int64_t TritonData< IO >::sizeDims ( ) const

inline

Definition at line 76 of file TritonData.h.

{ return productDims_; }

References TritonData< IO >::productDims_.

Referenced by TritonData< IO >::sizeShape().

sizeShape()

template<typename IO>

int64_t TritonData< IO >::sizeShape ( ) const

inline

Definition at line 78 of file TritonData.h.

{ return variableDims_ ? dimProduct(shape_) : sizeDims(); }

References TritonData< IO >::dimProduct(), TritonData< IO >::shape_, TritonData< IO >::sizeDims(), and TritonData< IO >::variableDims_.

toServer()

template<typename DT >

void TritonInputData::toServer

(

TritonInputContainer< DT >

ptr

)

Definition at line 164 of file TritonData.cc.

                                                           {
  //shouldn't be called twice
  if (done_)
    throw cms::Exception("TritonDataError") << name_ << " toServer() was already called for this event";
 
  const auto& data_in = *ptr;
 
  //check batch size
  if (data_in.size() != batchSize_) {
    throw cms::Exception("TritonDataError") << name_ << " toServer(): input vector has size " << data_in.size()
                                            << " but specified batch size is " << batchSize_;
  }
 
  //shape must be specified for variable dims or if batch size changes
  data_->SetShape(fullShape_);
 
  if (byteSize_ != sizeof(DT))
    throw cms::Exception("TritonDataError") << name_ << " toServer(): inconsistent byte size " << sizeof(DT)
                                            << " (should be " << byteSize_ << " for " << dname_ << ")";
 
  computeSizes();
  updateMem(totalByteSize_);
  for (unsigned i0 = 0; i0 < batchSize_; ++i0) {
    memResource_->copyInput(data_in[i0].data(), i0 * byteSizePerBatch_);
  }
  memResource_->set();
 
  //keep input data in scope
  holder_ = ptr;
  done_ = true;
}

References TritonData< IO >::batchSize_, TritonData< IO >::byteSize_, TritonData< IO >::byteSizePerBatch_, TritonData< IO >::computeSizes(), TritonData< IO >::data(), TritonData< IO >::data_, TritonData< IO >::dname_, TritonData< IO >::done_, GeomDetEnumerators::DT, Exception, TritonData< IO >::fullShape_, TritonData< IO >::holder_, TritonData< IO >::memResource_, TritonData< IO >::name_, TritonData< IO >::totalByteSize_, and TritonData< IO >::updateMem().

uid()

template<typename IO>

unsigned TritonData< IO >::uid ( ) const

inlineprivate

Definition at line 109 of file TritonData.h.

                       {
    static std::atomic<unsigned> uid{0};
    return ++uid;
  }

updateMem()

template<typename IO >

void TritonData< IO >::updateMem ( size_t  size )

private

Definition at line 121 of file TritonData.cc.

                                          {
  if (!memResource_ or size > memResource_->size()) {
    if (useShm_ and client_->serverType() == TritonServerType::LocalCPU) {
      //avoid unnecessarily throwing in destructor
      if (memResource_)
        memResource_->close();
      //need to destroy before constructing new instance because shared memory key will be reused
      memResource_.reset();
      memResource_ = std::make_shared<TritonCpuShmResource<IO>>(this, shmName_, size);
    }
#ifdef TRITON_ENABLE_GPU
    else if (useShm_ and client_->serverType() == TritonServerType::LocalGPU) {
      //avoid unnecessarily throwing in destructor
      if (memResource_)
        memResource_->close();
      //need to destroy before constructing new instance because shared memory key will be reused
      memResource_.reset();
      memResource_ = std::make_shared<TritonGpuShmResource<IO>>(this, shmName_, size);
    }
#endif
    //for remote/heap, size increases don't matter
    else if (!memResource_)
      memResource_ = std::make_shared<TritonHeapResource<IO>>(this, shmName_, size);
  }
}

References LocalCPU, LocalGPU, or, and findQualityFiles::size.

Referenced by TritonData< IO >::prepare(), and TritonData< IO >::toServer().

variableDims()

template<typename IO>

bool TritonData< IO >::variableDims ( ) const

inline

Definition at line 75 of file TritonData.h.

{ return variableDims_; }

References TritonData< IO >::variableDims_.

xput()

std::string TritonOutputData::xput ( ) const

private

Definition at line 58 of file TritonData.cc.

                                      {
  return "input";
}

Friends And Related Function Documentation

TritonClient

template<typename IO>

friend class TritonClient

friend

Definition at line 81 of file TritonData.h.

TritonCpuShmResource< IO >

template<typename IO>

friend class TritonCpuShmResource< IO >

friend

Definition at line 84 of file TritonData.h.

TritonHeapResource< IO >

template<typename IO>

friend class TritonHeapResource< IO >

friend

Definition at line 83 of file TritonData.h.

TritonMemResource< IO >

template<typename IO>

friend class TritonMemResource< IO >

friend

Definition at line 82 of file TritonData.h.

Member Data Documentation

batchSize_

template<typename IO>

unsigned TritonData< IO >::batchSize_

private

Definition at line 123 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::batchSize(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

byteSize_

template<typename IO>

int64_t TritonData< IO >::byteSize_

private

Definition at line 130 of file TritonData.h.

Referenced by TritonData< IO >::byteSize(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

byteSizePerBatch_

template<typename IO>

size_t TritonData< IO >::byteSizePerBatch_

private

Definition at line 132 of file TritonData.h.

Referenced by TritonData< IO >::toServer().

client_

template<typename IO>

TritonClient* TritonData< IO >::client_

private

Definition at line 118 of file TritonData.h.

data_

template<typename IO>

std::shared_ptr<IO> TritonData< IO >::data_

private

Definition at line 117 of file TritonData.h.

Referenced by TritonData< IO >::data(), TritonData< IO >::reset(), TritonData< IO >::toServer(), and TritonData< IO >::TritonData().

dims_

template<typename IO>

const ShapeType TritonData< IO >::dims_

private

Definition at line 121 of file TritonData.h.

Referenced by TritonData< IO >::reset().

dname_

template<typename IO>

std::string TritonData< IO >::dname_

private

Definition at line 128 of file TritonData.h.

Referenced by TritonData< IO >::dname(), TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

done_

template<typename IO>

bool TritonData< IO >::done_ {}

mutableprivate

Definition at line 141 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), TritonData< IO >::reset(), and TritonData< IO >::toServer().

dtype_

template<typename IO>

inference::DataType TritonData< IO >::dtype_

private

Definition at line 129 of file TritonData.h.

fullShape_

template<typename IO>

ShapeType TritonData< IO >::fullShape_

private

Definition at line 124 of file TritonData.h.

Referenced by TritonData< IO >::reset(), and TritonData< IO >::toServer().

holder_

template<typename IO>

std::shared_ptr<void> TritonData< IO >::holder_

private

Definition at line 137 of file TritonData.h.

Referenced by TritonData< IO >::reset(), and TritonData< IO >::toServer().

memResource_

template<typename IO>

std::shared_ptr<TritonMemResource<IO> > TritonData< IO >::memResource_

private

Definition at line 138 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), TritonData< IO >::prepare(), and TritonData< IO >::toServer().

name_

template<typename IO>

std::string TritonData< IO >::name_

private

Definition at line 116 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), and TritonData< IO >::toServer().

noBatch_

template<typename IO>

bool TritonData< IO >::noBatch_

private

Definition at line 122 of file TritonData.h.

Referenced by TritonData< IO >::fullLoc().

productDims_

template<typename IO>

int64_t TritonData< IO >::productDims_

private

Definition at line 127 of file TritonData.h.

Referenced by TritonData< IO >::sizeDims().

result_

template<typename IO>

std::shared_ptr<Result> TritonData< IO >::result_

private

Definition at line 139 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), and TritonData< IO >::setResult().

shape_

template<typename IO>

ShapeView TritonData< IO >::shape_

private

Definition at line 125 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::reset(), TritonData< IO >::shape(), and TritonData< IO >::sizeShape().

shmName_

template<typename IO>

std::string TritonData< IO >::shmName_

private

Definition at line 120 of file TritonData.h.

sizeShape_

template<typename IO>

size_t TritonData< IO >::sizeShape_

private

Definition at line 131 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), and TritonData< IO >::fromServer().

totalByteSize_

template<typename IO>

size_t TritonData< IO >::totalByteSize_

private

Definition at line 133 of file TritonData.h.

Referenced by TritonData< IO >::prepare(), and TritonData< IO >::toServer().

useShm_

template<typename IO>

bool TritonData< IO >::useShm_

private

Definition at line 119 of file TritonData.h.

variableDims_

template<typename IO>

bool TritonData< IO >::variableDims_

private

Definition at line 126 of file TritonData.h.

Referenced by TritonData< IO >::reset(), TritonData< IO >::sizeShape(), and TritonData< IO >::variableDims().