TritonData< IO > Class Template Reference

#include <TritonData.h>

Classes
class	TritonDataEntry

Public Types
using	Result = triton::client::InferResult
using	ShapeType = std::vector< int64_t >
using	ShapeView = edm::Span< ShapeType::const_iterator >
using	TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Public Member Functions
template<typename DT >
TritonInputContainer< DT >	allocate (bool reserve=true)
int64_t	byteSize () const
const std::string &	dname () const
template<typename DT >
TritonOutput< DT >	fromServer () const
void	prepare ()
void	setShape (const ShapeType &newShape, unsigned entry=0)
void	setShape (unsigned loc, int64_t val, unsigned entry=0)
const ShapeView &	shape (unsigned entry=0) const
int64_t	sizeDims () const
int64_t	sizeShape (unsigned entry=0) const
template<typename DT >
void	toServer (TritonInputContainer< DT > ptr)
	TritonData (const std::string &name, const TensorMetadata &model_info, TritonClient *client, const std::string &pid)
bool	variableDims () const

Private Member Functions
void	addEntry (unsigned entry)
void	addEntryImpl (unsigned entry)
bool	anyNeg (const ShapeView &vec) const
void	checkShm ()
template<typename DT >
void	checkType () const
triton::client::InferenceServerGrpcClient *	client ()
void	computeSizes ()
IO *	data (unsigned entry=0)
int64_t	dimProduct (const ShapeView &vec) const
unsigned	fullLoc (unsigned loc) const
void	reset ()
void	setResult (std::shared_ptr< Result > result, unsigned entry=0)
unsigned	uid () const
void	updateMem (size_t size)
std::string	xput () const

Private Attributes
int64_t	byteSize_
TritonClient *	client_
const ShapeType	dims_
std::string	dname_
bool	done_ {}
inference::DataType	dtype_
std::vector< TritonDataEntry >	entries_
std::shared_ptr< void >	holder_
std::shared_ptr< TritonMemResource< IO > >	memResource_
std::string	name_
int64_t	productDims_
std::string	shmName_
size_t	totalByteSize_
bool	useShm_
bool	variableDims_

Friends
class	TritonClient
class	TritonCpuShmResource< IO >
class	TritonHeapResource< IO >
class	TritonMemResource< IO >

Detailed Description

template<typename IO>
class TritonData< IO >

Definition at line 47 of file TritonData.h.

Member Typedef Documentation

Result

template<typename IO>

using TritonData< IO >::Result = triton::client::InferResult

Definition at line 49 of file TritonData.h.

ShapeType

template<typename IO>

using TritonData< IO >::ShapeType = std::vector<int64_t>

Definition at line 51 of file TritonData.h.

ShapeView

template<typename IO>

using TritonData< IO >::ShapeView = edm::Span<ShapeType::const_iterator>

Definition at line 52 of file TritonData.h.

TensorMetadata

template<typename IO>

using TritonData< IO >::TensorMetadata = inference::ModelMetadataResponse_TensorMetadata

Definition at line 50 of file TritonData.h.

Constructor & Destructor Documentation

TritonData()

template<typename IO>

TritonData< IO >::TritonData	(	const std::string &	name,
		const TensorMetadata &	model_info,
		TritonClient *	client,
		const std::string &	pid
	)

Definition at line 18 of file TritonData.cc.

References TritonData< IO >::addEntryImpl(), TritonData< IO >::anyNeg(), TritonData< IO >::checkShm(), TritonData< IO >::dimProduct(), TritonData< IO >::entries_, TritonData< IO >::productDims_, and TritonData< IO >::variableDims_.

    : name_(name),
      client_(client),
      useShm_(client_->useSharedMemory()),
      //ensure unique name for shared memory region
      shmName_(useShm_ ? pid + "_" + xput() + std::to_string(uid()) : ""),
      dims_(model_info.shape().begin(), model_info.shape().end()),
      dname_(model_info.datatype()),
      dtype_(tco::ProtocolStringToDataType(dname_)),
      byteSize_(tco::GetDataTypeByteSize(dtype_)),
      totalByteSize_(0) {
  //initialize first shape entry
  addEntryImpl(0);
  //one-time computation of some shape info
  variableDims_ = anyNeg(entries_.front().shape_);
  productDims_ = variableDims_ ? -1 : dimProduct(entries_.front().shape_);
  checkShm();
}

Member Function Documentation

addEntry()

template<typename IO >

void TritonData< IO >::addEntry ( unsigned  entry )

private

Definition at line 47 of file TritonData.cc.

References mps_splice::entry.

Referenced by TritonData< IO >::setShape().

                                            {
  //ensures consistency among all inputs
  client_->addEntry(entry);
}

addEntryImpl()

template<typename IO >

void TritonData< IO >::addEntryImpl ( unsigned  entry )

private

Definition at line 53 of file TritonData.cc.

References mps_splice::entry, and mps_fire::i.

Referenced by TritonData< IO >::reset(), and TritonData< IO >::TritonData().

                                                {
  if (entry >= entries_.size()) {
    entries_.reserve(entry + 1);
    for (unsigned i = entries_.size(); i < entry + 1; ++i) {
      entries_.emplace_back(dims_, client_->noOuterDim(), name_, dname_);
    }
  }
}

allocate()

template<typename DT >

TritonInputContainer< DT > TritonInputData::allocate

(

bool

reserve = true

)

Definition at line 170 of file TritonData.cc.

References TritonData< IO >::anyNeg(), TritonClient::batchSize(), TritonData< IO >::client_, TritonData< IO >::computeSizes(), TritonData< IO >::entries_, and mps_splice::entry.

                                                               {
  //automatically creates a vector for each item (if batch size known)
  auto ptr = std::make_shared<TritonInput<DT>>(client_->batchSize());
  if (reserve) {
    computeSizes();
    for (auto& entry : entries_) {
      if (anyNeg(entry.shape_))
        continue;
      for (auto& vec : *ptr) {
        vec.reserve(entry.sizeShape_);
      }
    }
  }
  return ptr;
}

anyNeg()

template<typename IO>

bool TritonData< IO >::anyNeg ( const ShapeView &  vec ) const

inlineprivate

Definition at line 160 of file TritonData.h.

References edm::Span< T >::begin(), edm::Span< T >::end(), and mps_fire::i.

Referenced by TritonData< IO >::allocate(), and TritonData< IO >::TritonData().

                                          {
    return std::any_of(vec.begin(), vec.end(), [](int64_t i) { return i < 0; });
  }

byteSize()

template<typename IO>

int64_t TritonData< IO >::byteSize ( ) const

inline

Definition at line 72 of file TritonData.h.

References TritonData< IO >::byteSize_.

Referenced by TritonData< IO >::TritonDataEntry::computeSizes().

{ return byteSize_; }

checkShm()

void TritonOutputData::checkShm ( )

inlineprivate

Definition at line 144 of file TritonData.h.

Referenced by TritonData< IO >::TritonData().

{}

checkType()

template<typename IO>

template<typename DT >

void TritonData< IO >::checkType ( ) const

inlineprivate

Definition at line 153 of file TritonData.h.

References TritonData< IO >::dname_, GeomDetEnumerators::DT, TritonData< IO >::dtype_, Exception, Skims_PA_cff::name, and TritonData< IO >::name_.

                         {
    if (!triton_utils::checkType<DT>(dtype_))
      throw cms::Exception("TritonDataError")
          << name_ << ": inconsistent data type " << typeid(DT).name() << " for " << dname_;
  }

client()

template<typename IO >

tc::InferenceServerGrpcClient * TritonData< IO >::client ( )

private

Definition at line 87 of file TritonData.cc.

References TritonClient::client(), and TritonData< IO >::client_.

                                                    {
  return client_->client();
}

computeSizes()

template<typename IO >

void TritonData< IO >::computeSizes ( )

private

Definition at line 128 of file TritonData.cc.

References TritonData< IO >::byteSize_, TritonData< IO >::client_, TritonData< IO >::entries_, mps_fire::i, TritonClient::outerDim(), TritonData< IO >::sizeShape(), and TritonData< IO >::totalByteSize_.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::prepare(), and TritonData< IO >::toServer().

                                  {
  totalByteSize_ = 0;
  unsigned outerDim = client_->outerDim();
  for (unsigned i = 0; i < entries_.size(); ++i) {
    entries_[i].computeSizes(sizeShape(i), byteSize_, outerDim);
    entries_[i].offset_ = totalByteSize_;
    totalByteSize_ += entries_[i].totalByteSize_;
  }
}

data()

template<typename IO>

IO* TritonData< IO >::data ( unsigned  entry = 0 )

inlineprivate

Definition at line 148 of file TritonData.h.

References TritonData< IO >::entries_, and mps_splice::entry.

Referenced by TritonData< IO >::toServer().

{ return entries_[entry].data_.get(); }

dimProduct()

template<typename IO>

int64_t TritonData< IO >::dimProduct ( const ShapeView &  vec ) const

inlineprivate

Definition at line 163 of file TritonData.h.

References edm::Span< T >::begin(), edm::Span< T >::end(), MainPageGenerator::l, and SiStripPI::max.

Referenced by TritonData< IO >::sizeShape(), and TritonData< IO >::TritonData().

                                                 {
    //lambda treats negative dimensions as 0 to avoid overflows
    return std::accumulate(
        vec.begin(), vec.end(), 1, [](int64_t dim1, int64_t dim2) { return dim1 * std::max(0l, dim2); });
  }

dname()

template<typename IO>

const std::string& TritonData< IO >::dname ( ) const

inline

Definition at line 73 of file TritonData.h.

References TritonData< IO >::dname_.

Referenced by TritonData< IO >::TritonDataEntry::TritonDataEntry().

{ return dname_; }

fromServer()

template<typename DT >

TritonOutput< DT > TritonOutputData::fromServer

(

)

const

Definition at line 244 of file TritonData.cc.

References TritonClient::batchSize(), TritonData< IO >::client_, TritonData< IO >::done_, GeomDetEnumerators::DT, TritonData< IO >::entries_, mps_splice::entry, Exception, mps_fire::i, TritonData< IO >::memResource_, TritonData< IO >::name_, hltrates_dqm_sourceclient-live_cfg::offset, TritonClient::outerDim(), and diffTwoXMLs::r1.

                                                    {
  //shouldn't be called twice
  if (done_)
    throw cms::Exception("TritonDataError") << name_ << " fromServer() was already called for this event";

  //check type
  checkType<DT>();

  memResource_->copyOutput();

  unsigned outerDim = client_->outerDim();
  TritonOutput<DT> dataOut;
  dataOut.reserve(client_->batchSize());
  for (unsigned i = 0; i < entries_.size(); ++i) {
    const auto& entry = entries_[i];
    const DT* r1 = reinterpret_cast<const DT*>(entry.output_);

    if (entry.totalByteSize_ > 0 and !entry.result_) {
      throw cms::Exception("TritonDataError") << name_ << " fromServer(): missing result";
    }

    for (unsigned i0 = 0; i0 < outerDim; ++i0) {
      auto offset = i0 * entry.sizeShape_;
      dataOut.emplace_back(r1 + offset, r1 + offset + entry.sizeShape_);
    }
  }

  done_ = true;
  return dataOut;
}

fullLoc()

template<typename IO >

unsigned TritonData< IO >::fullLoc ( unsigned  loc ) const

private

Definition at line 286 of file TritonData.cc.

References TritonData< IO >::client_, and TritonClient::noOuterDim().

Referenced by TritonData< IO >::setShape().

                                                   {
  return loc + (client_->noOuterDim() ? 0 : 1);
}

prepare()

void TritonOutputData::prepare

(

)

Definition at line 236 of file TritonData.cc.

References TritonData< IO >::computeSizes(), TritonData< IO >::memResource_, TritonData< IO >::totalByteSize_, and TritonData< IO >::updateMem().

                               {
  computeSizes();
  updateMem(totalByteSize_);
  memResource_->set();
}

reset()

template<typename IO >

void TritonData< IO >::reset ( void  )

private

Definition at line 276 of file TritonData.cc.

References TritonData< IO >::addEntryImpl(), TritonData< IO >::done_, TritonData< IO >::entries_, TritonData< IO >::holder_, and TritonData< IO >::totalByteSize_.

                           {
  done_ = false;
  holder_.reset();
  entries_.clear();
  totalByteSize_ = 0;
  //re-initialize first shape entry
  addEntryImpl(0);
}

setResult()

template<typename IO>

void TritonData< IO >::setResult ( std::shared_ptr< Result >  result, unsigned  entry = 0  )

inlineprivate

Definition at line 147 of file TritonData.h.

References TritonData< IO >::entries_, mps_splice::entry, and mps_fire::result.

{ entries_[entry].result_ = result; }

setShape() [1/2]

template<typename IO>

void TritonData< IO >::setShape	(	const ShapeType &	newShape,
		unsigned	entry = 0
	)

setShape() [2/2]

template<typename IO >

void TritonData< IO >::setShape	(	unsigned	loc,
		int64_t	val,
		unsigned	entry = 0
	)

Definition at line 101 of file TritonData.cc.

References TritonData< IO >::addEntry(), TritonData< IO >::dims_, TritonData< IO >::entries_, mps_splice::entry, Exception, TritonData< IO >::fullLoc(), TritonData< IO >::name_, and heppy_batch::val.

                                                                       {
  addEntry(entry);

  unsigned locFull = fullLoc(loc);

  //check boundary
  if (locFull >= entries_[entry].fullShape_.size())
    throw cms::Exception("TritonDataError") << name_ << " setShape(): dimension " << locFull << " out of bounds ("
                                            << entries_[entry].fullShape_.size() << ")";

  if (val != entries_[entry].fullShape_[locFull]) {
    if (dims_[locFull] == -1)
      entries_[entry].fullShape_[locFull] = val;
    else
      throw cms::Exception("TritonDataError")
          << name_ << " setShape(): attempt to change value of non-variable shape dimension " << loc;
  }
}

shape()

template<typename IO>

const ShapeView& TritonData< IO >::shape ( unsigned  entry = 0 ) const

inline

Definition at line 71 of file TritonData.h.

References TritonData< IO >::entries_, and mps_splice::entry.

{ return entries_.at(entry).shape_; }

sizeDims()

template<typename IO>

int64_t TritonData< IO >::sizeDims ( ) const

inline

Definition at line 77 of file TritonData.h.

References TritonData< IO >::productDims_.

Referenced by TritonData< IO >::sizeShape().

{ return productDims_; }

sizeShape()

template<typename IO>

int64_t TritonData< IO >::sizeShape ( unsigned  entry = 0 ) const

inline

Definition at line 79 of file TritonData.h.

References TritonData< IO >::dimProduct(), TritonData< IO >::entries_, mps_splice::entry, TritonData< IO >::sizeDims(), and TritonData< IO >::variableDims_.

Referenced by TritonData< IO >::computeSizes().

                                              {
    return variableDims_ ? dimProduct(entries_.at(entry).shape_) : sizeDims();
  }

toServer()

template<typename DT >

void TritonInputData::toServer

(

TritonInputContainer< DT >

ptr

)

Definition at line 188 of file TritonData.cc.

References TritonClient::batchSize(), HLT_FULL_cff::batchSize, TritonData< IO >::client_, TritonData< IO >::computeSizes(), counter, TritonData< IO >::data(), TritonData< IO >::done_, TritonData< IO >::entries_, mps_splice::entry, Exception, TritonData< IO >::holder_, mps_fire::i, TritonData< IO >::memResource_, TritonData< IO >::name_, TritonClient::noOuterDim(), hltrates_dqm_sourceclient-live_cfg::offset, TritonClient::outerDim(), TritonData< IO >::totalByteSize_, and TritonData< IO >::updateMem().

                                                           {
  //shouldn't be called twice
  if (done_)
    throw cms::Exception("TritonDataError") << name_ << " toServer() was already called for this event";

  const auto& data_in = *ptr;

  //check batch size
  unsigned batchSize = client_->batchSize();
  unsigned outerDim = client_->outerDim();
  if (data_in.size() != batchSize) {
    throw cms::Exception("TritonDataError") << name_ << " toServer(): input vector has size " << data_in.size()
                                            << " but specified batch size is " << batchSize;
  }

  //check type
  checkType<DT>();

  computeSizes();
  updateMem(totalByteSize_);

  unsigned offset = 0;
  unsigned counter = 0;
  for (unsigned i = 0; i < entries_.size(); ++i) {
    auto& entry = entries_[i];

    //shape must be specified for variable dims or if batch size changes
    if (!client_->noOuterDim())
      entry.fullShape_[0] = outerDim;
    entry.data_->SetShape(entry.fullShape_);

    for (unsigned i0 = 0; i0 < outerDim; ++i0) {
      //avoid copying empty input
      if (entry.byteSizePerBatch_ > 0)
        memResource_->copyInput(data_in[counter].data(), offset, i);
      offset += entry.byteSizePerBatch_;
      ++counter;
    }
  }
  memResource_->set();

  //keep input data in scope
  holder_ = ptr;
  done_ = true;
}

uid()

template<typename IO>

unsigned TritonData< IO >::uid ( ) const

inlineprivate

Definition at line 169 of file TritonData.h.

                       {
    static std::atomic<unsigned> uid{0};
    return ++uid;
  }

updateMem()

template<typename IO >

void TritonData< IO >::updateMem ( size_t  size )

private

Definition at line 141 of file TritonData.cc.

References TritonData< IO >::client_, LocalCPU, LocalGPU, TritonData< IO >::memResource_, or, TritonClient::serverType(), TritonData< IO >::shmName_, findQualityFiles::size, and TritonData< IO >::useShm_.

Referenced by TritonData< IO >::prepare(), and TritonData< IO >::toServer().

                                          {
  if (!memResource_ or size > memResource_->size()) {
    if (useShm_ and client_->serverType() == TritonServerType::LocalCPU) {
      //avoid unnecessarily throwing in destructor
      if (memResource_)
        memResource_->close();
      //need to destroy before constructing new instance because shared memory key will be reused
      memResource_.reset();
      memResource_ = std::make_shared<TritonCpuShmResource<IO>>(this, shmName_, size);
    }
#ifdef TRITON_ENABLE_GPU
    else if (useShm_ and client_->serverType() == TritonServerType::LocalGPU) {
      //avoid unnecessarily throwing in destructor
      if (memResource_)
        memResource_->close();
      //need to destroy before constructing new instance because shared memory key will be reused
      memResource_.reset();
      memResource_ = std::make_shared<TritonGpuShmResource<IO>>(this, shmName_, size);
    }
#endif
    //for remote/heap, size increases don't matter
    else if (!memResource_)
      memResource_ = std::make_shared<TritonHeapResource<IO>>(this, shmName_, size);
  }
}

variableDims()

template<typename IO>

bool TritonData< IO >::variableDims ( ) const

inline

Definition at line 76 of file TritonData.h.

References TritonData< IO >::variableDims_.

{ return variableDims_; }

xput()

std::string TritonOutputData::xput ( ) const

private

Definition at line 77 of file TritonData.cc.

                                      {
  return "input";
}

Friends And Related Function Documentation

TritonClient

template<typename IO>

friend class TritonClient

friend

Definition at line 84 of file TritonData.h.

TritonCpuShmResource< IO >

template<typename IO>

friend class TritonCpuShmResource< IO >

friend

Definition at line 87 of file TritonData.h.

TritonHeapResource< IO >

template<typename IO>

friend class TritonHeapResource< IO >

friend

Definition at line 86 of file TritonData.h.

TritonMemResource< IO >

template<typename IO>

friend class TritonMemResource< IO >

friend

Definition at line 85 of file TritonData.h.

Member Data Documentation

byteSize_

template<typename IO>

int64_t TritonData< IO >::byteSize_

private

Definition at line 187 of file TritonData.h.

Referenced by TritonData< IO >::byteSize(), and TritonData< IO >::computeSizes().

client_

template<typename IO>

TritonClient* TritonData< IO >::client_

private

Definition at line 179 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::client(), TritonData< IO >::computeSizes(), TritonData< IO >::fromServer(), TritonData< IO >::fullLoc(), TritonData< IO >::toServer(), and TritonData< IO >::updateMem().

dims_

template<typename IO>

const ShapeType TritonData< IO >::dims_

private

Definition at line 182 of file TritonData.h.

Referenced by TritonData< IO >::setShape().

dname_

template<typename IO>

std::string TritonData< IO >::dname_

private

Definition at line 185 of file TritonData.h.

Referenced by TritonData< IO >::checkType(), and TritonData< IO >::dname().

done_

template<typename IO>

bool TritonData< IO >::done_ {}

mutableprivate

Definition at line 196 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), TritonData< IO >::reset(), and TritonData< IO >::toServer().

dtype_

template<typename IO>

inference::DataType TritonData< IO >::dtype_

private

Definition at line 186 of file TritonData.h.

Referenced by TritonData< IO >::checkType().

entries_

template<typename IO>

std::vector<TritonDataEntry> TritonData< IO >::entries_

private

Definition at line 188 of file TritonData.h.

Referenced by TritonData< IO >::allocate(), TritonData< IO >::computeSizes(), TritonData< IO >::data(), TritonData< IO >::fromServer(), TritonData< IO >::reset(), TritonData< IO >::setResult(), TritonData< IO >::setShape(), TritonData< IO >::shape(), TritonData< IO >::sizeShape(), TritonData< IO >::toServer(), and TritonData< IO >::TritonData().

holder_

template<typename IO>

std::shared_ptr<void> TritonData< IO >::holder_

private

Definition at line 193 of file TritonData.h.

Referenced by TritonData< IO >::reset(), and TritonData< IO >::toServer().

memResource_

template<typename IO>

std::shared_ptr<TritonMemResource<IO> > TritonData< IO >::memResource_

private

Definition at line 194 of file TritonData.h.

Referenced by TritonData< IO >::fromServer(), TritonData< IO >::prepare(), TritonData< IO >::toServer(), and TritonData< IO >::updateMem().

name_

template<typename IO>

std::string TritonData< IO >::name_

private

Definition at line 178 of file TritonData.h.

Referenced by TritonData< IO >::checkType(), TritonData< IO >::fromServer(), TritonData< IO >::setShape(), and TritonData< IO >::toServer().

productDims_

template<typename IO>

int64_t TritonData< IO >::productDims_

private

Definition at line 184 of file TritonData.h.

Referenced by TritonData< IO >::sizeDims(), and TritonData< IO >::TritonData().

shmName_

template<typename IO>

std::string TritonData< IO >::shmName_

private

Definition at line 181 of file TritonData.h.

Referenced by TritonData< IO >::updateMem().

totalByteSize_

template<typename IO>

size_t TritonData< IO >::totalByteSize_

private

Definition at line 189 of file TritonData.h.

Referenced by TritonData< IO >::TritonDataEntry::computeSizes(), TritonData< IO >::computeSizes(), TritonData< IO >::prepare(), TritonData< IO >::reset(), and TritonData< IO >::toServer().

useShm_

template<typename IO>

bool TritonData< IO >::useShm_

private

Definition at line 180 of file TritonData.h.

Referenced by TritonData< IO >::updateMem().

variableDims_

template<typename IO>

bool TritonData< IO >::variableDims_

private

Definition at line 183 of file TritonData.h.

Referenced by TritonData< IO >::sizeShape(), TritonData< IO >::TritonData(), and TritonData< IO >::variableDims().