HDF5ProductResolver Class Reference

#include "HDF5ProductResolver.h"

Inheritance diagram for HDF5ProductResolver:

Public Member Functions
	HDF5ProductResolver (edm::SerialTaskQueue *iQueue, std::unique_ptr< cond::serialization::SerializationHelperBase >, cms::h5::File const *iFile, std::string const &iFileName, cond::hdf5::Compression iCompression, cond::hdf5::Record const *iRecord, cond::hdf5::DataProduct const *iDataProduct)
	HDF5ProductResolver (const HDF5ProductResolver &)=delete
const HDF5ProductResolver &	operator= (const HDF5ProductResolver &)=delete
	~HDF5ProductResolver () override
Public Member Functions inherited from edm::eventsetup::ESSourceProductResolverBase
	ESSourceProductResolverBase ()
Public Member Functions inherited from edm::eventsetup::ESProductResolver
bool	cacheIsValid () const
	ESProductResolver ()
	ESProductResolver (ESProductResolver const &)=delete
void const *	getAfterPrefetch (const EventSetupRecordImpl &iRecord, const DataKey &iKey, bool iTransiently) const
virtual void	initializeForNewIOV ()
void	invalidate ()
ESProductResolver const &	operator= (ESProductResolver const &)=delete
void	prefetchAsync (WaitingTaskHolder, EventSetupRecordImpl const &, DataKey const &, EventSetupImpl const *, ServiceToken const &, ESParentContext const &) const noexcept
ComponentDescription const *	providerDescription () const
	returns the description of the ESProductResolverProvider which owns this Resolver More...
void	resetIfTransient ()
void	setProviderDescription (ComponentDescription const *iDesc)
virtual	~ESProductResolver ()

Private Member Functions
std::vector< char >	decompress_lzma (std::vector< char >, std::size_t iMemSize) const
std::vector< char >	decompress_zlib (std::vector< char >, std::size_t iMemSize) const
void const *	getAfterPrefetchImpl () const final
std::ptrdiff_t	indexForInterval (edm::ValidityInterval const &iIOV) const
void	invalidateCache () final
void	prefetch (edm::eventsetup::DataKey const &iKey, edm::EventSetupRecordDetails) final
void	prefetchAsyncImpl (edm::WaitingTaskHolder iTask, edm::eventsetup::EventSetupRecordImpl const &iES, edm::eventsetup::DataKey const &iKey, edm::EventSetupImpl const *, edm::ServiceToken const &, edm::ESParentContext const &) noexcept final
void	readFromHDF5api (std::ptrdiff_t iIndex)
void	threadFriendlyPrefetch (uint64_t iFileOffset, std::size_t iStorageSize, std::size_t iMemSize, const std::string &iType)

Private Attributes
cond::hdf5::Compression	compression_
cond::serialization::unique_void_ptr	data_
cond::hdf5::DataProduct const *	dataProduct_
std::exception_ptr	exceptPtr_
cms::h5::File const *	file_
std::string	fileName_
uint64_t	fileOffset_
std::unique_ptr< cond::serialization::SerializationHelperBase >	helper_
std::size_t	memSize_
edm::SerialTaskQueue *	queue_
cond::hdf5::Record const *	record_
std::size_t	storageSize_
std::string	type_

Additional Inherited Members
Protected Member Functions inherited from edm::eventsetup::ESSourceProductResolverBase
void	invalidateCache () override
void	invalidateTransientCache () override
template<typename ASYNC , typename GUARD >
void	prefetchAsyncImplTemplate (ASYNC iAsync, GUARD iGuardFactory, edm::WaitingTaskHolder iTask, edm::eventsetup::EventSetupRecordImpl const &iRecord, edm::eventsetup::DataKey const &iKey, edm::ESParentContext const &iContext) noexcept
Protected Member Functions inherited from edm::eventsetup::ESProductResolver
void	clearCacheIsValid ()

Detailed Description

Description: [one line class summary]

Usage: <usage>

Definition at line 36 of file HDF5ProductResolver.h.

Constructor & Destructor Documentation

HDF5ProductResolver() [1/2]

HDF5ProductResolver::HDF5ProductResolver	(	edm::SerialTaskQueue *	iQueue,
		std::unique_ptr< cond::serialization::SerializationHelperBase >	iHelper,
		cms::h5::File const *	iFile,
		std::string const &	iFileName,
		cond::hdf5::Compression	iCompression,
		cond::hdf5::Record const *	iRecord,
		cond::hdf5::DataProduct const *	iDataProduct
	)

Definition at line 44 of file HDF5ProductResolver.cc.

    : edm::eventsetup::ESSourceProductResolverBase(),
      queue_(iQueue),
      helper_(std::move(iHelper)),
      file_(iFile),
      fileName_(iFileName),
      record_(iRecord),
      dataProduct_(iDataProduct),
      compression_(iCompression) {}

~HDF5ProductResolver()

HDF5ProductResolver::~HDF5ProductResolver ( )

override

Definition at line 65 of file HDF5ProductResolver.cc.

{}

HDF5ProductResolver() [2/2]

HDF5ProductResolver::HDF5ProductResolver ( const HDF5ProductResolver &  )

delete

Member Function Documentation

decompress_lzma()

std::vector< char > HDF5ProductResolver::decompress_lzma ( std::vector< char >  compressedBuffer, std::size_t  iMemSize  ) const

private

Definition at line 178 of file HDF5ProductResolver.cc.

References edmScanValgrind::buffer, Exception, eostools::move(), cms::cuda::stream, and mitigatedMETSequence_cff::U.

Referenced by threadFriendlyPrefetch().

                                                                                                               {
  std::vector<char> buffer;
  if (iMemSize == compressedBuffer.size()) {
    //memory was not compressed
    //std::cout <<"NOT COMPRESSED"<<std::endl;
    buffer = std::move(compressedBuffer);
  } else {
    // code 'cribbed' from ROOT
    lzma_stream stream = LZMA_STREAM_INIT;

    auto returnStatus = lzma_stream_decoder(&stream, UINT64_MAX, 0U);
    if (returnStatus != LZMA_OK) {
      throw cms::Exception("H5CondFailedDecompress") << "failed to setup lzma";
    }

    stream.next_in = reinterpret_cast<uint8_t*>(compressedBuffer.data());
    stream.avail_in = compressedBuffer.size();

    buffer = std::vector<char>(iMemSize);
    stream.next_out = reinterpret_cast<uint8_t*>(buffer.data());
    stream.avail_out = buffer.size();

    returnStatus = lzma_code(&stream, LZMA_FINISH);
    lzma_end(&stream);
    if (returnStatus != LZMA_STREAM_END) {
      throw cms::Exception("H5CondFailedDecompress") << "failed to decompress buffer using lzma";
    }
  }
  return buffer;
}

decompress_zlib()

std::vector< char > HDF5ProductResolver::decompress_zlib ( std::vector< char >  compressedBuffer, std::size_t  iMemSize  ) const

private

Definition at line 145 of file HDF5ProductResolver.cc.

References cms::cuda::assert(), edmScanValgrind::buffer, eostools::move(), runTheMatrix::ret, and funct::void.

Referenced by threadFriendlyPrefetch().

                                                                                                               {
  std::vector<char> buffer;
  if (iMemSize == compressedBuffer.size()) {
    //memory was not compressed
    //std::cout <<"NOT COMPRESSED"<<std::endl;
    buffer = std::move(compressedBuffer);
  } else {
    //zlib compression was used
    z_stream strm;
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.avail_in = 0;
    strm.next_in = Z_NULL;
    auto ret = inflateInit(&strm);
    assert(ret == Z_OK);

    strm.avail_in = compressedBuffer.size();
    strm.next_in = reinterpret_cast<unsigned char*>(compressedBuffer.data());

    buffer = std::vector<char>(iMemSize);
    strm.avail_out = buffer.size();
    strm.next_out = reinterpret_cast<unsigned char*>(buffer.data());
    ret = inflate(&strm, Z_FINISH);
    assert(ret != Z_STREAM_ERROR);
    //if(ret != Z_STREAM_END) {std::cout <<"mem "<<memSize<<" "<<ret<<" out "<<strm.avail_out<<std::endl;}
    assert(ret == Z_STREAM_END);

    (void)inflateEnd(&strm);
  }
  return buffer;
}

getAfterPrefetchImpl()

void const * HDF5ProductResolver::getAfterPrefetchImpl ( ) const

finalprivatevirtual

used to retrieve the data from the implementation. The data is then cached locally.

Implements edm::eventsetup::ESProductResolver.

Definition at line 247 of file HDF5ProductResolver.cc.

References data_, and cond::serialization::unique_void_ptr::get().

{ return data_.get(); }

indexForInterval()

std::ptrdiff_t HDF5ProductResolver::indexForInterval ( edm::ValidityInterval const &  iIOV ) const

private

Definition at line 112 of file HDF5ProductResolver.cc.

References cms::cuda::assert(), cond::hdf5::convertSyncValue(), cond::hdf5::findMatchingFirst(), edm::ValidityInterval::first(), cond::hdf5::Record::iovFirsts_, cond::hdf5::Record::iovIsRunLumi_, and record_.

                                                                                        {
  using namespace cond::hdf5;
  auto firstSync = convertSyncValue(iIOV.first(), record_->iovIsRunLumi_);

  auto itFound = findMatchingFirst(record_->iovFirsts_, firstSync);
  assert(itFound != record_->iovFirsts_.end());

  return itFound - record_->iovFirsts_.begin();
}

invalidateCache()

void HDF5ProductResolver::invalidateCache ( )

finalprivatevirtual

indicates that the Resolver should invalidate any cached information as that information has 'expired' (i.e. we have moved to a new IOV)

Implements edm::eventsetup::ESProductResolver.

Definition at line 239 of file HDF5ProductResolver.cc.

References data_.

                                          {
  ESSourceProductResolverBase::invalidateCache();
  data_ = cond::serialization::unique_void_ptr();
}

operator=()

const HDF5ProductResolver& HDF5ProductResolver::operator= ( const HDF5ProductResolver &  )

delete

prefetch()

void HDF5ProductResolver::prefetch ( edm::eventsetup::DataKey const &  iKey, edm::EventSetupRecordDetails  iRecord  )

finalprivatevirtual

Implements edm::eventsetup::ESSourceProductResolverBase.

Definition at line 135 of file HDF5ProductResolver.cc.

References exceptPtr_, fileOffset_, memSize_, storageSize_, threadFriendlyPrefetch(), and type_.

                                                                                                       {
  if (exceptPtr_) {
    rethrow_exception(exceptPtr_);
  }
  if (storageSize_ == 0) {
    return;
  }
  threadFriendlyPrefetch(fileOffset_, storageSize_, memSize_, type_);
}

prefetchAsyncImpl()

void HDF5ProductResolver::prefetchAsyncImpl ( edm::WaitingTaskHolder  , edm::eventsetup::EventSetupRecordImpl const &  , edm::eventsetup::DataKey const &  iKey, edm::EventSetupImpl const *  , edm::ServiceToken const &  , edm::ESParentContext const &    )

finalprivatevirtualnoexcept

This is the function which does the real work of getting the data if it is not already cached. The returning 'void const*' must point to an instance of the class type corresponding to the type designated in iKey. So if iKey refers to a base class interface the pointer must be a pointer to that base class interface and not a pointer to an inheriting class instance.

Implements edm::eventsetup::ESProductResolver.

Definition at line 71 of file HDF5ProductResolver.cc.

References CMS_SA_ALLOW, visDQMUpload::context, edm::ESModuleCallingContext::kRunning, and eostools::move().

                                                                                        {
  prefetchAsyncImplTemplate(
      [this, iov = iRecord.validityInterval(), iParent, &iRecord](auto& iGroup, auto iActivity) {
        queue_->push(iGroup, [this, &iGroup, act = std::move(iActivity), iov, iParent, &iRecord] {
          CMS_SA_ALLOW try {
            edm::ESModuleCallingContext context(providerDescription(),
                                                reinterpret_cast<std::uintptr_t>(this),
                                                edm::ESModuleCallingContext::State::kRunning,
                                                iParent);
            iRecord.activityRegistry()->preESModuleSignal_.emit(iRecord.key(), context);
            struct EndGuard {
              EndGuard(edm::eventsetup::EventSetupRecordImpl const& iRecord,
                       edm::ESModuleCallingContext const& iContext)
                  : record_{iRecord}, context_{iContext} {}
              ~EndGuard() { record_.activityRegistry()->postESModuleSignal_.emit(record_.key(), context_); }
              edm::eventsetup::EventSetupRecordImpl const& record_;
              edm::ESModuleCallingContext const& context_;
            } guardAR(iRecord, context);

            auto index = indexForInterval(iov);

            readFromHDF5api(index);
            iGroup.run(std::move(act));
            exceptPtr_ = {};
          } catch (...) {
            exceptPtr_ = std::current_exception();
          }
        });
      },
      []() { return true; },
      std::move(iTask),
      iRecord,
      iKey,
      iParent);
}

readFromHDF5api()

void HDF5ProductResolver::readFromHDF5api ( std::ptrdiff_t  iIndex )

private

Definition at line 122 of file HDF5ProductResolver.cc.

References dataProduct_, cms::h5::File::derefDataSet(), file_, fileOffset_, memSize_, cond::hdf5::DataProduct::payloadForIOVs_, storageSize_, and type_.

                                                             {
  auto payloadRef = dataProduct_->payloadForIOVs_[iIndex];
  auto ds = file_->derefDataSet(payloadRef);
  storageSize_ = ds->storageSize();
  if (storageSize_ == 0) {
    return;
  }

  fileOffset_ = ds->fileOffset();
  memSize_ = ds->findAttribute("memsize")->readUInt32();
  type_ = ds->findAttribute("type")->readString();
}

threadFriendlyPrefetch()

void HDF5ProductResolver::threadFriendlyPrefetch ( uint64_t  iFileOffset, std::size_t  iStorageSize, std::size_t  iMemSize, const std::string &  iType  )

private

Definition at line 209 of file HDF5ProductResolver.cc.

References edmScanValgrind::buffer, compression_, data_, decompress_lzma(), decompress_zlib(), Exception, geometryDiff::file, fileName_, cond::serialization::unique_void_ptr::get(), helper_, cond::hdf5::kLZMA, cond::hdf5::kZLIB, and eostools::move().

Referenced by prefetch().

                                                                             {
  //Done interacting with the hdf5 API

  //std::cout <<" prefetch "<<dataProduct_->fileOffsets_[index]<<" "<<dataProduct_->storageSizes_[index]<<" "<<memSize<<std::endl;
  std::vector<char> compressedBuffer(iStorageSize);
  std::fstream file(fileName_.c_str());
  file.seekg(iFileOffset);
  file.read(compressedBuffer.data(), compressedBuffer.size());

  std::vector<char> buffer;
  if (compression_ == cond::hdf5::Compression::kZLIB) {
    buffer = decompress_zlib(std::move(compressedBuffer), iMemSize);
  } else if (compression_ == cond::hdf5::Compression::kLZMA) {
    buffer = decompress_lzma(std::move(compressedBuffer), iMemSize);
  } else {
    buffer = std::move(compressedBuffer);
  }

  std::stringbuf sBuffer;
  sBuffer.pubsetbuf(&buffer[0], buffer.size());
  data_ = helper_->deserialize(sBuffer, iTypeName);
  if (data_.get() == nullptr) {
    throw cms::Exception("H5CondFailedDeserialization")
        << "failed to deserialize: buffer size:" << buffer.size() << " type: '" << iTypeName << "'";
  }
}

Member Data Documentation

compression_

cond::hdf5::Compression HDF5ProductResolver::compression_

private

Definition at line 80 of file HDF5ProductResolver.h.

Referenced by threadFriendlyPrefetch().

data_

cond::serialization::unique_void_ptr HDF5ProductResolver::data_

private

Definition at line 74 of file HDF5ProductResolver.h.

Referenced by getAfterPrefetchImpl(), invalidateCache(), and threadFriendlyPrefetch().

dataProduct_

cond::hdf5::DataProduct const* HDF5ProductResolver::dataProduct_

private

Definition at line 79 of file HDF5ProductResolver.h.

Referenced by readFromHDF5api().

exceptPtr_

std::exception_ptr HDF5ProductResolver::exceptPtr_

private

Definition at line 87 of file HDF5ProductResolver.h.

Referenced by prefetch().

file_

cms::h5::File const* HDF5ProductResolver::file_

private

Definition at line 76 of file HDF5ProductResolver.h.

Referenced by readFromHDF5api().

fileName_

std::string HDF5ProductResolver::fileName_

private

Definition at line 77 of file HDF5ProductResolver.h.

Referenced by threadFriendlyPrefetch().

fileOffset_

uint64_t HDF5ProductResolver::fileOffset_

private

Definition at line 83 of file HDF5ProductResolver.h.

Referenced by prefetch(), and readFromHDF5api().

helper_

std::unique_ptr<cond::serialization::SerializationHelperBase> HDF5ProductResolver::helper_

private

Definition at line 75 of file HDF5ProductResolver.h.

Referenced by threadFriendlyPrefetch().

memSize_

std::size_t HDF5ProductResolver::memSize_

private

Definition at line 86 of file HDF5ProductResolver.h.

Referenced by prefetch(), and readFromHDF5api().

queue_

edm::SerialTaskQueue* HDF5ProductResolver::queue_

private

Definition at line 73 of file HDF5ProductResolver.h.

record_

cond::hdf5::Record const* HDF5ProductResolver::record_

private

Definition at line 78 of file HDF5ProductResolver.h.

Referenced by indexForInterval().

storageSize_

std::size_t HDF5ProductResolver::storageSize_

private

Definition at line 85 of file HDF5ProductResolver.h.

Referenced by prefetch(), and readFromHDF5api().

type_

std::string HDF5ProductResolver::type_

private

Definition at line 84 of file HDF5ProductResolver.h.

Referenced by Modules.Service::__str__(), Modules.Service::_placeImpl(), Modules.ESSource::_placeImpl(), Modules.ESProducer::_placeImpl(), Modules.ESPrefer::copy(), Mixins._TypedParameterizable::dumpPython(), Modules.ESPrefer::dumpPythonAs(), Modules.Service::dumpSequencePython(), RandomRunSource.RandomRunSource::insertInto(), Modules.Service::insertInto(), ExternalGeneratorFilter.ExternalGeneratorFilter::insertInto(), Mixins._TypedParameterizable::insertInto(), Modules.ESSource::moduleLabel_(), Modules.ESProducer::moduleLabel_(), Modules.ESSource::nameInProcessDesc_(), Modules.ESProducer::nameInProcessDesc_(), Modules.ESPrefer::nameInProcessDesc_(), prefetch(), and readFromHDF5api().