#include <StreamerInputSource.h>

Inheritance diagram for edm::streamer::StreamerInputSource:

Classes
class	EventPrincipalHolder

Public Member Functions
void	deserializeAndMergeWithRegistry (InitMsgView const &initView, bool subsequent=false)

void	deserializeEvent (EventMsgView const &eventView)

void	deserializeEventMetaData (EventMsgView const &eventView)

std::unique_ptr< SendJobHeader >	deserializeRegistry (InitMsgView const &initView)

uint32_t	eventMetaDataChecksum (EventMsgView const &eventView) const

bool	isBufferLZMA (unsigned char const *inputBuffer, unsigned int inputSize)

bool	isBufferZSTD (unsigned char const *inputBuffer, unsigned int inputSize)

uint32_t	presentEventMetaDataChecksum () const

	StreamerInputSource (ParameterSet const &pset, InputSourceDescription const &desc)

void	updateEventMetaData ()

	~StreamerInputSource () override

Public Member Functions inherited from edm::RawInputSource
	RawInputSource (ParameterSet const &pset, InputSourceDescription const &desc)

	~RawInputSource () override

Public Member Functions inherited from edm::InputSource
std::shared_ptr< ActivityRegistry >	actReg () const
	Accessor for Activity Registry. More...

std::shared_ptr< BranchIDListHelper const >	branchIDListHelper () const
	Accessors for branchIDListHelper. More...

std::shared_ptr< BranchIDListHelper > &	branchIDListHelper ()

void	closeFile (FileBlock *, bool cleaningUpAfterException)
	close current file More...

void	doBeginJob ()
	Called by framework at beginning of job. More...

virtual void	doBeginLumi (LuminosityBlockPrincipal &lbp, ProcessContext const *)
	Called by framework at beginning of lumi block. More...

virtual void	doBeginRun (RunPrincipal &rp, ProcessContext const *)
	Called by framework at beginning of run. More...

void	doEndJob ()
	Called by framework at end of job. More...

void	fillProcessBlockHelper ()
	Fill the ProcessBlockHelper with info for the current file. More...

ProcessingController::ForwardState	forwardState () const

bool	goToEvent (EventID const &eventID)

	InputSource (ParameterSet const &, InputSourceDescription const &)
	Constructor. More...

	InputSource (InputSource const &)=delete

void	issueReports (EventID const &eventID, StreamID streamID)
	issue an event report More...

LuminosityBlockNumber_t	luminosityBlock () const
	Accessor for current luminosity block number. More...

std::shared_ptr< LuminosityBlockAuxiliary >	luminosityBlockAuxiliary () const
	Called by the framework to merge or insert lumi in principal cache. More...

int	maxEvents () const

int	maxLuminosityBlocks () const

ModuleDescription const &	moduleDescription () const
	Accessor for 'module' description. More...

ItemTypeInfo	nextItemType ()
	Advances the source to the next item. More...

bool	nextProcessBlock (ProcessBlockPrincipal &)
	Next process block, return false if there is none, sets the processName in the principal. More...

InputSource &	operator= (InputSource const &)=delete

std::shared_ptr< ProcessBlockHelper const >	processBlockHelper () const
	Accessors for processBlockHelper. More...

std::shared_ptr< ProcessBlockHelper > &	processBlockHelper ()

ProcessConfiguration const &	processConfiguration () const
	Accessor for Process Configuration. More...

std::string const &	processGUID () const
	Accessor for global process identifier. More...

ProcessHistoryRegistry const &	processHistoryRegistry () const
	Accessors for process history registry. More...

ProcessHistoryRegistry &	processHistoryRegistry ()

ProcessingMode	processingMode () const
	RunsLumisAndEvents (default), RunsAndLumis, or Runs. More...

std::shared_ptr< ProductRegistry const >	productRegistry () const
	Accessors for product registry. More...

bool	randomAccess () const

void	readAndMergeLumi (LuminosityBlockPrincipal &lbp)
	Read next luminosity block (same as a prior lumi) More...

void	readAndMergeRun (RunPrincipal &rp)
	Read next run (same as a prior run) More...

void	readEvent (EventPrincipal &ep, StreamContext &)
	Read next event. More...

bool	readEvent (EventPrincipal &ep, EventID const &, StreamContext &)
	Read a specific event. More...

std::shared_ptr< FileBlock >	readFile ()
	Read next file. More...

void	readLuminosityBlock (LuminosityBlockPrincipal &lumiPrincipal, HistoryAppender &historyAppender)
	Read next luminosity block (new lumi) More...

std::shared_ptr< LuminosityBlockAuxiliary >	readLuminosityBlockAuxiliary ()
	Read next luminosity block Auxilary. More...

void	readProcessBlock (ProcessBlockPrincipal &)
	Read next process block. More...

void	readRun (RunPrincipal &runPrincipal, HistoryAppender &historyAppender)
	Read next run (new run) More...

std::shared_ptr< RunAuxiliary >	readRunAuxiliary ()
	Read next run Auxiliary. More...

ProcessHistoryID const &	reducedProcessHistoryID () const

virtual void	registerProducts ()
	Register any produced products. More...

int	remainingEvents () const

int	remainingLuminosityBlocks () const

void	repeat ()
	Reset the remaining number of events/lumis to the maximum number. More...

std::pair< SharedResourcesAcquirer , std::recursive_mutex >	resourceSharedWithDelayedReader ()
	Returns nullptr if no resource shared between the Source and a DelayedReader. More...

ProcessingController::ReverseState	reverseState () const

void	rewind ()
	Begin again at the first event. More...

RunNumber_t	run () const
	Accessor for current run number. More...

std::shared_ptr< RunAuxiliary >	runAuxiliary () const
	Called by the framework to merge or insert run in principal cache. More...

void	setLuminosityBlockNumber_t (LuminosityBlockNumber_t lb)
	Set the luminosity block ID. More...

void	setRunNumber (RunNumber_t r)
	Set the run number. More...

void	skipEvents (int offset)

void	switchTo (std::shared_ptr< ProductRegistry > iOther)
	switch to a different ProductRegistry. More...

std::shared_ptr< ThinnedAssociationsHelper const >	thinnedAssociationsHelper () const
	Accessors for thinnedAssociationsHelper. More...

std::shared_ptr< ThinnedAssociationsHelper > &	thinnedAssociationsHelper ()

Timestamp const &	timestamp () const
	Accessor for the current time, as seen by the input source. More...

virtual	~InputSource () noexcept(false)
	Destructor. More...

Static Public Member Functions
static void	fillDescription (ParameterSetDescription &description)

static void	mergeIntoRegistry (SendJobHeader const &header, ProductRegistry &, bool subsequent)

static unsigned int	uncompressBuffer (unsigned char *inputBuffer, unsigned int inputSize, std::vector< unsigned char > &outputBuffer, unsigned int expectedFullSize)

static unsigned int	uncompressBufferLZMA (unsigned char *inputBuffer, unsigned int inputSize, std::vector< unsigned char > &outputBuffer, unsigned int expectedFullSize, bool hasHeader=true)

static unsigned int	uncompressBufferZSTD (unsigned char *inputBuffer, unsigned int inputSize, std::vector< unsigned char > &outputBuffer, unsigned int expectedFullSize, bool hasHeader=true)

Static Public Member Functions inherited from edm::RawInputSource
static void	fillDescription (ParameterSetDescription &description)

Static Public Member Functions inherited from edm::InputSource
static const std::string &	baseType ()

static void	fillDescription (ParameterSetDescription &desc)

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &)

Protected Member Functions
void	resetAfterEndRun ()

Protected Member Functions inherited from edm::RawInputSource
virtual Next	checkNext ()=0

void	makeEvent (EventPrincipal &eventPrincipal, EventAuxiliary const &eventAuxiliary)

void	setInputFileTransitionsEachEvent ()

Protected Member Functions inherited from edm::InputSource
virtual void	beginJob ()
	Begin protected makes it easier to do template programming. More...

void	decreaseRemainingEventsBy (int iSkipped)

bool	eventCached () const

bool	newLumi () const

bool	newRun () const

ProcessHistoryRegistry &	processHistoryRegistryForUpdate ()

ProductRegistry &	productRegistryUpdate ()

void	reset () const

void	resetEventCached ()

void	resetLuminosityBlockAuxiliary (bool isNewLumi=true) const

void	resetNewLumi ()

void	resetNewRun ()

void	resetRunAuxiliary (bool isNewRun=true) const

void	setEventCached ()
	Called by the framework to merge or ached() const {return eventCached_;}. More...

void	setLuminosityBlockAuxiliary (LuminosityBlockAuxiliary *lbp)

void	setNewLumi ()

void	setNewRun ()

void	setRunAuxiliary (RunAuxiliary *rp)

void	setTimestamp (Timestamp const &theTime)
	To set the current time, as seen by the input source. More...

virtual void	skip (int offset)

ItemTypeInfo	state () const

Static Protected Member Functions
static void	buildClassCache (SendDescs const &descs)

static void	declareStreamers (SendDescs const &descs)

Private Member Functions
void	deserializeEventCommon (EventMsgView const &eventView, bool isMetaData)

void	read (EventPrincipal &eventPrincipal) override

void	setRun (RunNumber_t r) override

Private Attributes
bool	adjustEventToNewProductRegistry_

std::vector< unsigned char >	dest_

uint32_t	eventMetaDataChecksum_ = 0

edm::propagate_const< std::unique_ptr< EventPrincipalHolder > >	eventPrincipalHolder_

std::string	processName_

unsigned int	protocolVersion_

edm::propagate_const< std::unique_ptr< SendEvent > >	sendEvent_

std::vector< edm::propagate_const< std::unique_ptr< EventPrincipalHolder > > >	streamToEventPrincipalHolders_

edm::propagate_const< TClass * >	tc_

TBufferFile	xbuf_

Additional Inherited Members
Public Types inherited from edm::RawInputSource
enum	Next { Next::kEvent, Next::kFile, Next::kStop }

Public Types inherited from edm::InputSource
enum	ItemPosition : char { ItemPosition::Invalid, ItemPosition::LastItemToBeMerged, ItemPosition::NotLastItemToBeMerged }

enum	ItemType : char { ItemType::IsInvalid, ItemType::IsStop, ItemType::IsFile, ItemType::IsRun, ItemType::IsLumi, ItemType::IsEvent, ItemType::IsRepeat, ItemType::IsSynchronize }

enum	ProcessingMode { Runs, RunsAndLumis, RunsLumisAndEvents }

Public Attributes inherited from edm::InputSource
signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)>	postEventReadFromSourceSignal_

signalslot::Signal< void(StreamContext const &, ModuleCallingContext const &)>	preEventReadFromSourceSignal_

Detailed Description

Definition at line 33 of file StreamerInputSource.h.

Constructor & Destructor Documentation

◆ StreamerInputSource()

edm::streamer::StreamerInputSource::StreamerInputSource	(	ParameterSet const &	pset,
		InputSourceDescription const &	desc
	)

explicit

Definition at line 45 of file StreamerInputSource.cc.

       : RawInputSource(pset, desc),
         tc_(getTClass(typeid(SendEvent))),
         dest_(init_size),
         xbuf_(TBuffer::kRead, init_size),
         sendEvent_(),
         eventPrincipalHolder_(),
         adjustEventToNewProductRegistry_(false),
         processName_(),
         protocolVersion_(0U) {}

◆ ~StreamerInputSource()

edm::streamer::StreamerInputSource::~StreamerInputSource ( )

override

Definition at line 56 of file StreamerInputSource.cc.

56 {}

Member Function Documentation

◆ buildClassCache()

void edm::streamer::StreamerInputSource::buildClassCache ( SendDescs const & descs )

staticprotected

Definition at line 100 of file StreamerInputSource.cc.

References edm::streamer::doBuildRealData(), FDEBUG, B2GTnPMonitor_cfi::item, AlCaHLTBitMon_QueryRunRegistry::string, and edm::wrappedClassName().

Referenced by mergeIntoRegistry().

                                                                   {
     for (auto const& item : descs) {
       //pi->init();
       std::string const real_name = wrappedClassName(item.className());
       FDEBUG(6) << "BuildReadData: " << real_name << std::endl;
       doBuildRealData(real_name);
     }
   }

◆ declareStreamers()

void edm::streamer::StreamerInputSource::declareStreamers ( SendDescs const & descs )

staticprotected

Definition at line 81 of file StreamerInputSource.cc.

References visDQMUpload::context, FDEBUG, B2GTnPMonitor_cfi::item, edm::streamer::loadCap(), AlCaHLTBitMon_QueryRunRegistry::string, edm::throwMissingDictionariesException(), and edm::wrappedClassName().

Referenced by mergeIntoRegistry().

                                                                    {
     std::vector<std::string> missingDictionaries;
     std::vector<std::string> branchNamesForMissing;
     std::vector<std::string> producedTypes;
     for (auto const& item : descs) {
       //pi->init();
       std::string const real_name = wrappedClassName(item.className());
       FDEBUG(6) << "declare: " << real_name << std::endl;
       if (!loadCap(real_name, missingDictionaries)) {
         branchNamesForMissing.emplace_back(item.branchName());
         producedTypes.emplace_back(item.className() + std::string(" (read from input)"));
       }
     }
     if (!missingDictionaries.empty()) {
       std::string context("Calling StreamerInputSource::declareStreamers, checking dictionaries for input types");
       throwMissingDictionariesException(missingDictionaries, context, producedTypes, branchNamesForMissing, true);
     }
   }

◆ deserializeAndMergeWithRegistry()

void edm::streamer::StreamerInputSource::deserializeAndMergeWithRegistry	(	InitMsgView const &	initView,
		bool	subsequent = `false`
	)

Deserializes the specified init message into a SendJobHeader object and merges registries.

Definition at line 159 of file StreamerInputSource.cc.

References adjustEventToNewProductRegistry_, deserializeRegistry(), edm::pset::Registry::insertMapped(), edm::pset::Registry::instance(), B2GTnPMonitor_cfi::item, mergeIntoRegistry(), edm::InputSource::productRegistryUpdate(), and muonDTDigis_cfi::pset.

Referenced by edm::streamer::StreamerInputModule< Producer >::setupMetaData(), dqmservices::DQMStreamerReader::setupMetaData(), and edm::streamer::StreamerFileReader::updateMetaData().

                                                                                                         {
     std::unique_ptr<SendJobHeader> sd = deserializeRegistry(initView);
     mergeIntoRegistry(*sd, productRegistryUpdate(), subsequent);
     if (subsequent) {
       adjustEventToNewProductRegistry_ = true;
     }
     SendJobHeader::ParameterSetMap const& psetMap = sd->processParameterSet();
     pset::Registry& psetRegistry = *pset::Registry::instance();
     for (auto const& item : psetMap) {
       ParameterSet pset(item.second.pset());
       pset.setID(item.first);
       psetRegistry.insertMapped(pset);
     }
   }

◆ deserializeEvent()

void edm::streamer::StreamerInputSource::deserializeEvent ( EventMsgView const & eventView )

Deserializes the specified event message.

Definition at line 189 of file StreamerInputSource.cc.

References deserializeEventCommon().

Referenced by dqmservices::DQMStreamerReader::checkNext(), and edm::streamer::StreamerFileReader::checkNext().

                                                                           {
     deserializeEventCommon(eventView, false);
   }

◆ deserializeEventCommon()

void edm::streamer::StreamerInputSource::deserializeEventCommon	(	EventMsgView const &	eventView,
		bool	isMetaData
	)

private

Definition at line 193 of file StreamerInputSource.cc.

Referenced by deserializeEvent(), and deserializeEventMetaData().

                                                                                                  {
     if (eventView.code() != Header::EVENT)
       throw cms::Exception("StreamTranslation", "Event deserialization error")
           << "received wrong message type: expected EVENT, got " << eventView.code() << "\n";
     FDEBUG(9) << "Decode event: " << eventView.event() << " " << eventView.run() << " " << eventView.size() << " "
               << eventView.adler32_chksum() << " " << eventView.eventLength() << " " << eventView.eventData()
               << std::endl;
     // uncompress if we need to
     // 78 was a dummy value (for no uncompressed) - should be 0 for uncompressed
     // need to get rid of this when 090 MTCC streamers are gotten rid of
     unsigned long origsize = eventView.origDataSize();
     unsigned long dest_size;  //(should be >= eventView.origDataSize())
 
     uint32_t adler32_chksum = cms::Adler32((char const*)eventView.eventData(), eventView.eventLength());
     //std::cout << "Adler32 checksum of event = " << adler32_chksum << std::endl;
     //std::cout << "Adler32 checksum from header = " << eventView.adler32_chksum() << " "
     //          << "host name = " << eventView.hostName() << " len = " << eventView.hostName_len() << std::endl;
     if (static_cast<uint32>(adler32_chksum) != eventView.adler32_chksum()) {
       // skip event (based on option?) or throw exception?
       throw cms::Exception("StreamDeserialization", "Checksum error")
           << " chksum from event = " << adler32_chksum << " from header = " << eventView.adler32_chksum()
           << " host name = " << eventView.hostName() << std::endl;
     }
     if (origsize != 78 && origsize != 0) {
       // compressed
       if (isBufferLZMA((unsigned char const*)eventView.eventData(), eventView.eventLength())) {
         dest_size = uncompressBufferLZMA(const_cast<unsigned char*>((unsigned char const*)eventView.eventData()),
                                          eventView.eventLength(),
                                          dest_,
                                          origsize);
       } else if (isBufferZSTD((unsigned char const*)eventView.eventData(), eventView.eventLength())) {
         dest_size = uncompressBufferZSTD(const_cast<unsigned char*>((unsigned char const*)eventView.eventData()),
                                          eventView.eventLength(),
                                          dest_,
                                          origsize);
       } else
         dest_size = uncompressBuffer(const_cast<unsigned char*>((unsigned char const*)eventView.eventData()),
                                      eventView.eventLength(),
                                      dest_,
                                      origsize);
     } else {  // not compressed
       // we need to copy anyway the buffer as we are using dest in xbuf
       dest_size = eventView.eventLength();
       dest_.resize(dest_size);
       unsigned char* pos = (unsigned char*)&dest_[0];
       unsigned char const* from = (unsigned char const*)eventView.eventData();
       std::copy(from, from + dest_size, pos);
     }
     //TBuffer xbuf(TBuffer::kRead, dest_size,
     //             (char const*) &dest[0],kFALSE);
     //TBuffer xbuf(TBuffer::kRead, eventView.eventLength(),
     //             (char const*) eventView.eventData(),kFALSE);
     xbuf_.Reset();
     xbuf_.SetBuffer(&dest_[0], dest_size, kFALSE);
     RootDebug tracer(10, 10);
 
     //We do not yet know which EventPrincipal we will use, therefore
     // we are using a new EventPrincipalHolder as a proxy. We need to
     // make a new one instead of reusing the same one becuase when running
     // multi-threaded there will be multiple EventPrincipals being used
     // simultaneously.
     eventPrincipalHolder_ = std::make_unique<EventPrincipalHolder>();  // propagate_const<T> has no reset() function
     setRefCoreStreamer(eventPrincipalHolder_.get());
     {
       std::shared_ptr<void> refCoreStreamerGuard(nullptr, [](void*) {
         setRefCoreStreamer();
         ;
       });
       sendEvent_ = std::unique_ptr<SendEvent>(reinterpret_cast<SendEvent*>(xbuf_.ReadObjectAny(tc_)));
     }
 
     if (sendEvent_.get() == nullptr) {
       throw cms::Exception("StreamTranslation", "Event deserialization error")
           << "got a null event from input stream\n";
     }
 
     if (isMetaData) {
       eventMetaDataChecksum_ = adler32_chksum;
       return;
     }
 
     if (sendEvent_->metaDataChecksum() != eventMetaDataChecksum_) {
       throw cms::Exception("StreamTranslation") << " meta data checksum from event " << sendEvent_->metaDataChecksum()
                                                 << " does not match last read meta data " << eventMetaDataChecksum_;
     }
 
     processHistoryRegistryForUpdate().registerProcessHistory(sendEvent_->processHistory());
 
     FDEBUG(5) << "Got event: " << sendEvent_->aux().id() << " " << sendEvent_->products().size() << std::endl;
     if (runAuxiliary().get() == nullptr || runAuxiliary()->run() != sendEvent_->aux().run() ||
         runAuxiliary()->processHistoryID() != sendEvent_->processHistory().id()) {
       RunAuxiliary* runAuxiliary =
           new RunAuxiliary(sendEvent_->aux().run(), sendEvent_->aux().time(), Timestamp::invalidTimestamp());
       runAuxiliary->setProcessHistoryID(sendEvent_->processHistory().id());
       setRunAuxiliary(runAuxiliary);
       resetLuminosityBlockAuxiliary();
     }
     if (!luminosityBlockAuxiliary() || luminosityBlockAuxiliary()->luminosityBlock() != eventView.lumi()) {
       LuminosityBlockAuxiliary* luminosityBlockAuxiliary = new LuminosityBlockAuxiliary(
           runAuxiliary()->run(), eventView.lumi(), sendEvent_->aux().time(), Timestamp::invalidTimestamp());
       luminosityBlockAuxiliary->setProcessHistoryID(sendEvent_->processHistory().id());
       setLuminosityBlockAuxiliary(luminosityBlockAuxiliary);
     }
     setEventCached();
   }

◆ deserializeEventMetaData()

void edm::streamer::StreamerInputSource::deserializeEventMetaData ( EventMsgView const & eventView )

Definition at line 183 of file StreamerInputSource.cc.

References deserializeEventCommon().

Referenced by edm::streamer::StreamerFileReader::checkNext(), dqmservices::DQMStreamerReader::prepareNextEvent(), edm::streamer::StreamerInputModule< Producer >::setupMetaData(), dqmservices::DQMStreamerReader::setupMetaData(), and edm::streamer::StreamerFileReader::updateMetaData().

                                                                                   {
     deserializeEventCommon(eventView, true);
   }

◆ deserializeRegistry()

std::unique_ptr< SendJobHeader > edm::streamer::StreamerInputSource::deserializeRegistry ( InitMsgView const & initView )

Deserializes the specified init message into a SendJobHeader object (which is related to the product registry).

Definition at line 113 of file StreamerInputSource.cc.

References cms::Adler32(), edm::streamer::InitMsgView::adler32_chksum(), edm::streamer::InitMsgView::code(), submitPVResolutionJobs::desc, edm::streamer::InitMsgView::descData(), edm::streamer::InitMsgView::descLength(), Exception, FDEBUG, edm::streamer::getTClass(), edm::streamer::InitMsgView::hostName(), edm::streamer::Header::INIT, edm::streamer::InitMsgView::processName(), processName_, edm::streamer::InitMsgView::protocolVersion(), and protocolVersion_.

Referenced by deserializeAndMergeWithRegistry().

                                                                                                    {
     if (initView.code() != Header::INIT)
       throw cms::Exception("StreamTranslation", "Registry deserialization error")
           << "received wrong message type: expected INIT, got " << initView.code() << "\n";
 
     //Get the process name and store if for Protocol version 4 and above.
     if (initView.protocolVersion() > 3) {
       processName_ = initView.processName();
       protocolVersion_ = initView.protocolVersion();
 
       FDEBUG(10) << "StreamerInputSource::deserializeRegistry processName = " << processName_ << std::endl;
       FDEBUG(10) << "StreamerInputSource::deserializeRegistry protocolVersion_= " << protocolVersion_ << std::endl;
     }
 
     // calculate the adler32 checksum
     uint32_t adler32_chksum = cms::Adler32((char const*)initView.descData(), initView.descLength());
     //std::cout << "Adler32 checksum of init message = " << adler32_chksum << std::endl;
     //std::cout << "Adler32 checksum of init messsage from header = " << initView.adler32_chksum() << " "
     //          << "host name = " << initView.hostName() << " len = " << initView.hostName_len() << std::endl;
     if ((uint32)adler32_chksum != initView.adler32_chksum()) {
       // skip event (based on option?) or throw exception?
       throw cms::Exception("StreamDeserialization", "Checksum error")
           << " chksum from registry data = " << adler32_chksum << " from header = " << initView.adler32_chksum()
           << " host name = " << initView.hostName() << std::endl;
     }
 
     TClass* desc = getTClass(typeid(SendJobHeader));
 
     TBufferFile xbuf(
         TBuffer::kRead, initView.descLength(), const_cast<char*>((char const*)initView.descData()), kFALSE);
     RootDebug tracer(10, 10);
     std::unique_ptr<SendJobHeader> sd((SendJobHeader*)xbuf.ReadObjectAny(desc));
 
     if (sd.get() == nullptr) {
       throw cms::Exception("StreamTranslation", "Registry deserialization error")
           << "Could not read the initial product registry list\n";
     }
 
     sd->initializeTransients();
     return sd;
   }

◆ eventMetaDataChecksum()

uint32_t edm::streamer::StreamerInputSource::eventMetaDataChecksum ( EventMsgView const & eventView ) const

Definition at line 179 of file StreamerInputSource.cc.

References edm::streamer::EventMsgView::adler32_chksum().

Referenced by edm::streamer::StreamerFileReader::checkNext(), and dqmservices::DQMStreamerReader::prepareNextEvent().

                                                                                          {
     return eventView.adler32_chksum();
   }

◆ fillDescription()

void edm::streamer::StreamerInputSource::fillDescription ( ParameterSetDescription & description )

static

Definition at line 532 of file StreamerInputSource.cc.

References submitPVResolutionJobs::desc, and edm::RawInputSource::fillDescription().

Referenced by dqmservices::DQMStreamerReader::fillDescriptions(), and edm::streamer::StreamerFileReader::fillDescriptions().

532 { RawInputSource::fillDescription(desc); }

submitPVResolutionJobs.desc

string desc

Definition: submitPVResolutionJobs.py:254

edm::RawInputSource::fillDescription

static void fillDescription(ParameterSetDescription &description)

Definition: RawInputSource.cc:105

◆ isBufferLZMA()

bool edm::streamer::StreamerInputSource::isBufferLZMA	(	unsigned char const *	inputBuffer,
		unsigned int	inputSize
	)

Detect if buffer starts with "XZ\0" which means it is compressed in LZMA format

Definition at line 398 of file StreamerInputSource.cc.

Referenced by deserializeEventCommon().

                                                                                                  {
     if (inputSize >= 4 && !strcmp((const char*)inputBuffer, "XZ"))
       return true;
     else
       return false;
   }

◆ isBufferZSTD()

bool edm::streamer::StreamerInputSource::isBufferZSTD	(	unsigned char const *	inputBuffer,
		unsigned int	inputSize
	)

Detect if buffer starts with "Z\0" which means it is compressed in ZStandard format

Definition at line 450 of file StreamerInputSource.cc.

Referenced by deserializeEventCommon().

                                                                                                  {
     if (inputSize >= 4 && !strcmp((const char*)inputBuffer, "ZS"))
       return true;
     else
       return false;
   }

◆ mergeIntoRegistry()

void edm::streamer::StreamerInputSource::mergeIntoRegistry	(	SendJobHeader const &	header,
		ProductRegistry &	reg,
		bool	subsequent
	)

static

Definition at line 59 of file StreamerInputSource.cc.

References buildClassCache(), declareStreamers(), Exception, FDEBUG, edm::ProductRegistry::frozen(), RecoTauValidation_cfi::header, edm::streamer::loadExtraClasses(), edm::ProductRegistry::merge(), edm::BranchDescription::Permissive, AlCaHLTBitMon_QueryRunRegistry::string, and edm::ProductRegistry::updateFromInput().

Referenced by deserializeAndMergeWithRegistry().

                                                                                                                 {
     SendDescs const& descs = header.descs();
 
     FDEBUG(6) << "mergeIntoRegistry: Product List: " << std::endl;
 
     if (subsequent) {
       ProductRegistry pReg;
       pReg.updateFromInput(descs);
       std::string mergeInfo = reg.merge(pReg, std::string(), BranchDescription::Permissive);
       if (!mergeInfo.empty()) {
         throw cms::Exception("MismatchedInput", "RootInputFileSequence::previousEvent()") << mergeInfo;
       }
     } else {
       declareStreamers(descs);
       buildClassCache(descs);
       loadExtraClasses();
       if (!reg.frozen()) {
         reg.updateFromInput(descs);
       }
     }
   }

◆ presentEventMetaDataChecksum()

uint32_t edm::streamer::StreamerInputSource::presentEventMetaDataChecksum ( ) const

inline

Definition at line 49 of file StreamerInputSource.h.

References eventMetaDataChecksum_.

Referenced by edm::streamer::StreamerFileReader::checkNext(), and dqmservices::DQMStreamerReader::prepareNextEvent().

49 { return eventMetaDataChecksum_; }

edm::streamer::StreamerInputSource::eventMetaDataChecksum_

uint32_t eventMetaDataChecksum_

Definition: StreamerInputSource.h:135

◆ read()

void edm::streamer::StreamerInputSource::read ( EventPrincipal & eventPrincipal )

overrideprivatevirtual

Implements edm::RawInputSource.

Definition at line 299 of file StreamerInputSource.cc.

References adjustEventToNewProductRegistry_, edm::Principal::adjustIndexesAfterProductRegistryAddition(), edm::Principal::adjustToNewProductRegistry(), cms::cuda::assert(), edm::BranchDescription::branchID(), edm::InputSource::branchIDListHelper(), eventPrincipalHolder_, FDEBUG, edm::EventPrincipal::fillEventPrincipal(), edm::ProcessHistoryRegistry::getMapped(), eostools::move(), edm::InputSource::processHistoryRegistry(), edm::InputSource::productRegistry(), edm::EventPrincipal::putOnRead(), sendEvent_, edm::Principal::size(), edm::EventPrincipal::streamID(), streamToEventPrincipalHolders_, and edm::StreamID::value().

Referenced by edmIntegrityCheck.PublishToFileSystem::get().

                                                                {
     if (adjustEventToNewProductRegistry_) {
       eventPrincipal.adjustIndexesAfterProductRegistryAddition();
       bool eventOK = eventPrincipal.adjustToNewProductRegistry(*productRegistry());
       assert(eventOK);
       adjustEventToNewProductRegistry_ = false;
     }
     EventSelectionIDVector ids(sendEvent_->eventSelectionIDs());
     BranchListIndexes indexes(sendEvent_->branchListIndexes());
     branchIDListHelper()->fixBranchListIndexes(indexes);
     auto history = processHistoryRegistry().getMapped(sendEvent_->aux().processHistoryID());
     eventPrincipal.fillEventPrincipal(sendEvent_->aux(), history, std::move(ids), std::move(indexes));
 
     //We now know which eventPrincipal to use and we can reuse the slot in
     // streamToEventPrincipalHolders to own the memory
     eventPrincipalHolder_->setEventPrincipal(&eventPrincipal);
     if (streamToEventPrincipalHolders_.size() < eventPrincipal.streamID().value() + 1) {
       streamToEventPrincipalHolders_.resize(eventPrincipal.streamID().value() + 1);
     }
     streamToEventPrincipalHolders_[eventPrincipal.streamID().value()] = std::move(eventPrincipalHolder_);
 
     // no process name list handling
 
     SendProds& sps = sendEvent_->products();
     for (auto& spitem : sps) {
       FDEBUG(10) << "check prodpair" << std::endl;
       if (spitem.desc() == nullptr)
         throw cms::Exception("StreamTranslation", "Empty Provenance");
       FDEBUG(5) << "Prov:"
                 << " " << spitem.desc()->className() << " " << spitem.desc()->productInstanceName() << " "
                 << spitem.desc()->branchID() << std::endl;
 
       BranchDescription const branchDesc(*spitem.desc());
       // This ProductProvenance constructor inserts into the entry description registry
       if (spitem.parents()) {
         std::optional<ProductProvenance> productProvenance{std::in_place, spitem.branchID(), *spitem.parents()};
         if (spitem.prod() != nullptr) {
           FDEBUG(10) << "addproduct next " << spitem.branchID() << std::endl;
           eventPrincipal.putOnRead(branchDesc,
                                    std::unique_ptr<WrapperBase>(const_cast<WrapperBase*>(spitem.prod())),
                                    std::move(productProvenance));
           FDEBUG(10) << "addproduct done" << std::endl;
         } else {
           FDEBUG(10) << "addproduct empty next " << spitem.branchID() << std::endl;
           eventPrincipal.putOnRead(branchDesc, std::unique_ptr<WrapperBase>(), std::move(productProvenance));
           FDEBUG(10) << "addproduct empty done" << std::endl;
         }
       } else {
         std::optional<ProductProvenance> productProvenance;
         if (spitem.prod() != nullptr) {
           FDEBUG(10) << "addproduct next " << spitem.branchID() << std::endl;
           eventPrincipal.putOnRead(
               branchDesc, std::unique_ptr<WrapperBase>(const_cast<WrapperBase*>(spitem.prod())), productProvenance);
           FDEBUG(10) << "addproduct done" << std::endl;
         } else {
           FDEBUG(10) << "addproduct empty next " << spitem.branchID() << std::endl;
           eventPrincipal.putOnRead(branchDesc, std::unique_ptr<WrapperBase>(), productProvenance);
           FDEBUG(10) << "addproduct empty done" << std::endl;
         }
       }
       spitem.clear();
     }
 
     FDEBUG(10) << "Size = " << eventPrincipal.size() << std::endl;
   }

◆ resetAfterEndRun()

void edm::streamer::StreamerInputSource::resetAfterEndRun ( )

protected

Definition at line 477 of file StreamerInputSource.cc.

References cms::cuda::assert(), edm::InputSource::eventCached(), edm::InputSource::reset(), edm::InputSource::resetLuminosityBlockAuxiliary(), and edm::InputSource::resetRunAuxiliary().

                                              {
     // called from an online streamer source to reset after a stop command
     // so an enable command will work
     resetLuminosityBlockAuxiliary();
     resetRunAuxiliary();
     assert(!eventCached());
     reset();
   }

◆ setRun()

void edm::streamer::StreamerInputSource::setRun ( RunNumber_t r )

overrideprivatevirtual

Reimplemented from edm::InputSource.

Definition at line 486 of file StreamerInputSource.cc.

References Exception, and edm::errors::LogicError.

                                               {
     // Need to define a dummy setRun here or else the InputSource::setRun is called
     // if we have a source inheriting from this and wants to define a setRun method
     throw Exception(errors::LogicError) << "StreamerInputSource::setRun()\n"
                                         << "Run number cannot be modified for this type of Input Source\n"
                                         << "Contact a Storage Manager Developer\n";
   }

◆ uncompressBuffer()

unsigned int edm::streamer::StreamerInputSource::uncompressBuffer	(	unsigned char *	inputBuffer,
		unsigned int	inputSize,
		std::vector< unsigned char > &	outputBuffer,
		unsigned int	expectedFullSize
	)

static

Uncompresses the data in the specified input buffer into the specified output buffer. The inputSize should be set to the size of the compressed data in the inputBuffer. The expectedFullSize should be set to the original size of the data (before compression). Returns the actual size of the uncompressed data. Errors are reported by throwing exceptions.

Definition at line 373 of file StreamerInputSource.cc.

References Exception, FDEBUG, and runTheMatrix::ret.

Referenced by deserializeEventCommon().

                                                                                     {
     unsigned long origSize = expectedFullSize;
     unsigned long uncompressedSize = expectedFullSize * 1.1;
     FDEBUG(1) << "Uncompress: original size = " << origSize << ", compressed size = " << inputSize << std::endl;
     outputBuffer.resize(uncompressedSize);
     int ret = uncompress(&outputBuffer[0], &uncompressedSize, inputBuffer, inputSize);  // do not need compression level
     //std::cout << "unCompress Return value: " << ret << " Okay = " << Z_OK << std::endl;
     if (ret == Z_OK) {
       // check the length against original uncompressed length
       FDEBUG(10) << " original size = " << origSize << " final size = " << uncompressedSize << std::endl;
       if (origSize != uncompressedSize) {
         // we throw an error and return without event! null pointer
         throw cms::Exception("StreamDeserialization", "Uncompression error")
             << "mismatch event lengths should be" << origSize << " got " << uncompressedSize << "\n";
       }
     } else {
       // we throw an error and return without event! null pointer
       throw cms::Exception("StreamDeserialization", "Uncompression error") << "Error code = " << ret << "\n ";
     }
     return (unsigned int)uncompressedSize;
   }

◆ uncompressBufferLZMA()

unsigned int edm::streamer::StreamerInputSource::uncompressBufferLZMA	(	unsigned char *	inputBuffer,
		unsigned int	inputSize,
		std::vector< unsigned char > &	outputBuffer,
		unsigned int	expectedFullSize,
		bool	hasHeader = `true`
	)

static

Definition at line 405 of file StreamerInputSource.cc.

References Exception, FDEBUG, createfilelist::int, cms::cuda::stream, and mitigatedMETSequence_cff::U.

Referenced by deserializeEventCommon().

                                                                          {
     unsigned long origSize = expectedFullSize;
     unsigned long uncompressedSize = expectedFullSize * 1.1;
     FDEBUG(1) << "Uncompress: original size = " << origSize << ", compressed size = " << inputSize << std::endl;
     outputBuffer.resize(uncompressedSize);
 
     lzma_stream stream = LZMA_STREAM_INIT;
     lzma_ret returnStatus;
 
     returnStatus = lzma_stream_decoder(&stream, UINT64_MAX, 0U);
     if (returnStatus != LZMA_OK) {
       throw cms::Exception("StreamDeserializationLZM", "LZMA stream decoder error")
           << "Error code = " << returnStatus << "\n ";
     }
 
     size_t hdrSize = hasHeader ? 4 : 0;
     stream.next_in = (const uint8_t*)(inputBuffer + hdrSize);
     stream.avail_in = (size_t)(inputSize - hdrSize);
     stream.next_out = (uint8_t*)&outputBuffer[0];
     stream.avail_out = (size_t)uncompressedSize;
 
     returnStatus = lzma_code(&stream, LZMA_FINISH);
     if (returnStatus != LZMA_STREAM_END) {
       lzma_end(&stream);
       throw cms::Exception("StreamDeserializationLZM", "LZMA uncompression error")
           << "Error code = " << returnStatus << "\n ";
     }
     lzma_end(&stream);
 
     uncompressedSize = (unsigned int)stream.total_out;
 
     FDEBUG(10) << " original size = " << origSize << " final size = " << uncompressedSize << std::endl;
     if (origSize != uncompressedSize) {
       // we throw an error and return without event! null pointer
       throw cms::Exception("StreamDeserialization", "LZMA uncompression error")
           << "mismatch event lengths should be" << origSize << " got " << uncompressedSize << "\n";
     }
 
     return uncompressedSize;
   }

◆ uncompressBufferZSTD()

unsigned int edm::streamer::StreamerInputSource::uncompressBufferZSTD	(	unsigned char *	inputBuffer,
		unsigned int	inputSize,
		std::vector< unsigned char > &	outputBuffer,
		unsigned int	expectedFullSize,
		bool	hasHeader = `true`
	)

static

Definition at line 457 of file StreamerInputSource.cc.

References Exception, FDEBUG, and runTheMatrix::ret.

Referenced by deserializeEventCommon().

                                                                          {
     unsigned long uncompressedSize = expectedFullSize * 1.1;
     FDEBUG(1) << "Uncompress: original size = " << expectedFullSize << ", compressed size = " << inputSize << std::endl;
     outputBuffer.resize(uncompressedSize);
 
     size_t hdrSize = hasHeader ? 4 : 0;
     size_t ret = ZSTD_decompress(
         (void*)&(outputBuffer[0]), uncompressedSize, (const void*)(inputBuffer + hdrSize), inputSize - hdrSize);
 
     if (ZSTD_isError(ret)) {
       throw cms::Exception("StreamDeserializationZSTD", "ZSTD uncompression error")
           << "Error core " << ret << ", message:" << ZSTD_getErrorName(ret);
     }
     return (unsigned int)ret;
   }

◆ updateEventMetaData()

void edm::streamer::StreamerInputSource::updateEventMetaData ( )

Definition at line 174 of file StreamerInputSource.cc.

References edm::InputSource::branchIDListHelper(), sendEvent_, and edm::InputSource::thinnedAssociationsHelper().

Referenced by dqmservices::DQMStreamerReader::genuineReadFile(), edm::streamer::StreamerFileReader::genuineReadFile(), edm::streamer::StreamerInputModule< Producer >::genuineReadFile(), edm::streamer::StreamerInputModule< Producer >::setupMetaData(), dqmservices::DQMStreamerReader::setupMetaData(), and edm::streamer::StreamerFileReader::updateMetaData().

                                                 {
     branchIDListHelper()->updateFromInput(sendEvent_->branchIDLists());
     thinnedAssociationsHelper()->updateFromPrimaryInput(sendEvent_->thinnedAssociationsHelper());
   }

Member Data Documentation

◆ adjustEventToNewProductRegistry_

bool edm::streamer::StreamerInputSource::adjustEventToNewProductRegistry_

private

Definition at line 131 of file StreamerInputSource.h.

Referenced by deserializeAndMergeWithRegistry(), and read().

◆ dest_

std::vector<unsigned char> edm::streamer::StreamerInputSource::dest_

private

Definition at line 126 of file StreamerInputSource.h.

Referenced by deserializeEventCommon().

◆ eventMetaDataChecksum_

uint32_t edm::streamer::StreamerInputSource::eventMetaDataChecksum_ = 0

private

Definition at line 135 of file StreamerInputSource.h.

Referenced by deserializeEventCommon(), and presentEventMetaDataChecksum().

◆ eventPrincipalHolder_

edm::propagate_const<std::unique_ptr<EventPrincipalHolder> > edm::streamer::StreamerInputSource::eventPrincipalHolder_

private

Definition at line 129 of file StreamerInputSource.h.

Referenced by deserializeEventCommon(), and read().

◆ processName_

std::string edm::streamer::StreamerInputSource::processName_

private

Definition at line 133 of file StreamerInputSource.h.

Referenced by deserializeRegistry().

◆ protocolVersion_

unsigned int edm::streamer::StreamerInputSource::protocolVersion_

private

Definition at line 134 of file StreamerInputSource.h.

Referenced by deserializeRegistry().

◆ sendEvent_

edm::propagate_const<std::unique_ptr<SendEvent> > edm::streamer::StreamerInputSource::sendEvent_

private

Definition at line 128 of file StreamerInputSource.h.

Referenced by deserializeEventCommon(), read(), and updateEventMetaData().

◆ streamToEventPrincipalHolders_

std::vector<edm::propagate_const<std::unique_ptr<EventPrincipalHolder> > > edm::streamer::StreamerInputSource::streamToEventPrincipalHolders_

private

Definition at line 130 of file StreamerInputSource.h.

Referenced by read().

◆ tc_

edm::propagate_const<TClass*> edm::streamer::StreamerInputSource::tc_

private

Definition at line 125 of file StreamerInputSource.h.

Referenced by deserializeEventCommon().

◆ xbuf_

TBufferFile edm::streamer::StreamerInputSource::xbuf_

private

Definition at line 127 of file StreamerInputSource.h.

Referenced by deserializeEventCommon().

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Static Protected Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ StreamerInputSource()

◆ ~StreamerInputSource()

Member Function Documentation

◆ buildClassCache()

◆ declareStreamers()

◆ deserializeAndMergeWithRegistry()

◆ deserializeEvent()

◆ deserializeEventCommon()

◆ deserializeEventMetaData()

◆ deserializeRegistry()

◆ eventMetaDataChecksum()

◆ fillDescription()

◆ isBufferLZMA()

◆ isBufferZSTD()

◆ mergeIntoRegistry()

◆ presentEventMetaDataChecksum()

◆ read()

◆ resetAfterEndRun()

◆ setRun()

◆ uncompressBuffer()

◆ uncompressBufferLZMA()

◆ uncompressBufferZSTD()

◆ updateEventMetaData()

Member Data Documentation

◆ adjustEventToNewProductRegistry_

◆ dest_

◆ eventMetaDataChecksum_

◆ eventPrincipalHolder_

◆ processName_

◆ protocolVersion_

◆ sendEvent_

◆ streamToEventPrincipalHolders_

◆ tc_

◆ xbuf_