#include <EvFOutputModule.h>

Inheritance diagram for evf::EvFOutputModule:

Public Member Functions
	EvFOutputModule (edm::ParameterSet const &ps)

	~EvFOutputModule () override

Public Member Functions inherited from edm::one::OutputModule< T >
SerialTaskQueue *	globalLuminosityBlocksQueue () final

SerialTaskQueue *	globalRunsQueue () final

const OutputModule &	operator= (const OutputModule &)=delete

	OutputModule (edm::ParameterSet const &iPSet)

	OutputModule (const OutputModule &)=delete

bool	wantsGlobalLuminosityBlocks () const final

bool	wantsGlobalRuns () const final

bool	wantsInputProcessBlocks () const final

bool	wantsProcessBlocks () const final

Public Member Functions inherited from edm::one::OutputModuleBase
OutputModuleBase &	operator= (OutputModuleBase const &)=delete

	OutputModuleBase (ParameterSet const &pset)

	OutputModuleBase (OutputModuleBase const &)=delete

SharedResourcesAcquirer &	sharedResourcesAcquirer ()

SubProcessParentageHelper const *	subProcessParentageHelper () const

bool	wantsStreamLuminosityBlocks () const

bool	wantsStreamRuns () const

Public Member Functions inherited from edm::core::OutputModuleCore
BranchIDLists const *	branchIDLists () const

void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)

std::array< bool, NumBranchTypes > const &	hasNewlyDroppedBranch () const

SelectedProductsForBranchType const &	keptProducts () const

int	maxEvents () const

const ModuleDescription &	moduleDescription () const

OutputModuleCore &	operator= (OutputModuleCore const &)=delete

	OutputModuleCore (ParameterSet const &pset)

	OutputModuleCore (OutputModuleCore const &)=delete

OutputProcessBlockHelper const &	outputProcessBlockHelper () const

std::string const &	processName () const

int	remainingEvents () const

bool	selected (BranchDescription const &desc) const

void	selectProducts (ProductRegistry const &preg, ThinnedAssociationsHelper const &, ProcessBlockHelperBase const &)

ThinnedAssociationsHelper const *	thinnedAssociationsHelper () const

bool	wantAllEvents () const

	~OutputModuleCore () override

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ESResolverIndex const *	esGetTokenIndices (edm::Transition iTrans) const

std::vector< ESResolverIndex > const &	esGetTokenIndicesVector (edm::Transition iTrans) const

std::vector< ESRecordIndex > const &	esGetTokenRecordIndicesVector (edm::Transition iTrans) const

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::array< std::vector< ModuleDescription const > , NumBranchTypes > &modulesAll, std::vector< ModuleProcessName > &modulesInPreviousProcesses, ProductRegistry const &preg, std::map< std::string, ModuleDescription const *> const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

void	selectInputProcessBlocks (ProductRegistry const &productRegistry, ProcessBlockHelperBase const &processBlockHelperBase)

ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

void	updateLookup (eventsetup::ESRecordsToProductResolverIndices const &)

virtual	~EDConsumerBase () noexcept(false)

Static Public Member Functions
static void	fillDescriptions (edm::ConfigurationDescriptions &descriptions)

Static Public Member Functions inherited from edm::core::OutputModuleCore
static const std::string &	baseType ()

static void	fillDescription (ParameterSetDescription &desc, std::vector< std::string > const &iDefaultOutputCommands=ProductSelectorRules::defaultSelectionStrings())

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &)

Private Member Functions
void	beginRun (edm::RunForOutput const &run) override

void	endRun (edm::RunForOutput const &) override

Trig	getTriggerResults (edm::EDGetTokenT< edm::TriggerResults > const &token, edm::EventForOutput const &e) const

std::shared_ptr< EvFOutputEventWriter >	globalBeginLuminosityBlock (edm::LuminosityBlockForOutput const &iLB) const override

void	globalEndLuminosityBlock (edm::LuminosityBlockForOutput const &iLB) override

void	write (edm::EventForOutput const &e) override

void	writeLuminosityBlock (edm::LuminosityBlockForOutput const &) override

void	writeRun (edm::RunForOutput const &) override

Private Attributes
evf::FastMonitoringService *	fms_

std::unique_ptr< evf::EvFOutputJSONWriter >	jsonWriter_

edm::ParameterSet const &	ps_

edm::EDGetTokenT< edm::SendJobHeader::ParameterSetMap >	psetToken_

std::string	streamLabel_

edm::EDGetTokenT< edm::TriggerResults >	trToken_

Additional Inherited Members
Public Types inherited from edm::one::OutputModuleBase
typedef OutputModuleBase	ModuleType

Public Types inherited from edm::core::OutputModuleCore
typedef OutputModuleCore	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Protected Member Functions inherited from edm::one::OutputModuleBase
void	configure (OutputModuleDescription const &desc)

void	doBeginJob ()

bool	doEvent (EventTransitionInfo const &, ActivityRegistry , ModuleCallingContext const )

void	doPreallocate (PreallocationConfiguration const &)

Protected Member Functions inherited from edm::core::OutputModuleCore
void	configure (OutputModuleDescription const &desc)

ModuleDescription const &	description () const

void	doAccessInputProcessBlock (ProcessBlockPrincipal const &, ModuleCallingContext const *)

void	doBeginJob_ ()

bool	doBeginLuminosityBlock (LumiTransitionInfo const &, ModuleCallingContext const *)

void	doBeginProcessBlock (ProcessBlockPrincipal const &, ModuleCallingContext const *)

bool	doBeginRun (RunTransitionInfo const &, ModuleCallingContext const *)

void	doEndJob ()

bool	doEndLuminosityBlock (LumiTransitionInfo const &, ModuleCallingContext const *)

void	doEndProcessBlock (ProcessBlockPrincipal const &, ModuleCallingContext const *)

bool	doEndRun (RunTransitionInfo const &, ModuleCallingContext const *)

bool	doEvent_ (EventTransitionInfo const &, ActivityRegistry , ModuleCallingContext const )

void	doPreallocate_ (PreallocationConfiguration const &)

std::map< BranchID::value_type, BranchID::value_type > const &	droppedBranchIDToKeptBranchID ()

virtual void	preallocLumis (unsigned int)

ParameterSetID	selectorConfig () const

void	setEventSelectionInfo (std::map< std::string, std::vector< std::pair< std::string, int >>> const &outputModulePathPositions, bool anyProductProduced)

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

template<BranchType B = InEvent>
EDConsumerBaseAdaptor< B >	consumes (edm::InputTag tag) noexcept

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes ()

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag const &tag)

template<Transition Tr = Transition::Event>
constexpr auto	esConsumes ()

template<Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag tag)

template<Transition Tr = Transition::Event>
ESGetTokenGeneric	esConsumes (eventsetup::EventSetupRecordKey const &iRecord, eventsetup::DataKey const &iKey)
	Used with EventSetupRecord::doGet. More...

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

void	resetItemsToGetFrom (BranchType iType)

Protected Attributes inherited from edm::core::OutputModuleCore
std::atomic< int >	remainingEvents_

Detailed Description

Definition at line 74 of file EvFOutputModule.h.

Constructor & Destructor Documentation

◆ EvFOutputModule()

evf::EvFOutputModule::EvFOutputModule ( edm::ParameterSet const & ps )

explicit

Definition at line 109 of file EvFOutputModule.cc.

References Exception, fms_, streamLabel_, AlCaHLTBitMon_QueryRunRegistry::string, and to_lower().

       : edm::one::OutputModuleBase(ps),
         EvFOutputModuleType(ps),
         ps_(ps),
         streamLabel_(ps.getParameter<std::string>("@module_label")),
         trToken_(consumes<edm::TriggerResults>(edm::InputTag("TriggerResults"))),
         psetToken_(consumes<edm::SendJobHeader::ParameterSetMap, edm::InRun>(
             ps.getUntrackedParameter<edm::InputTag>("psetMap"))) {
     //replace hltOutoputA with stream if the HLT menu uses this convention
     std::string testPrefix = "hltOutput";
     if (streamLabel_.find(testPrefix) == 0)
       streamLabel_ = std::string("stream") + streamLabel_.substr(testPrefix.size());
 
     if (streamLabel_.find('_') != std::string::npos) {
       throw cms::Exception("EvFOutputModule") << "Underscore character is reserved can not be used for stream names in "
                                                  "FFF, but was detected in stream name -: "
                                               << streamLabel_;
     }
 
     std::string streamLabelLow = streamLabel_;
     boost::algorithm::to_lower(streamLabelLow);
     auto streampos = streamLabelLow.rfind("stream");
     if (streampos != 0 && streampos != std::string::npos)
       throw cms::Exception("EvFOutputModule")
           << "stream (case-insensitive) sequence was found in stream suffix. This is reserved and can not be used for "
              "names in FFF based HLT, but was detected in stream name";
 
     fms_ = (evf::FastMonitoringService*)(edm::Service<evf::MicroStateService>().operator->());
   }

◆ ~EvFOutputModule()

evf::EvFOutputModule::~EvFOutputModule ( )

override

Definition at line 139 of file EvFOutputModule.cc.

139 {}

Member Function Documentation

◆ beginRun()

void evf::EvFOutputModule::beginRun ( edm::RunForOutput const & run )

overrideprivate

Definition at line 150 of file EvFOutputModule.cc.

References cms::Adler32(), StreamerOutputFile::adler32(), edm::core::OutputModuleCore::branchIDLists(), StreamerOutputFile::close(), edm::core::OutputModuleCore::description(), Exception, edm::InEvent, edm::Hash< I >::isValid(), jsonWriter_, edm::core::OutputModuleCore::keptProducts(), LogDebug, edm::ModuleDescription::mainParameterSetID(), edm::core::OutputModuleCore::moduleDescription(), edm::ModuleDescription::moduleLabel(), SimL1EmulatorRepack_CalouGT_cff::processName, ps_, psetToken_, writedatasetfile::run, TrackRefitter_38T_cff::src, edm_modernize_messagelogger::stat, streamLabel_, AlCaHLTBitMon_QueryRunRegistry::string, edm::core::OutputModuleCore::thinnedAssociationsHelper(), and StreamerOutputFile::write().

                                                            {
     //create run Cache holding JSON file writer and variables
     jsonWriter_ = std::make_unique<EvFOutputJSONWriter>(
         ps_, &keptProducts()[edm::InEvent], streamLabel_, description().moduleLabel());
 
     //output INI file (non-const). This doesn't require globalBeginRun to be finished
     const std::string openIniFileName = edm::Service<evf::EvFDaqDirector>()->getOpenInitFilePath(streamLabel_);
     edm::LogInfo("EvFOutputModule") << "beginRun init stream -: " << openIniFileName;
 
     StreamerOutputFile stream_writer_preamble(openIniFileName);
     uint32 preamble_adler32 = 1;
     edm::BranchIDLists const* bidlPtr = branchIDLists();
 
     auto psetMapHandle = run.getHandle(psetToken_);
 
     std::unique_ptr<InitMsgBuilder> init_message =
         jsonWriter_->streamerCommon_.serializeRegistry(*jsonWriter_->streamerCommon_.getSerializerBuffer(),
                                                        *bidlPtr,
                                                        *thinnedAssociationsHelper(),
                                                        OutputModule::processName(),
                                                        description().moduleLabel(),
                                                        moduleDescription().mainParameterSetID(),
                                                        psetMapHandle.isValid() ? psetMapHandle.product() : nullptr);
 
     //Let us turn it into a View
     InitMsgView view(init_message->startAddress());
 
     //output header
     stream_writer_preamble.write(view);
     preamble_adler32 = stream_writer_preamble.adler32();
     stream_writer_preamble.close();
 
     struct stat istat;
     stat(openIniFileName.c_str(), &istat);
     //read back file to check integrity of what was written
     off_t readInput = 0;
     uint32_t adlera = 1, adlerb = 0;
     FILE* src = fopen(openIniFileName.c_str(), "r");
 
     //allocate buffer to write INI file
     unsigned char* outBuf = new unsigned char[1024 * 1024];
     while (readInput < istat.st_size) {
       size_t toRead = readInput + 1024 * 1024 < istat.st_size ? 1024 * 1024 : istat.st_size - readInput;
       fread(outBuf, toRead, 1, src);
       cms::Adler32((const char*)outBuf, toRead, adlera, adlerb);
       readInput += toRead;
     }
     fclose(src);
 
     //clear serialization buffers
     jsonWriter_->streamerCommon_.getSerializerBuffer()->clearHeaderBuffer();
 
     //free output buffer needed only for the file write
     delete[] outBuf;
     outBuf = nullptr;
 
     uint32_t adler32c = (adlerb << 16) | adlera;
     if (adler32c != preamble_adler32) {
       throw cms::Exception("EvFOutputModule") << "Checksum mismatch of ini file -: " << openIniFileName
                                               << " expected:" << preamble_adler32 << " obtained:" << adler32c;
     } else {
       LogDebug("EvFOutputModule") << "Ini file checksum -: " << streamLabel_ << " " << adler32c;
       std::filesystem::rename(openIniFileName, edm::Service<evf::EvFDaqDirector>()->getInitFilePath(streamLabel_));
     }
   }

◆ endRun()

void evf::EvFOutputModule::endRun ( edm::RunForOutput const & )

inlineoverrideprivate

Definition at line 87 of file EvFOutputModule.h.

87 {}

◆ fillDescriptions()

void evf::EvFOutputModule::fillDescriptions ( edm::ConfigurationDescriptions & descriptions )

static

Definition at line 141 of file EvFOutputModule.cc.

References edm::ConfigurationDescriptions::add(), submitPVResolutionJobs::desc, edm::StreamerOutputModuleCommon::fillDescription(), and edm::core::OutputModuleCore::fillDescription().

                                                                                    {
     edm::ParameterSetDescription desc;
     edm::StreamerOutputModuleCommon::fillDescription(desc);
     EvFOutputModuleType::fillDescription(desc);
     desc.addUntracked<edm::InputTag>("psetMap", {"hltPSetMap"})
         ->setComment("Optionally allow the map of ParameterSets to be calculated externally.");
     descriptions.add("evfOutputModule", desc);
   }

◆ getTriggerResults()

Trig evf::EvFOutputModule::getTriggerResults	(	edm::EDGetTokenT< edm::TriggerResults > const &	token,
		edm::EventForOutput const &	e
	)		const

private

Definition at line 216 of file EvFOutputModule.cc.

References MillePedeFileConverter_cfg::e, mps_fire::result, and unpackBuffers-CaloStage2::token.

Referenced by write().

                                                                             {
     Trig result;
     e.getByToken<edm::TriggerResults>(token, result);
     return result;
   }

◆ globalBeginLuminosityBlock()

std::shared_ptr< EvFOutputEventWriter > evf::EvFOutputModule::globalBeginLuminosityBlock ( edm::LuminosityBlockForOutput const & iLB ) const

overrideprivate

Definition at line 223 of file EvFOutputModule.cc.

References edm::LuminosityBlockForOutput::luminosityBlock(), and streamLabel_.

                                                     {
     auto openDatFilePath = edm::Service<evf::EvFDaqDirector>()->getOpenDatFilePath(iLB.luminosityBlock(), streamLabel_);
 
     return std::make_shared<EvFOutputEventWriter>(openDatFilePath);
   }

◆ globalEndLuminosityBlock()

void evf::EvFOutputModule::globalEndLuminosityBlock ( edm::LuminosityBlockForOutput const & iLB )

overrideprivate

Definition at line 251 of file EvFOutputModule.cc.

References fms_, evf::FastMonitoringService::getEventsProcessedForLumi(), edm::LuminosityBlockForOutput::index(), jsonWriter_, edm::LuminosityBlockForOutput::luminosityBlock(), castor_dqm_sourceclient_file_cfg::path, edm_modernize_messagelogger::stat, streamLabel_, and AlCaHLTBitMon_QueryRunRegistry::string.

                                                                                        {
     auto lumiWriter = luminosityBlockCache(iLB.index());
     //close dat file
     lumiWriter->close();
 
     jsonWriter_->fileAdler32_.value() = lumiWriter->get_adler32();
     jsonWriter_->accepted_.value() = lumiWriter->getAccepted();
 
     bool abortFlag = false;
     jsonWriter_->processed_.value() = fms_->getEventsProcessedForLumi(iLB.luminosityBlock(), &abortFlag);
     if (abortFlag) {
       edm::LogInfo("EvFOutputModule") << "Abort flag has been set. Output is suppressed";
       return;
     }
 
     if (jsonWriter_->processed_.value() != 0) {
       struct stat istat;
       std::filesystem::path openDatFilePath = lumiWriter->getFilePath();
       stat(openDatFilePath.string().c_str(), &istat);
       jsonWriter_->filesize_ = istat.st_size;
       std::filesystem::rename(openDatFilePath.string().c_str(),
                               edm::Service<evf::EvFDaqDirector>()->getDatFilePath(iLB.luminosityBlock(), streamLabel_));
       jsonWriter_->filelist_ = openDatFilePath.filename().string();
     } else {
       //remove empty file when no event processing has occurred
       remove(lumiWriter->getFilePath().c_str());
       jsonWriter_->filesize_ = 0;
       jsonWriter_->filelist_ = "";
       jsonWriter_->fileAdler32_.value() = -1;  //no files in signed long
     }
 
     //produce JSON file
     jsonWriter_->jsonMonitor_->snap(iLB.luminosityBlock());
     const std::string outputJsonNameStream =
         edm::Service<evf::EvFDaqDirector>()->getOutputJsonFilePath(iLB.luminosityBlock(), streamLabel_);
     jsonWriter_->jsonMonitor_->outputFullJSON(outputJsonNameStream, iLB.luminosityBlock());
   }

◆ write()

void evf::EvFOutputModule::write ( edm::EventForOutput const & e )

overrideprivatevirtual

Implements edm::core::OutputModuleCore.

Definition at line 230 of file EvFOutputModule.cc.

References MillePedeFileConverter_cfg::e, getTriggerResults(), jsonWriter_, svgfig::load(), mps_check::msg, edm::core::OutputModuleCore::selectorConfig(), edm::shutdown_flag, triggerResults, and trToken_.

Referenced by pkg.AbstractPkg::generate().

                                                         {
     unsigned int counter = 0;
     while (edm::Service<evf::EvFDaqDirector>()->inputThrottled()) {
       if (edm::shutdown_flag.load(std::memory_order_relaxed))
         break;
       if (!(counter % 100))
         edm::LogWarning("FedRawDataInputSource") << "Input throttled detected, writing is paused...";
       usleep(100000);
       counter++;
     }
 
     edm::Handle<edm::TriggerResults> const& triggerResults = getTriggerResults(trToken_, e);
 
     //auto lumiWriter = const_cast<EvFOutputEventWriter*>(luminosityBlockCache(e.getLuminosityBlock().index() ));
     auto lumiWriter = luminosityBlockCache(e.getLuminosityBlock().index());
     std::unique_ptr<EventMsgBuilder> msg = jsonWriter_->streamerCommon_.serializeEvent(
         *jsonWriter_->streamerCommon_.getSerializerBuffer(), e, triggerResults, selectorConfig());
     lumiWriter->incAccepted();
     lumiWriter->doOutputEvent(*msg);  //msg is written and discarded at this point
   }

◆ writeLuminosityBlock()

void evf::EvFOutputModule::writeLuminosityBlock ( edm::LuminosityBlockForOutput const & )

inlineoverrideprivatevirtual

Implements edm::core::OutputModuleCore.

Definition at line 85 of file EvFOutputModule.h.

85 {}

◆ writeRun()

void evf::EvFOutputModule::writeRun ( edm::RunForOutput const & )

inlineoverrideprivatevirtual

Implements edm::core::OutputModuleCore.

Definition at line 86 of file EvFOutputModule.h.

86 {}

Member Data Documentation

◆ fms_

evf::FastMonitoringService* evf::EvFOutputModule::fms_

private

Definition at line 100 of file EvFOutputModule.h.

Referenced by EvFOutputModule(), and globalEndLuminosityBlock().

◆ jsonWriter_

std::unique_ptr<evf::EvFOutputJSONWriter> evf::EvFOutputModule::jsonWriter_

private

Definition at line 102 of file EvFOutputModule.h.

Referenced by beginRun(), globalEndLuminosityBlock(), and write().

◆ ps_

edm::ParameterSet const& evf::EvFOutputModule::ps_

private

Definition at line 95 of file EvFOutputModule.h.

Referenced by beginRun().

◆ psetToken_

edm::EDGetTokenT<edm::SendJobHeader::ParameterSetMap> evf::EvFOutputModule::psetToken_

private

Definition at line 98 of file EvFOutputModule.h.

Referenced by beginRun().

◆ streamLabel_

std::string evf::EvFOutputModule::streamLabel_

private

Definition at line 96 of file EvFOutputModule.h.

Referenced by beginRun(), EvFOutputModule(), globalBeginLuminosityBlock(), and globalEndLuminosityBlock().

◆ trToken_

edm::EDGetTokenT<edm::TriggerResults> evf::EvFOutputModule::trToken_

private

Definition at line 97 of file EvFOutputModule.h.

Referenced by write().

Public Member Functions

Static Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ EvFOutputModule()

◆ ~EvFOutputModule()

Member Function Documentation

◆ beginRun()

◆ endRun()

◆ fillDescriptions()

◆ getTriggerResults()

◆ globalBeginLuminosityBlock()

◆ globalEndLuminosityBlock()

◆ write()

◆ writeLuminosityBlock()

◆ writeRun()

Member Data Documentation

◆ fms_

◆ jsonWriter_

◆ ps_

◆ psetToken_

◆ streamLabel_

◆ trToken_