#include <RecoEventOutputModuleForFU.h>

Inheritance diagram for evf::RecoEventOutputModuleForFU< Consumer >:

Public Member Functions
	RecoEventOutputModuleForFU (edm::ParameterSet const &ps)

	~RecoEventOutputModuleForFU () override

Public Member Functions inherited from edm::StreamerOutputModuleBase
	StreamerOutputModuleBase (ParameterSet const &ps)

	~StreamerOutputModuleBase () override

Public Member Functions inherited from edm::one::OutputModule< one::WatchRuns, one::WatchLuminosityBlocks >
	OutputModule (edm::ParameterSet const &iPSet)

bool	wantsGlobalLuminosityBlocks () const final

bool	wantsGlobalRuns () const final

Public Member Functions inherited from edm::one::OutputModuleBase
BranchIDLists const *	branchIDLists ()

SerialTaskQueue *	globalLuminosityBlocksQueue ()

SerialTaskQueue *	globalRunsQueue ()

std::array< bool, NumBranchTypes > const &	hasNewlyDroppedBranch () const

SelectedProductsForBranchType const &	keptProducts () const

int	maxEvents () const

const ModuleDescription &	moduleDescription () const

OutputModuleBase &	operator= (OutputModuleBase const &)=delete

	OutputModuleBase (ParameterSet const &pset)

	OutputModuleBase (OutputModuleBase const &)=delete

std::string const &	processName () const

int	remainingEvents () const

bool	selected (BranchDescription const &desc) const

void	selectProducts (ProductRegistry const &preg, ThinnedAssociationsHelper const &)

SharedResourcesAcquirer &	sharedResourcesAcquirer ()

SubProcessParentageHelper const *	subProcessParentageHelper () const

ThinnedAssociationsHelper const *	thinnedAssociationsHelper () const

bool	wantAllEvents () const

bool	wantsStreamLuminosityBlocks () const

bool	wantsStreamRuns () const

	~OutputModuleBase () override

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

bool	registeredToConsumeMany (TypeID const &, BranchType) const

ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

virtual	~EDConsumerBase () noexcept(false)

Static Public Member Functions
static void	fillDescriptions (edm::ConfigurationDescriptions &descriptions)

Static Public Member Functions inherited from edm::StreamerOutputModuleBase
static void	fillDescription (ParameterSetDescription &desc)

Static Public Member Functions inherited from edm::one::OutputModuleBase
static const std::string &	baseType ()

static void	fillDescription (ParameterSetDescription &desc)

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &)

Private Member Functions
void	beginJob () override

void	beginLuminosityBlock (edm::LuminosityBlockForOutput const &) override

void	doOutputEvent (EventMsgBuilder const &msg) override

void	doOutputHeader (InitMsgBuilder const &init_message) override

void	endLuminosityBlock (edm::LuminosityBlockForOutput const &) override

void	initRun ()

void	start () override

void	stop () override

Private Attributes
jsoncollector::IntJ	accepted_

std::auto_ptr< Consumer >	c_

jsoncollector::IntJ	errorEvents_

jsoncollector::IntJ	fileAdler32_

jsoncollector::StringJ	filelist_

jsoncollector::IntJ	filesize_

evf::FastMonitoringService *	fms_

jsoncollector::IntJ	hltErrorEvents_

jsoncollector::StringJ	inputFiles_

boost::shared_ptr< jsoncollector::FastMonitor >	jsonMonitor_

jsoncollector::StringJ	mergeType_

boost::filesystem::path	openDatChecksumFilePath_

boost::filesystem::path	openDatFilePath_

unsigned char *	outBuf_ = 0

jsoncollector::DataPointDefinition	outJsonDef_

jsoncollector::IntJ	processed_

bool	readAdler32Check_ =false

jsoncollector::IntJ	retCodeMask_

std::string	streamLabel_

jsoncollector::StringJ	transferDestination_

Additional Inherited Members
Public Types inherited from edm::one::OutputModuleBase
typedef OutputModuleBase	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Protected Member Functions inherited from edm::one::OutputModuleBase
void	configure (OutputModuleDescription const &desc)

ModuleDescription const &	description () const

void	doBeginJob ()

bool	doBeginLuminosityBlock (LuminosityBlockPrincipal const &lbp, EventSetup const &c, ModuleCallingContext const *)

bool	doBeginRun (RunPrincipal const &rp, EventSetup const &c, ModuleCallingContext const *)

void	doEndJob ()

bool	doEndLuminosityBlock (LuminosityBlockPrincipal const &lbp, EventSetup const &c, ModuleCallingContext const *)

bool	doEndRun (RunPrincipal const &rp, EventSetup const &c, ModuleCallingContext const *)

bool	doEvent (EventPrincipal const &ep, EventSetup const &c, ActivityRegistry , ModuleCallingContext const )

void	doPreallocate (PreallocationConfiguration const &)

std::map< BranchID::value_type, BranchID::value_type > const &	droppedBranchIDToKeptBranchID ()

ParameterSetID	selectorConfig () const

void	setEventSelectionInfo (std::map< std::string, std::vector< std::pair< std::string, int > > > const &outputModulePathPositions, bool anyProductProduced)

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()

void	consumesMany (const TypeToGet &id)

template<BranchType B>
void	consumesMany (const TypeToGet &id)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Detailed Description

template<typename Consumer>
class evf::RecoEventOutputModuleForFU< Consumer >

Definition at line 26 of file RecoEventOutputModuleForFU.h.

Constructor & Destructor Documentation

template<typename Consumer >

evf::RecoEventOutputModuleForFU< Consumer >::RecoEventOutputModuleForFU ( edm::ParameterSet const & ps )

explicit

Consumers are supposed to provide void doOutputHeader(InitMsgBuilder const& init_message) void doOutputEvent(EventMsgBuilder const& msg) void start() void stop() static void fillDescription(ParameterSetDescription&)

Definition at line 76 of file RecoEventOutputModuleForFU.h.

References Exception, evf::RecoEventOutputModuleForFU< Consumer >::fms_, evf::RecoEventOutputModuleForFU< Consumer >::streamLabel_, and AlCaHLTBitMon_QueryRunRegistry::string.

                                                                                             :
     edm::one::OutputModuleBase::OutputModuleBase(ps),
     edm::StreamerOutputModuleBase(ps),
     c_(new Consumer(ps)),
     streamLabel_(ps.getParameter<std::string>("@module_label")),
     processed_(0),
     accepted_(0),
     errorEvents_(0),
     retCodeMask_(0),
     filelist_(),
     filesize_(0),
     inputFiles_(),
     fileAdler32_(1),
     transferDestination_(),
     mergeType_(),
     hltErrorEvents_(0),
     outBuf_(new unsigned char[1024*1024])
   {
     //replace hltOutoputA with stream if the HLT menu uses this convention
     std::string testPrefix="hltOutput";
     if (streamLabel_.find(testPrefix)==0) 
       streamLabel_=std::string("stream")+streamLabel_.substr(testPrefix.size());
 
     if (streamLabel_.find("_")!=std::string::npos) {
       throw cms::Exception("RecoEventOutputModuleForFU")
         << "Underscore character is reserved can not be used for stream names in FFF, but was detected in stream name -: " << streamLabel_;
     }
 
     std::string streamLabelLow = streamLabel_;
     boost::algorithm::to_lower(streamLabelLow);
     auto streampos = streamLabelLow.rfind("stream");
     if (streampos !=0 && streampos!=std::string::npos)
       throw cms::Exception("RecoEventOutputModuleForFU")
         << "stream (case-insensitive) sequence was found in stream suffix. This is reserved and can not be used for names in FFF based HLT, but was detected in stream name";
 
     fms_ = (evf::FastMonitoringService *)(edm::Service<evf::MicroStateService>().operator->());
   }

template<typename Consumer >

evf::RecoEventOutputModuleForFU< Consumer >::~RecoEventOutputModuleForFU ( )

override

Definition at line 182 of file RecoEventOutputModuleForFU.h.

182 {}

Member Function Documentation

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::beginJob ( void )

overrideprivatevirtual

Reimplemented from edm::one::OutputModuleBase.

Definition at line 268 of file RecoEventOutputModuleForFU.h.

References evf::RecoEventOutputModuleForFU< Consumer >::mergeType_, evf::MergeTypeDAT, evf::RecoEventOutputModuleForFU< Consumer >::streamLabel_, and evf::RecoEventOutputModuleForFU< Consumer >::transferDestination_.

   {
     //get stream transfer destination
     transferDestination_ = edm::Service<evf::EvFDaqDirector>()->getStreamDestinations(streamLabel_);
     mergeType_ = edm::Service<evf::EvFDaqDirector>()->getStreamMergeType(streamLabel_,evf::MergeTypeDAT);
   }

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::beginLuminosityBlock ( edm::LuminosityBlockForOutput const & ls )

overrideprivate

Definition at line 277 of file RecoEventOutputModuleForFU.h.

References evf::RecoEventOutputModuleForFU< Consumer >::c_, evf::RecoEventOutputModuleForFU< Consumer >::filelist_, edm::LuminosityBlockForOutput::luminosityBlock(), evf::RecoEventOutputModuleForFU< Consumer >::openDatChecksumFilePath_, evf::RecoEventOutputModuleForFU< Consumer >::openDatFilePath_, and evf::RecoEventOutputModuleForFU< Consumer >::streamLabel_.

   {
     //edm::LogInfo("RecoEventOutputModuleForFU") << "begin lumi";
     openDatFilePath_ = edm::Service<evf::EvFDaqDirector>()->getOpenDatFilePath(ls.luminosityBlock(),streamLabel_);
     openDatChecksumFilePath_ = edm::Service<evf::EvFDaqDirector>()->getOpenDatFilePath(ls.luminosityBlock(),streamLabel_);
     c_->setOutputFile(openDatFilePath_.string());
     filelist_ = openDatFilePath_.filename().string();
   }

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::doOutputEvent ( EventMsgBuilder const & msg )

overrideprivatevirtual

Implements edm::StreamerOutputModuleBase.

Definition at line 242 of file RecoEventOutputModuleForFU.h.

References evf::RecoEventOutputModuleForFU< Consumer >::accepted_, evf::RecoEventOutputModuleForFU< Consumer >::c_, and jsoncollector::IntJ::value().

                                                                                 {
     accepted_.value()++;
     c_->doOutputEvent(msg); // You can't use msg in RecoEventOutputModuleForFU after this point
   }

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::doOutputHeader ( InitMsgBuilder const & init_message )

overrideprivatevirtual

Implements edm::StreamerOutputModuleBase.

Definition at line 206 of file RecoEventOutputModuleForFU.h.

References cms::Adler32(), evf::RecoEventOutputModuleForFU< Consumer >::c_, Exception, LogDebug, evf::RecoEventOutputModuleForFU< Consumer >::outBuf_, TrackRefitter_38T_cff::src, trackingPlots::stat, evf::RecoEventOutputModuleForFU< Consumer >::streamLabel_, and AlCaHLTBitMon_QueryRunRegistry::string.

   {
     c_->doOutputHeader(init_message);
 
     const std::string openIniFileName = edm::Service<evf::EvFDaqDirector>()->getOpenInitFilePath(streamLabel_);
     struct stat istat;
     stat(openIniFileName.c_str(), &istat);
     //read back file to check integrity of what was written
     off_t readInput=0;
     uint32_t adlera=1,adlerb=0;
     FILE *src = fopen(openIniFileName.c_str(),"r");
     while (readInput<istat.st_size)
     {
       size_t toRead=  readInput+1024*1024 < istat.st_size ? 1024*1024 : istat.st_size-readInput;
       fread(outBuf_,toRead,1,src);
       cms::Adler32((const char*)outBuf_,toRead,adlera,adlerb);
       readInput+=toRead;
     }
     fclose(src);
     //free output buffer needed only for the INI file
     delete [] outBuf_;
     outBuf_=nullptr;
 
     uint32_t adler32c = (adlerb << 16) | adlera;
     if (adler32c != c_->get_adler32_ini()) {
       throw cms::Exception("RecoEventOutputModuleForFU") << "Checksum mismatch of ini file -: " << openIniFileName
                            << " expected:" << c_->get_adler32_ini() << " obtained:" << adler32c;
     }
     else {
       LogDebug("RecoEventOutputModuleForFU") << "Ini file checksum -: "<< streamLabel_ << " " << adler32c;
       boost::filesystem::rename(openIniFileName,edm::Service<evf::EvFDaqDirector>()->getInitFilePath(streamLabel_));
     }
   }

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::endLuminosityBlock ( edm::LuminosityBlockForOutput const & ls )

overrideprivate

Definition at line 287 of file RecoEventOutputModuleForFU.h.

   {
     //edm::LogInfo("RecoEventOutputModuleForFU") << "end lumi";
     long filesize=0;
     fileAdler32_.value() = c_->get_adler32();
     c_->closeOutputFile();
     bool abortFlag = false;
     processed_.value() = fms_->getEventsProcessedForLumi(ls.luminosityBlock(),&abortFlag);
 
     if (abortFlag) {
         edm::LogInfo("RecoEventOutputModuleForFU") << "output suppressed";
         return;
     }
     
     if(processed_.value()!=0) {
       //lock
       struct stat istat;
       stat(openDatFilePath_.string().c_str(), &istat);
       filesize = istat.st_size;
       boost::filesystem::rename(openDatFilePath_.string().c_str(), edm::Service<evf::EvFDaqDirector>()->getDatFilePath(ls.luminosityBlock(),streamLabel_));
     } else {
       filelist_ = "";
       fileAdler32_.value()=-1;
     }
 
     //remove file
     remove(openDatFilePath_.string().c_str());
     filesize_=filesize;
 
     jsonMonitor_->snap(ls.luminosityBlock());
     const std::string outputJsonNameStream =
       edm::Service<evf::EvFDaqDirector>()->getOutputJsonFilePath(ls.luminosityBlock(),streamLabel_);
     jsonMonitor_->outputFullJSON(outputJsonNameStream,ls.luminosityBlock());
 
     // reset monitoring params
     accepted_.value() = 0;
     filelist_ = "";
   }

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::fillDescriptions ( edm::ConfigurationDescriptions & descriptions )

static

Definition at line 249 of file RecoEventOutputModuleForFU.h.

References edm::ConfigurationDescriptions::addDefault(), and edm::StreamerOutputModuleBase::fillDescription().

                                                                                                    {
     edm::ParameterSetDescription desc;
     edm::StreamerOutputModuleBase::fillDescription(desc);
     Consumer::fillDescription(desc);
     // Use addDefault here instead of add for 4 reasons:
     // 1. Because EvFOutputModule_cfi.py is explicitly defined it does not need to be autogenerated
     // The explicitly defined version overrides the autogenerated version of the cfi file.
     // 2. That cfi file is not used anywhere in the release anyway
     // 3. There are two plugin names used for the same template instantiation of this
     // type, "ShmStreamConsumer" and "EvFOutputModule" and this causes name conflict
     // problems for the cfi generation code which are avoided with addDefault.
     // 4. At the present time, there is only one type of Consumer used to instantiate
     // instances of this template, but if there were more than one type then this function
     // would need to be specialized for each type unless the descriptions were the same
     // and addDefault was used.
     descriptions.addDefault(desc);
   }

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::initRun ( )

private

Definition at line 115 of file RecoEventOutputModuleForFU.h.

Referenced by evf::RecoEventOutputModuleForFU< Consumer >::start().

   {
     std::string baseRunDir = edm::Service<evf::EvFDaqDirector>()->baseRunDir();
     readAdler32Check_ =  edm::Service<evf::EvFDaqDirector>()->outputAdler32Recheck();
     LogDebug("RecoEventOutputModuleForFU") << "writing .dat files to -: " << baseRunDir;
     // create open dir if not already there
     edm::Service<evf::EvFDaqDirector>()->createRunOpendirMaybe();
 
     processed_.setName("Processed");
     accepted_.setName("Accepted");
     errorEvents_.setName("ErrorEvents");
     retCodeMask_.setName("ReturnCodeMask");
     filelist_.setName("Filelist");
     filesize_.setName("Filesize");
     inputFiles_.setName("InputFiles");
     fileAdler32_.setName("FileAdler32");
     transferDestination_.setName("TransferDestination");
     mergeType_.setName("MergeType");
     hltErrorEvents_.setName("HLTErrorEvents");
 
     outJsonDef_.setDefaultGroup("data");
     outJsonDef_.addLegendItem("Processed","integer",jsoncollector::DataPointDefinition::SUM);
     outJsonDef_.addLegendItem("Accepted","integer",jsoncollector::DataPointDefinition::SUM);
     outJsonDef_.addLegendItem("ErrorEvents","integer",jsoncollector::DataPointDefinition::SUM);
     outJsonDef_.addLegendItem("ReturnCodeMask","integer",jsoncollector::DataPointDefinition::BINARYOR);
     outJsonDef_.addLegendItem("Filelist","string",jsoncollector::DataPointDefinition::MERGE);
     outJsonDef_.addLegendItem("Filesize","integer",jsoncollector::DataPointDefinition::SUM);
     outJsonDef_.addLegendItem("InputFiles","string",jsoncollector::DataPointDefinition::CAT);
     outJsonDef_.addLegendItem("FileAdler32","integer",jsoncollector::DataPointDefinition::ADLER32);
     outJsonDef_.addLegendItem("TransferDestination","string",jsoncollector::DataPointDefinition::SAME);
     outJsonDef_.addLegendItem("MergeType","string",jsoncollector::DataPointDefinition::SAME);
     outJsonDef_.addLegendItem("HLTErrorEvents","integer",jsoncollector::DataPointDefinition::SUM);
     std::stringstream tmpss,ss;
     tmpss << baseRunDir << "/open/" << "output_" << getpid() << ".jsd";
     ss << baseRunDir << "/" << "output_" << getpid() << ".jsd";
     std::string outTmpJsonDefName = tmpss.str();
     std::string outJsonDefName = ss.str();
 
     edm::Service<evf::EvFDaqDirector>()->lockInitLock();
     struct stat fstat;
     if (stat (outJsonDefName.c_str(), &fstat) != 0) { //file does not exist
       LogDebug("RecoEventOutputModuleForFU") << "writing output definition file -: " << outJsonDefName;
       std::string content;
       jsoncollector::JSONSerializer::serialize(&outJsonDef_,content);
       jsoncollector::FileIO::writeStringToFile(outTmpJsonDefName, content);
       boost::filesystem::rename(outTmpJsonDefName,outJsonDefName);
     }
     edm::Service<evf::EvFDaqDirector>()->unlockInitLock();
 
     jsonMonitor_.reset(new jsoncollector::FastMonitor(&outJsonDef_,true));
     jsonMonitor_->setDefPath(outJsonDefName);
     jsonMonitor_->registerGlobalMonitorable(&processed_,false);
     jsonMonitor_->registerGlobalMonitorable(&accepted_,false);
     jsonMonitor_->registerGlobalMonitorable(&errorEvents_,false);
     jsonMonitor_->registerGlobalMonitorable(&retCodeMask_,false);
     jsonMonitor_->registerGlobalMonitorable(&filelist_,false);
     jsonMonitor_->registerGlobalMonitorable(&filesize_,false);
     jsonMonitor_->registerGlobalMonitorable(&inputFiles_,false);
     jsonMonitor_->registerGlobalMonitorable(&fileAdler32_,false);
     jsonMonitor_->registerGlobalMonitorable(&transferDestination_,false);
     jsonMonitor_->registerGlobalMonitorable(&mergeType_,false);
     jsonMonitor_->registerGlobalMonitorable(&hltErrorEvents_,false);
     jsonMonitor_->commit(nullptr);
 
   }

template<typename Consumer >

void evf::RecoEventOutputModuleForFU< Consumer >::start ( )

overrideprivatevirtual

Implements edm::StreamerOutputModuleBase.

Definition at line 186 of file RecoEventOutputModuleForFU.h.

References evf::RecoEventOutputModuleForFU< Consumer >::c_, evf::RecoEventOutputModuleForFU< Consumer >::initRun(), evf::RecoEventOutputModuleForFU< Consumer >::streamLabel_, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by progressbar.ProgressBar::__next__(), Types.LuminosityBlockRange::cppID(), and Types.EventRange::cppID().

   {
     initRun();
     const std::string openInitFileName = edm::Service<evf::EvFDaqDirector>()->getOpenInitFilePath(streamLabel_);
     edm::LogInfo("RecoEventOutputModuleForFU") << "start() method, initializing streams. init stream -: "  
                                            << openInitFileName;
     c_->setInitMessageFile(openInitFileName);
     c_->start();
     
   }