#include <DQMFileSaverOnline.h>

Inheritance diagram for dqm::DQMFileSaverOnline:

Classes
struct	SnapshotFiles

Public Member Functions
	DQMFileSaverOnline (const edm::ParameterSet &ps)

	~DQMFileSaverOnline () override

Public Member Functions inherited from dqm::DQMFileSaverBase
	DQMFileSaverBase (const edm::ParameterSet &ps)

	~DQMFileSaverBase () override

Public Member Functions inherited from edm::global::EDAnalyzer< edm::RunCache< NoCache >, edm::LuminosityBlockCache< NoCache > >
	EDAnalyzer ()=default

bool	wantsGlobalLuminosityBlocks () const final

bool	wantsGlobalRuns () const final

bool	wantsStreamLuminosityBlocks () const final

bool	wantsStreamRuns () const final

Public Member Functions inherited from edm::global::EDAnalyzerBase
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)

	EDAnalyzerBase ()

ModuleDescription const &	moduleDescription () const

	~EDAnalyzerBase () override

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ESProxyIndex const *	esGetTokenIndices (edm::Transition iTrans) const

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

bool	registeredToConsumeMany (TypeID const &, BranchType) const

ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

void	updateLookup (eventsetup::ESRecordsToProxyIndices const &)

virtual	~EDConsumerBase () noexcept(false)

Static Public Member Functions
static void	fillDescriptions (edm::ConfigurationDescriptions &descriptions)

static const std::string	fillOrigin (const std::string &filename, const std::string &final_filename)

Static Public Member Functions inherited from dqm::DQMFileSaverBase
static void	fillDescription (edm::ParameterSetDescription &d)

Static Public Member Functions inherited from edm::global::EDAnalyzerBase
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &descriptions)

Protected Member Functions
void	appendSnapshot (SnapshotFiles new_snap) const

void	checkError (const char *msg, const std::string &file, int status) const

void	makeSnapshot (const FileParameters &fp, bool final) const

void	saveLumi (const FileParameters &fp) const override

void	saveRun (const FileParameters &fp) const override

Protected Member Functions inherited from dqm::DQMFileSaverBase
void	analyze (edm::StreamID, const edm::Event &e, const edm::EventSetup &) const final

std::shared_ptr< NoCache >	globalBeginLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &) const final

std::shared_ptr< NoCache >	globalBeginRun (const edm::Run &, const edm::EventSetup &) const final

void	globalEndLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &) const final

void	globalEndRun (const edm::Run &, const edm::EventSetup &) const final

virtual void	initRun () const

void	logFileAction (const std::string &msg, const std::string &fileName) const

void	saveJobReport (const std::string &filename) const

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()

void	consumesMany (const TypeToGet &id)

template<BranchType B>
void	consumesMany (const TypeToGet &id)

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes ()

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag const &tag)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Protected Attributes
int	backupLumiCount_

bool	keepBackupLumi_

std::list< SnapshotFiles >	snapshots_

std::mutex	snapshots_lock_

Protected Attributes inherited from dqm::DQMFileSaverBase
FileParameters	initial_fp_

std::mutex	initial_fp_lock_

Additional Inherited Members
Public Types inherited from dqm::DQMFileSaverBase
typedef dqm::legacy::DQMStore	DQMStore

typedef dqm::legacy::MonitorElement	MonitorElement

Public Types inherited from edm::global::EDAnalyzerBase
typedef EDAnalyzerBase	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Static Protected Member Functions inherited from dqm::DQMFileSaverBase
static const std::string	filename (const FileParameters &fp, bool useLumi=false)

Detailed Description

Definition at line 15 of file DQMFileSaverOnline.h.

Constructor & Destructor Documentation

DQMFileSaverOnline::DQMFileSaverOnline ( const edm::ParameterSet & ps )

Definition at line 30 of file DQMFileSaverOnline.cc.

References backupLumiCount_, edm::ParameterSet::getUntrackedParameter(), keepBackupLumi_, and ~DQMFileSaverOnline().

                                                                 : DQMFileSaverBase(ps) {
   backupLumiCount_ = ps.getUntrackedParameter<int>("backupLumiCount", 1);
   keepBackupLumi_ = ps.getUntrackedParameter<bool>("keepBackupLumi", false);
 }

DQMFileSaverOnline::~DQMFileSaverOnline ( )

overridedefault

Referenced by DQMFileSaverOnline().

Member Function Documentation

void DQMFileSaverOnline::appendSnapshot ( SnapshotFiles new_snap ) const

protected

Definition at line 109 of file DQMFileSaverOnline.cc.

References checkError(), dqm::DQMFileSaverOnline::SnapshotFiles::data, keepBackupLumi_, CommonMethods::lock(), dqm::DQMFileSaverBase::logFileAction(), dqm::DQMFileSaverOnline::SnapshotFiles::meta, snapshots_, and snapshots_lock_.

Referenced by makeSnapshot().

                                                              {
   std::lock_guard<std::mutex> lock(snapshots_lock_);
 
   if (!keepBackupLumi_) {
     while (!snapshots_.empty()) {
       SnapshotFiles& x = snapshots_.front();
 
       // logFileAction("Deleting old snapshot (origin): ", x.meta);
       checkError("Unlink failed: ", x.meta, ::unlink(x.meta.c_str()));
 
       logFileAction("Deleting old snapshot (root): ", x.data);
       checkError("Unlink failed: ", x.data, ::unlink(x.data.c_str()));
 
       snapshots_.pop_front();
     }
   }
 
   if (!f.data.empty()) {
     snapshots_.push_back(f);
   }
 }

void DQMFileSaverOnline::checkError	(	const char *	msg,
		const std::string &	file,
		int	status
	)		const

protected

Definition at line 131 of file DQMFileSaverOnline.cc.

References dqm::DQMFileSaverBase::logFileAction(), mps_check::msg, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by appendSnapshot(), and makeSnapshot().

                                                                                             {
   if (status != 0) {
     std::string actual_msg = msg;
     actual_msg += std::strerror(status);
     logFileAction(actual_msg, file);
   }
 }

void DQMFileSaverOnline::fillDescriptions ( edm::ConfigurationDescriptions & descriptions )

static

Definition at line 159 of file DQMFileSaverOnline.cc.

References edm::ConfigurationDescriptions::addDefault(), edm::ParameterSetDescription::addUntracked(), DEFINE_FWK_MODULE, dqm::DQMFileSaverBase::fillDescription(), and edm::ParameterSetDescription::setComment().

                                                                                     {
   edm::ParameterSetDescription desc;
   desc.setComment("Saves histograms from DQM store, online workflow.");
 
   desc.addUntracked<int>("backupLumiCount", 10)
       ->setComment(
           "How often the backup file will be generated, in lumisections (-1 "
           "disables).");
 
   desc.addUntracked<bool>("keepBackupLumi", false)
       ->setComment(
           "Usually the backup old backup is deleted once the new file is "
           "available. Setting this to true ensures that no backup files are "
           "ever deleted. Useful for ML applications, which use backups as a "
           "'history' of what happened during the run.");
 
   DQMFileSaverBase::fillDescription(desc);
 
   // Changed to use addDefault instead of add here because previously
   // DQMFileSaverOnline and DQMFileSaverPB both used the module label
   // "saver" which caused conflicting cfi filenames to be generated.
   // add could be used if unique module labels were given.
   descriptions.addDefault(desc);
 }

const std::string DQMFileSaverOnline::fillOrigin	(	const std::string &	filename,
		const std::string &	final_filename
	)

static

Definition at line 139 of file DQMFileSaverOnline.cc.

References personalPlayback::fp, cond::hash, mps_fire::i, createfilelist::int, and MillePedeFileConverter_cfg::out.

Referenced by makeSnapshot().

                                                                                                          {
   // format.origin (one line):
   //   md5:d566a34b27f48d507150a332b189398b 294835 final_filename.root
 
   unsigned char md5[MD5_DIGEST_LENGTH];
 
   boost::iostreams::mapped_file_source fp(filename);
 
   MD5((unsigned char*)fp.data(), fp.size(), md5);
 
   std::ostringstream hash;
   for (unsigned char& i : md5) {
     hash << std::hex << std::setfill('0') << std::setw(2) << (int)i;
   }
 
   std::ostringstream out;
   out << "md5:" << hash.str() << " " << fp.size() << " " << final_filename;
   return out.str();
 }

void DQMFileSaverOnline::makeSnapshot	(	const FileParameters &	fp,
		bool	final
	)		const

protected

Definition at line 48 of file DQMFileSaverOnline.cc.

References appendSnapshot(), checkError(), dqm::DQMFileSaverBase::filename(), MainPageGenerator::files, fillOrigin(), dqm::DQMFileSaverBase::logFileAction(), dqm::DQMFileSaverBase::FileParameters::lumi_, dqm::dqmstoreimpl::DQMStore::mtEnabled(), ZMuMuAnalysisNtupler_cff::prefix, dqm::DQMFileSaverBase::FileParameters::run_, dqm::dqmstoreimpl::DQMStore::save(), dqm::DQMFileSaverBase::saveJobReport(), dqm::DQMFileSaverBase::FileParameters::saveReference_, dqm::DQMFileSaverBase::FileParameters::saveReferenceQMin_, AlCaHLTBitMon_QueryRunRegistry::string, and createPayload::suffix.

Referenced by saveLumi(), and saveRun().

                                                                                 {
   int pid = getpid();
   char hostname[64];
   gethostname(hostname, 64);
   hostname[63] = 0;
 
   char suffix[128];
   if (!final) {
     snprintf(suffix, 127, ".ls%08ld_host%s_pid%08d", fp.lumi_, hostname, pid);
   } else {
     suffix[0] = 0;
   }
 
   std::string prefix = filename(fp, false);
 
   std::string root_fp = prefix + ".root" + suffix;
   std::string meta_fp = prefix + ".root.origin" + suffix;
 
   std::string tmp_root_fp = root_fp + ".tmp";
   std::string tmp_meta_fp = meta_fp + ".tmp";
 
   // run_ and lumi_ are ignored if dqmstore is not in multithread mode
   edm::Service<DQMStore> store;
 
   logFileAction("Writing DQM Root file: ", root_fp);
   // logFileAction("Writing DQM Origin file: ", meta_fp);
 
   char rewrite[128];
   snprintf(rewrite, 128, "\\1Run %ld/\\2/Run summary", fp.run_);
 
   store->save(tmp_root_fp,                      /* filename      */
               "",                               /* path          */
               "^(Reference/)?([^/]+)",          /* pattern       */
               rewrite,                          /* rewrite       */
               store->mtEnabled() ? fp.run_ : 0, /* run           */
               0,                                /* lumi          */
               fp.saveReference_,                /* ref           */
               fp.saveReferenceQMin_,            /* ref minStatus */
               "RECREATE");                      /* fileupdate    */
 
   // write metadata
   // format.origin: md5:d566a34b27f48d507150a332b189398b 294835
   // /home/dqmprolocal/output/DQM_V0001_FED_R000194224.root
   std::ofstream meta_fd(tmp_meta_fp);
   meta_fd << fillOrigin(tmp_root_fp, root_fp);
   meta_fd.close();
 
   checkError("Rename failed: ", root_fp, ::rename(tmp_root_fp.c_str(), root_fp.c_str()));
   checkError("Rename failed: ", meta_fp, ::rename(tmp_meta_fp.c_str(), meta_fp.c_str()));
 
   SnapshotFiles files = {root_fp, meta_fp};
   if (final) {
     // final will never be cleared
     appendSnapshot(SnapshotFiles{});
 
     saveJobReport(root_fp);
   } else {
     appendSnapshot(SnapshotFiles{root_fp, meta_fp});
   }
 }

void DQMFileSaverOnline::saveLumi ( const FileParameters & fp ) const

overrideprotectedvirtual

Reimplemented from dqm::DQMFileSaverBase.

Definition at line 37 of file DQMFileSaverOnline.cc.

References backupLumiCount_, dqm::DQMFileSaverBase::FileParameters::lumi_, and makeSnapshot().

                                                                 {
   if (backupLumiCount_ > 0) {
     if (fp.lumi_ % backupLumiCount_ == 0) {
       // actual saving is done here
       makeSnapshot(fp, false);
     }
   }
 }