#include <DQMFileSaverOnline.h>

Inheritance diagram for dqm::DQMFileSaverOnline:

Classes
struct	SnapshotFiles

Public Member Functions
	DQMFileSaverOnline (const edm::ParameterSet &ps)

	~DQMFileSaverOnline () override

Public Member Functions inherited from dqm::DQMFileSaverBase
	DQMFileSaverBase (const edm::ParameterSet &ps)

	~DQMFileSaverBase () override

Public Member Functions inherited from edm::global::EDAnalyzer< edm::RunCache< NoCache >, edm::LuminosityBlockCache< NoCache > >
	EDAnalyzer ()=default

bool	wantsGlobalLuminosityBlocks () const final

bool	wantsGlobalRuns () const final

bool	wantsStreamLuminosityBlocks () const final

bool	wantsStreamRuns () const final

Public Member Functions inherited from edm::global::EDAnalyzerBase
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)

	EDAnalyzerBase ()

ModuleDescription const &	moduleDescription () const

	~EDAnalyzerBase () override

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ESProxyIndex const *	esGetTokenIndices (edm::Transition iTrans) const

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

bool	registeredToConsumeMany (TypeID const &, BranchType) const

ProductResolverIndexAndSkipBit	uncheckedIndexFrom (EDGetToken) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

void	updateLookup (eventsetup::ESRecordsToProxyIndices const &)

virtual	~EDConsumerBase () noexcept(false)

Static Public Member Functions
static void	fillDescriptions (edm::ConfigurationDescriptions &descriptions)

static const std::string	fillOrigin (const std::string &filename, const std::string &final_filename)

Static Public Member Functions inherited from dqm::DQMFileSaverBase
static void	fillDescription (edm::ParameterSetDescription &d)

Static Public Member Functions inherited from edm::global::EDAnalyzerBase
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &descriptions)

Protected Member Functions
void	appendSnapshot (SnapshotFiles new_snap) const

void	checkError (const char *msg, const std::string &file, int status) const

void	makeSnapshot (const FileParameters &fp, bool final) const

void	saveLumi (const FileParameters &fp) const override

void	saveRun (const FileParameters &fp) const override

Protected Member Functions inherited from dqm::DQMFileSaverBase
void	analyze (edm::StreamID, const edm::Event &e, const edm::EventSetup &) const final

std::shared_ptr< NoCache >	globalBeginLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &) const final

std::shared_ptr< NoCache >	globalBeginRun (const edm::Run &, const edm::EventSetup &) const final

void	globalEndLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &) const final

void	globalEndRun (const edm::Run &, const edm::EventSetup &) const final

virtual void	initRun () const

void	logFileAction (const std::string &msg, const std::string &fileName) const

void	saveJobReport (const std::string &filename) const

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()

void	consumesMany (const TypeToGet &id)

template<BranchType B>
void	consumesMany (const TypeToGet &id)

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes ()

template<typename ESProduct , typename ESRecord , Transition Tr = Transition::Event>
auto	esConsumes (ESInputTag const &tag)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Protected Attributes
int	backupLumiCount_

bool	keepBackupLumi_

std::list< SnapshotFiles >	snapshots_

std::mutex	snapshots_lock_

Protected Attributes inherited from dqm::DQMFileSaverBase
FileParameters	initial_fp_

std::mutex	initial_fp_lock_

Additional Inherited Members
Public Types inherited from edm::global::EDAnalyzerBase
typedef EDAnalyzerBase	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Static Protected Member Functions inherited from dqm::DQMFileSaverBase
static const std::string	filename (const FileParameters &fp, bool useLumi=false)

Detailed Description

Definition at line 15 of file DQMFileSaverOnline.h.

Constructor & Destructor Documentation

DQMFileSaverOnline::DQMFileSaverOnline ( const edm::ParameterSet & ps )

Definition at line 31 of file DQMFileSaverOnline.cc.

References backupLumiCount_, edm::ParameterSet::getUntrackedParameter(), keepBackupLumi_, and ~DQMFileSaverOnline().

     : DQMFileSaverBase(ps) {
   backupLumiCount_ = ps.getUntrackedParameter<int>("backupLumiCount", 1);
   keepBackupLumi_ = ps.getUntrackedParameter<bool>("keepBackupLumi", false);
 }

DQMFileSaverOnline::~DQMFileSaverOnline ( )

overridedefault

Referenced by DQMFileSaverOnline().

Member Function Documentation

void DQMFileSaverOnline::appendSnapshot ( SnapshotFiles new_snap ) const

protected

Definition at line 116 of file DQMFileSaverOnline.cc.

References checkError(), dqm::DQMFileSaverOnline::SnapshotFiles::data, keepBackupLumi_, CommonMethods::lock(), dqm::DQMFileSaverBase::logFileAction(), dqm::DQMFileSaverOnline::SnapshotFiles::meta, snapshots_, and snapshots_lock_.

Referenced by makeSnapshot().

                                                              {
   std::lock_guard<std::mutex> lock(snapshots_lock_);
 
   if (!keepBackupLumi_) {
     while (!snapshots_.empty()) {
       SnapshotFiles& x = snapshots_.front();
 
       // logFileAction("Deleting old snapshot (origin): ", x.meta);
       checkError("Unlink failed: ", x.meta, ::unlink(x.meta.c_str()));
 
       logFileAction("Deleting old snapshot (root): ", x.data);
       checkError("Unlink failed: ", x.data, ::unlink(x.data.c_str()));
 
       snapshots_.pop_front();
     }
   }
 
   if (!f.data.empty()) {
     snapshots_.push_back(f);
   }
 }

void DQMFileSaverOnline::checkError	(	const char *	msg,
		const std::string &	file,
		int	status
	)		const

protected

Definition at line 138 of file DQMFileSaverOnline.cc.

References dqm::DQMFileSaverBase::logFileAction(), mps_check::msg, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by appendSnapshot(), and makeSnapshot().

                                                       {
   if (status != 0) {
     std::string actual_msg = msg;
     actual_msg += std::strerror(status);
     logFileAction(actual_msg, file);
   }
 }

void DQMFileSaverOnline::fillDescriptions ( edm::ConfigurationDescriptions & descriptions )

static

Definition at line 168 of file DQMFileSaverOnline.cc.

References edm::ConfigurationDescriptions::addDefault(), edm::ParameterSetDescription::addUntracked(), DEFINE_FWK_MODULE, dqm::DQMFileSaverBase::fillDescription(), and edm::ParameterSetDescription::setComment().

                                                 {
   edm::ParameterSetDescription desc;
   desc.setComment("Saves histograms from DQM store, online workflow.");
 
   desc.addUntracked<int>("backupLumiCount", 10)
       ->setComment(
           "How often the backup file will be generated, in lumisections (-1 "
           "disables).");
 
   desc.addUntracked<bool>("keepBackupLumi", false)
       ->setComment(
           "Usually the backup old backup is deleted once the new file is "
           "available. Setting this to true ensures that no backup files are "
           "ever deleted. Useful for ML applications, which use backups as a "
           "'history' of what happened during the run.");
 
   DQMFileSaverBase::fillDescription(desc);
 
   // Changed to use addDefault instead of add here because previously
   // DQMFileSaverOnline and DQMFileSaverPB both used the module label
   // "saver" which caused conflicting cfi filenames to be generated.
   // add could be used if unique module labels were given.
   descriptions.addDefault(desc);
 }

const std::string DQMFileSaverOnline::fillOrigin	(	const std::string &	filename,
		const std::string &	final_filename
	)

static

Definition at line 147 of file DQMFileSaverOnline.cc.

References cond::hash, mps_fire::i, createfilelist::int, and MillePedeFileConverter_cfg::out.

Referenced by makeSnapshot().

                                                                 {
   // format.origin (one line):
   //   md5:d566a34b27f48d507150a332b189398b 294835 final_filename.root
 
   unsigned char md5[MD5_DIGEST_LENGTH];
 
   boost::iostreams::mapped_file_source fp(filename);
 
   MD5((unsigned char*)fp.data(), fp.size(), md5);
 
   std::ostringstream hash;
   for (unsigned char & i : md5) {
     hash << std::hex << std::setfill('0') << std::setw(2) << (int)i;
   }
 
   std::ostringstream out;
   out << "md5:" << hash.str() << " " << fp.size() << " " << final_filename;
   return out.str();
 }

void DQMFileSaverOnline::makeSnapshot	(	const FileParameters &	fp,
		bool	final
	)		const

protected

Definition at line 52 of file DQMFileSaverOnline.cc.

References appendSnapshot(), checkError(), dqm::DQMFileSaverBase::filename(), PatBasicFWLiteAnalyzer::files, fillOrigin(), dqm::DQMFileSaverBase::logFileAction(), dqm::DQMFileSaverBase::FileParameters::lumi_, sysUtil::pid, tablePrinter::prefix, dqm::DQMFileSaverBase::FileParameters::run_, dqm::DQMFileSaverBase::saveJobReport(), dqm::DQMFileSaverBase::FileParameters::saveReference_, dqm::DQMFileSaverBase::FileParameters::saveReferenceQMin_, AlCaHLTBitMon_QueryRunRegistry::string, and createPayload::suffix.

Referenced by saveLumi(), and saveRun().

                                                         {
   int pid = getpid();
   char hostname[64];
   gethostname(hostname, 64);
   hostname[63] = 0;
 
   char suffix[128];
   if (!final) {
     snprintf(suffix, 127, ".ls%08ld_host%s_pid%08d", fp.lumi_, hostname, pid);
   } else {
     suffix[0] = 0;
   }
 
   std::string prefix = filename(fp, false);
 
   std::string root_fp = prefix + ".root" + suffix;
   std::string meta_fp = prefix + ".root.origin" + suffix;
 
   std::string tmp_root_fp = root_fp + ".tmp";
   std::string tmp_meta_fp = meta_fp + ".tmp";
 
   // run_ and lumi_ are ignored if dqmstore is not in multithread mode
   edm::Service<DQMStore> store;
 
   logFileAction("Writing DQM Root file: ", root_fp);
   // logFileAction("Writing DQM Origin file: ", meta_fp);
 
   char rewrite[128];
   snprintf(rewrite, 128, "\\1Run %ld/\\2/Run summary", fp.run_);
 
   store->save(tmp_root_fp,                      /* filename      */
               "",                               /* path          */
               "^(Reference/)?([^/]+)",          /* pattern       */
               rewrite,                          /* rewrite       */
               store->mtEnabled() ? fp.run_ : 0, /* run           */
               0,                                /* lumi          */
               fp.saveReference_,                /* ref           */
               fp.saveReferenceQMin_,            /* ref minStatus */
               "RECREATE");                      /* fileupdate    */
 
   // write metadata
   // format.origin: md5:d566a34b27f48d507150a332b189398b 294835
   // /home/dqmprolocal/output/DQM_V0001_FED_R000194224.root
   std::ofstream meta_fd(tmp_meta_fp);
   meta_fd << fillOrigin(tmp_root_fp, root_fp);
   meta_fd.close();
 
   checkError("Rename failed: ", root_fp,
              ::rename(tmp_root_fp.c_str(), root_fp.c_str()));
   checkError("Rename failed: ", meta_fp,
              ::rename(tmp_meta_fp.c_str(), meta_fp.c_str()));
 
   SnapshotFiles files = {root_fp, meta_fp};
   if (final) {
     // final will never be cleared
     appendSnapshot(SnapshotFiles{});
 
     saveJobReport(root_fp);
   } else {
     appendSnapshot(SnapshotFiles{root_fp, meta_fp});
   }
 }

void DQMFileSaverOnline::saveLumi ( const FileParameters & fp ) const

overrideprotectedvirtual

Reimplemented from dqm::DQMFileSaverBase.

Definition at line 39 of file DQMFileSaverOnline.cc.

References backupLumiCount_, dqm::DQMFileSaverBase::FileParameters::lumi_, and makeSnapshot().

                                                                 {
   if (backupLumiCount_ > 0) {
     if (fp.lumi_ % backupLumiCount_ == 0) {
       // actual saving is done here
       makeSnapshot(fp, false);
     }
   }
 }

void DQMFileSaverOnline::saveRun ( const FileParameters & fp ) const

overrideprotectedvirtual

Reimplemented from dqm::DQMFileSaverBase.

Definition at line 48 of file DQMFileSaverOnline.cc.

References makeSnapshot().

                                                                {
   makeSnapshot(fp, true);
 }

Member Data Documentation

int dqm::DQMFileSaverOnline::backupLumiCount_

protected

Definition at line 28 of file DQMFileSaverOnline.h.

Referenced by DQMFileSaverOnline(), and saveLumi().

bool dqm::DQMFileSaverOnline::keepBackupLumi_

protected

Definition at line 29 of file DQMFileSaverOnline.h.

Referenced by appendSnapshot(), and DQMFileSaverOnline().

std::list<SnapshotFiles> dqm::DQMFileSaverOnline::snapshots_

mutableprotected

Definition at line 41 of file DQMFileSaverOnline.h.

Referenced by appendSnapshot().

std::mutex dqm::DQMFileSaverOnline::snapshots_lock_

mutableprotected

Definition at line 40 of file DQMFileSaverOnline.h.

Referenced by appendSnapshot().

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation