#include <DQMFileSaver.h>

Inheritance diagram for DQMFileSaver:

Public Types
enum	Convention { Online, Offline, FilterUnit }

enum	FileFormat { ROOT, PB }

Public Types inherited from edm::global::EDAnalyzerBase
typedef EDAnalyzerBase	ModuleType

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Public Member Functions
	DQMFileSaver (const edm::ParameterSet &ps)

Public Member Functions inherited from edm::global::EDAnalyzer< edm::RunCache< saverDetails::NoCache >, edm::LuminosityBlockCache< saverDetails::NoCache > >
	EDAnalyzer ()=default

Public Member Functions inherited from edm::global::EDAnalyzerBase
	EDAnalyzerBase ()

ModuleDescription const &	moduleDescription () const

virtual	~EDAnalyzerBase ()

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

bool	registeredToConsumeMany (TypeID const &, BranchType) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

virtual	~EDConsumerBase () noexcept(false)

Static Public Member Functions
static boost::property_tree::ptree	fillJson (int run, int lumi, const std::string &dataFilePathName, const std::string transferDestinationStr, const std::string mergeTypeStr, evf::FastMonitoringService *fms)

Static Public Member Functions inherited from edm::global::EDAnalyzerBase
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &descriptions)

Protected Member Functions
virtual void	analyze (edm::StreamID, const edm::Event &e, const edm::EventSetup &) const override

virtual void	beginJob (void) override

virtual void	endJob (void) override

virtual std::shared_ptr< saverDetails::NoCache >	globalBeginLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &) const override

virtual std::shared_ptr< saverDetails::NoCache >	globalBeginRun (const edm::Run &, const edm::EventSetup &) const override

virtual void	globalEndLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &) const override

virtual void	globalEndRun (const edm::Run &, const edm::EventSetup &) const override

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()

void	consumesMany (const TypeToGet &id)

template<BranchType B>
void	consumesMany (const TypeToGet &id)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Private Member Functions
void	saveForFilterUnit (const std::string &rewrite, int run, int lumi, const FileFormat fileFormat) const

void	saveForOffline (const std::string &workflow, int run, int lumi) const

void	saveForOfflinePB (const std::string &workflow, int run) const

void	saveForOnline (int run, const std::string &suffix, const std::string &rewrite) const

void	saveForOnlinePB (int run, const std::string &suffix) const

void	saveJobReport (const std::string &filename) const

Private Attributes
std::string	child_

Convention	convention_

DQMStore *	dbe_

std::string	dirName_

bool	enableMultiThread_

bool	fakeFilterUnitMode_

std::string	fileBaseName_

FileFormat	fileFormat_

std::atomic< int >	fileUpdate_

std::string	filterName_

evf::FastMonitoringService *	fms_

int	forceRunNumber_

std::atomic< int >	irun_

std::string	mergeType_

std::atomic< int >	nlumi_

std::atomic< int >	nrun_

std::string	producer_

bool	runIsComplete_

bool	saveAtJobEnd_

int	saveByLumiSection_

int	saveByRun_

int	saveReference_

int	saveReferenceQMin_

std::string	stream_label_

std::string	transferDestination_

int	version_

std::string	workflow_

Static Private Attributes
static const std::string	streamPrefix_

static const std::string	streamSuffix_

Detailed Description

Definition at line 15 of file DQMFileSaver.h.

Member Enumeration Documentation

enum DQMFileSaver::Convention

Enumerator
Online
Offline
FilterUnit

Definition at line 38 of file DQMFileSaver.h.

   {
     Online,
     Offline,
     FilterUnit,
   };

enum DQMFileSaver::FileFormat

Enumerator
ROOT
PB

Definition at line 45 of file DQMFileSaver.h.

   {
     ROOT,
     PB
   };

Constructor & Destructor Documentation

DQMFileSaver::DQMFileSaver ( const edm::ParameterSet & ps )

Definition at line 438 of file DQMFileSaver.cc.

References globaldigis_analyze_cff::convention, convention_, KineDebug3::count(), dirName_, Exception, fakeFilterUnitMode_, fileBaseName_, fileFormat_, filterName_, FilterUnit, fms_, forceRunNumber_, getAnInt(), edm::ParameterSet::getUntrackedParameter(), Offline, Online, PB, producer_, ROOT, runIsComplete_, alignCSCRings::s, saveAtJobEnd_, saveByLumiSection_, saveByRun_, saveReference_, saveReferenceQMin_, DQMStore::SaveWithoutReference, DQMStore::SaveWithReference, DQMStore::SaveWithReferenceForQTest, trackingPlots::stat, stream_label_, streamPrefix_, streamSuffix_, AlCaHLTBitMon_QueryRunRegistry::string, jetIDSelector_cfi::version, version_, and workflow_.

   : convention_ (Offline),
     fileFormat_(ROOT),
     workflow_ (""),
     producer_ ("DQM"),
     stream_label_ (""),
     dirName_ ("."),
     child_ (""),
     filterName_(""),
     version_ (1),
     runIsComplete_ (false),
     enableMultiThread_ (false),
     saveByLumiSection_ (-1),
     saveByRun_ (-1),
     saveAtJobEnd_ (false),
     saveReference_ (DQMStore::SaveWithReference),
     saveReferenceQMin_ (dqm::qstatus::STATUS_OK),
     forceRunNumber_ (-1),
     fileBaseName_ (""),
     fileUpdate_ (0),
     dbe_ (&*edm::Service<DQMStore>()),
     nrun_ (0),
     nlumi_ (0),
     irun_ (0),
     fms_(nullptr)
 {
   // Determine the file saving convention, and adjust defaults accordingly.
   std::string convention = ps.getUntrackedParameter<std::string>("convention", "Offline");
   fakeFilterUnitMode_ = ps.getUntrackedParameter<bool>("fakeFilterUnitMode", false);
 
   if (convention == "Offline")
     convention_ = Offline;
   else if (convention == "Online")
     convention_ = Online;
   else if (convention == "FilterUnit")
     convention_ = FilterUnit;
   else
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'convention' parameter '" << convention << "'."
       << "  Expected one of 'Online' or 'Offline' or 'FilterUnit'.";
 
   // If this is neither online nor FU convention, check workflow.
   // In this way, FU is treated as online, so we cannot specify a workflow. TBC
   if (convention_ != Online && convention_ != FilterUnit)
   {
     workflow_ = ps.getUntrackedParameter<std::string>("workflow", workflow_);
     if (workflow_.empty()
     || workflow_[0] != '/'
     || *workflow_.rbegin() == '/'
     || std::count(workflow_.begin(), workflow_.end(), '/') != 3
         || workflow_.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                        "abcdefghijklmnopqrstuvwxyz"
                        "0123456789"
                        "-_/") != std::string::npos)
       throw cms::Exception("DQMFileSaver")
     << "Invalid 'workflow' parameter '" << workflow_
     << "'.  Expected '/A/B/C'.";
   }
   else if (! ps.getUntrackedParameter<std::string>("workflow", "").empty())
     throw cms::Exception("DQMFileSaver")
       << "The 'workflow' parameter must be empty in 'Online' and 'FilterUnit' conventions.";
   else // for online set parameters
   {
     workflow_="/Global/Online/P5";
   }
 
   // Determine the file format, and adjust defaults accordingly.
   std::string fileFormat = ps.getUntrackedParameter<std::string>("fileFormat", "ROOT");
   if (fileFormat == "ROOT")
     fileFormat_ = ROOT;
   else if (fileFormat == "PB")
     fileFormat_ = PB;
   else
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'fileFormat' parameter '" << fileFormat << "'."
       << "  Expected one of 'ROOT' or 'PB'.";
 
   // Allow file producer to be set to specific values in certain conditions.
   producer_ = ps.getUntrackedParameter<std::string>("producer", producer_);
   // Setting the same constraints on file producer both for online and FilterUnit conventions
   // TODO(diguida): limit the producer for FilterUnit to be 'DQM' or 'HLTDQM'?
   // TODO(diguida): how to handle histograms produced in the playback for the FU case?
   if ((convention_ == Online || convention_ == FilterUnit)
       && producer_ != "DQM"
       && producer_ != "HLTDQM"
       && producer_ != "Playback")
   {
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'producer' parameter '" << producer_
       << "'.  Expected 'DQM', 'HLTDQM' or 'Playback'.";
   }
   else if (convention_ != Online
            && convention != FilterUnit
            && producer_ != "DQM")
   {
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'producer' parameter '" << producer_
       << "'.  Expected 'DQM'.";
   }
 
   stream_label_ = streamPrefix_ + producer_ + streamSuffix_;
 
   // version number to be used in filename
   version_ = ps.getUntrackedParameter<int>("version", version_);
   // flag to signal that file contains data from complete run
   runIsComplete_ = ps.getUntrackedParameter<bool>("runIsComplete", runIsComplete_);
 
   // Check how we should save the references.
   std::string refsave = ps.getUntrackedParameter<std::string>("referenceHandling", "default");
   if (refsave == "default")
     ;
   else if (refsave == "skip")
   {
     saveReference_ = DQMStore::SaveWithoutReference;
   //  std::cout << "skip saving all references" << std::endl;
   }
   else if (refsave == "all")
   {
     saveReference_ = DQMStore::SaveWithReference;
   //  std::cout << "saving all references" << std::endl;
   }
   else if (refsave == "qtests")
   {
     saveReference_ = DQMStore::SaveWithReferenceForQTest;
   //  std::cout << "saving qtest references" << std::endl;
   }
   else
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'referenceHandling' parameter '" << refsave
       << "'.  Expected 'default', 'skip', 'all' or 'qtests'.";
 
   // Check minimum required quality test result for which reference is saved.
   saveReferenceQMin_ = ps.getUntrackedParameter<int>("referenceRequireStatus", saveReferenceQMin_);
 
   // Get and check the output directory.
   struct stat s;
   dirName_ = ps.getUntrackedParameter<std::string>("dirName", dirName_);
   if (dirName_.empty() || stat(dirName_.c_str(), &s) == -1)
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'dirName' parameter '" << dirName_ << "'.";
 
   filterName_ = ps.getUntrackedParameter<std::string>("filterName", filterName_);
   // Find out when and how to save files.  The following contraints apply:
   // - For online, filter unit, and offline, allow files to be saved per lumi
   // - For online, allow files to be saved per run, at event and time intervals.
   // - For offline allow files to be saved per run, at job end, and run number to be overridden (for mc data).
   if (convention_ == Online || convention_ == Offline || convention_ == FilterUnit)
   {
     getAnInt(ps, saveByLumiSection_, "saveByLumiSection");
   }
 
   if (convention_ == Online)
   {
     getAnInt(ps, saveByRun_, "saveByRun");
   }
 
   if (convention_ == Offline)
   {
     getAnInt(ps, saveByRun_, "saveByRun");
     getAnInt(ps, forceRunNumber_, "forceRunNumber");
     saveAtJobEnd_ = ps.getUntrackedParameter<bool>("saveAtJobEnd", saveAtJobEnd_);
   }
 
   // Set up base file name:
   // - for online and offline, follow the convention <dirName>/<producer>_V<4digits>_
   // - for FilterUnit, we need to follow the DAQ2 convention, so we need the run and lumisection:
   //   the path is provided by the DAQ director service.
   if (convention_ != FilterUnit)
   {
     char version[8];
     sprintf(version, "_V%04d_", int(version_));
     version[7]='\0';
     fileBaseName_ = dirName_ + "/" + producer_ + version;
   }
   else if (fakeFilterUnitMode_)
   {
     edm::LogInfo("DQMFileSaver")
       << "Fake FU mode, files are saved under <dirname>/runXXXXXX/runXXXXXX_lsXXXX_<stream_Label>.pb.\n";
   }
   else
   {
     // For FU, dirName_ will not be considered at all
     edm::LogInfo("DQMFileSaver")
       << "The base dir provided in the configuration '" << dirName_ << "'\n"
       << " will not be considered: for FU, the DAQ2 services will handle directories\n";
     //check that DAQ2 services are available: throw if not
     fms_ = (evf::FastMonitoringService *) (edm::Service<evf::MicroStateService>().operator->());
     evf::EvFDaqDirector * daqDirector = (evf::EvFDaqDirector *) (edm::Service<evf::EvFDaqDirector>().operator->());
 
     if (!(fms_ && daqDirector))
       throw cms::Exception("DQMFileSaver")
               << "Internal error, cannot initialize DAQ services.";
   }
 
   // Log some information what we will do.
   edm::LogInfo("DQMFileSaver")
     << "DQM file saving settings:\n"
     << " using base file name '" << fileBaseName_ << "'\n"
     << " forcing run number " << forceRunNumber_ << "\n"
     << " saving every " << saveByLumiSection_ << " lumi section(s)\n"
     << " saving every " << saveByRun_ << " run(s)\n"
     << " saving at job end: " << (saveAtJobEnd_ ? "yes" : "no") << "\n";
 }

Member Function Documentation

void DQMFileSaver::analyze	(	edm::StreamID	,
		const edm::Event &	e,
		const edm::EventSetup &
	)		const

overrideprotectedvirtual

Definition at line 685 of file DQMFileSaver.cc.

 {
   //save by event and save by time are not supported
   //anymore in the threaded framework. please use
   //savebyLumiSection instead.
 }

void DQMFileSaver::beginJob ( void )

overrideprotectedvirtual

Reimplemented from edm::global::EDAnalyzerBase.

Definition at line 644 of file DQMFileSaver.cc.

References convention_, dbe_, enableMultiThread_, DQMStore::enableMultiThread_, fakeFilterUnitMode_, FilterUnit, irun_, mergeType_, evf::MergeTypePB, nlumi_, nrun_, stream_label_, and transferDestination_.

 {
   nrun_ = nlumi_ = irun_ = 0;
   
   // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor
   enableMultiThread_ = dbe_->enableMultiThread_;
 
   if ((convention_ == FilterUnit) && (!fakeFilterUnitMode_))
   {
     transferDestination_ = edm::Service<evf::EvFDaqDirector>()->getStreamDestinations(stream_label_);
     mergeType_ = edm::Service<evf::EvFDaqDirector>()->getStreamMergeType(stream_label_,evf::MergeTypePB);
   } 
 }

void DQMFileSaver::endJob ( void )

overrideprotectedvirtual

Reimplemented from edm::global::EDAnalyzerBase.

Definition at line 821 of file DQMFileSaver.cc.

References convention_, Exception, forceRunNumber_, irun_, Offline, saveAtJobEnd_, saveForOffline(), and workflow_.

 {
   if (saveAtJobEnd_)
     {
       if (convention_ == Offline && forceRunNumber_ > 0)
     saveForOffline(workflow_, forceRunNumber_, 0);
       else if (convention_ == Offline)
     saveForOffline(workflow_, irun_, 0);
       else
     throw cms::Exception("DQMFileSaver")
       << "Internal error.  Can only save files at the end of the"
       << " job in Offline mode.";
     }
 }

boost::property_tree::ptree DQMFileSaver::fillJson	(	int	run,
		int	lumi,
		const std::string &	dataFilePathName,
		const std::string	transferDestinationStr,
		const std::string	mergeTypeStr,
		evf::FastMonitoringService *	fms
	)

static

Definition at line 264 of file DQMFileSaver.cc.

References data, Exception, evf::FastMonitoringService::getEventsProcessedForLumi(), query::host, merge::inputFiles, callgraph::path, sysUtil::pid, EnergyCorrector::pt, trackingPlots::stat, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by dqmservices::JsonWritingTimeoutPoolOutputModule::doExtrasAfterCloseFile(), and saveForFilterUnit().

 {
   namespace bpt = boost::property_tree;
   namespace bfs = boost::filesystem;
   
   bpt::ptree pt;
 
   int hostnameReturn;
   char host[32];
   hostnameReturn = gethostname(host ,sizeof(host));
   if (hostnameReturn == -1)
     throw cms::Exception("fillJson")
           << "Internal error, cannot get host name";
   
   int pid = getpid();
   std::ostringstream oss_pid;
   oss_pid << pid;
 
   int nProcessed = fms ? (fms->getEventsProcessedForLumi(lumi)) : -1;
 
   // Stat the data file: if not there, throw
   std::string dataFileName;
   struct stat dataFileStat;
   dataFileStat.st_size=0;
   if (nProcessed) {
     if (stat(dataFilePathName.c_str(), &dataFileStat) != 0)
       throw cms::Exception("fillJson")
             << "Internal error, cannot get data file: "
             << dataFilePathName;
     // Extract only the data file name from the full path
     dataFileName = bfs::path(dataFilePathName).filename().string();
   }
   // The availability test of the FastMonitoringService was done in the ctor.
   bpt::ptree data;
   bpt::ptree processedEvents, acceptedEvents, errorEvents, bitmask, fileList, fileSize, inputFiles, fileAdler32, transferDestination, mergeType, hltErrorEvents;
 
   processedEvents.put("", nProcessed); // Processed events
   acceptedEvents.put("", nProcessed); // Accepted events, same as processed for our purposes
 
   errorEvents.put("", 0); // Error events
   bitmask.put("", 0); // Bitmask of abs of CMSSW return code
   fileList.put("", dataFileName); // Data file the information refers to
   fileSize.put("", dataFileStat.st_size); // Size in bytes of the data file
   inputFiles.put("", ""); // We do not care about input files!
   fileAdler32.put("", -1); // placeholder to match output json definition
   transferDestination.put("", transferDestinationStr); // SM Transfer destination field
   mergeType.put("", mergeTypeStr); // Merging type for merger and transfer services
   hltErrorEvents.put("", 0); // Error events
 
   data.push_back(std::make_pair("", processedEvents));
   data.push_back(std::make_pair("", acceptedEvents));
   data.push_back(std::make_pair("", errorEvents));
   data.push_back(std::make_pair("", bitmask));
   data.push_back(std::make_pair("", fileList));
   data.push_back(std::make_pair("", fileSize));
   data.push_back(std::make_pair("", inputFiles));
   data.push_back(std::make_pair("", fileAdler32));
   data.push_back(std::make_pair("", transferDestination));
   data.push_back(std::make_pair("", mergeType));
   data.push_back(std::make_pair("", hltErrorEvents));
 
   pt.add_child("data", data);
 
   if (fms == nullptr) {
     pt.put("definition", "/fakeDefinition.jsn");
   } else {
     // The availability test of the EvFDaqDirector Service was done in the ctor.
     bfs::path outJsonDefName(edm::Service<evf::EvFDaqDirector>()->baseRunDir()); //we assume this file is written bu the EvF Output module
     outJsonDefName /= (std::string("output_") + oss_pid.str() + std::string(".jsd"));
     pt.put("definition", outJsonDefName.string());
   }
 
   char sourceInfo[64]; //host and pid information
   sprintf(sourceInfo, "%s_%d", host, pid);
   pt.put("source", sourceInfo);
 
   return pt;
 }

std::shared_ptr< saverDetails::NoCache > DQMFileSaver::globalBeginLuminosityBlock	(	const edm::LuminosityBlock &	l,
		const edm::EventSetup &
	)		const

overrideprotectedvirtual

Definition at line 679 of file DQMFileSaver.cc.

References nlumi_.

 {
   ++nlumi_;
   return nullptr;
 }

std::shared_ptr< saverDetails::NoCache > DQMFileSaver::globalBeginRun	(	const edm::Run &	r,
		const edm::EventSetup &
	)		const

overrideprotectedvirtual

Definition at line 659 of file DQMFileSaver.cc.

References convention_, fakeFilterUnitMode_, FrontierConditions_GlobalTag_cff::file, FilterUnit, evf::EvFDaqDirector::getInitFilePath(), nrun_, stream_label_, and AlCaHLTBitMon_QueryRunRegistry::string.

 {
   ++nrun_;
 
   // For Filter Unit, create an empty ini file:
   // it is needed by the HLT deamon in order to start merging
   // The run number is established in the service
   // TODO(diguida): check that they are the same?
   if ((convention_ == FilterUnit) && (!fakeFilterUnitMode_))
   {
     evf::EvFDaqDirector * daqDirector = (evf::EvFDaqDirector *) (edm::Service<evf::EvFDaqDirector>().operator->());
     const std::string initFileName = daqDirector->getInitFilePath(stream_label_);
     std::ofstream file(initFileName);
     file.close();
   }
 
   return nullptr;
 }

void DQMFileSaver::globalEndLuminosityBlock	(	const edm::LuminosityBlock &	iLS,
		const edm::EventSetup &
	)		const

overrideprotectedvirtual

Definition at line 693 of file DQMFileSaver.cc.

References convention_, dbe_, DQMStore::deleteUnusedLumiHistograms(), enableMultiThread_, Exception, fileFormat_, FilterUnit, fms_, edm::LuminosityBlockBase::id(), edm::LuminosityBlockID::luminosityBlock(), nlumi_, Offline, Online, PB, edm::LuminosityBlockID::run(), saveByLumiSection_, saveForFilterUnit(), saveForOffline(), saveForOnline(), saveForOnlinePB(), evf::FastMonitoringService::shouldWriteFiles(), createPayload::suffix, and workflow_.

 {
   int ilumi    = iLS.id().luminosityBlock();
   int irun     = iLS.id().run();
   if (ilumi > 0 && saveByLumiSection_ > 0 )
   {
     if (convention_ != Online && convention_ != FilterUnit && convention_ != Offline )
       throw cms::Exception("DQMFileSaver")
     << "Internal error, can save files at end of lumi block"
     << " only in Online, FilterUnit or Offline mode.";
 
     if (convention_ == Online && (nlumi_ % saveByLumiSection_) == 0) // insist on lumi section ordering
     {
       char suffix[64];
       char rewrite[128];
       sprintf(suffix, "_R%09d_L%06d", irun, ilumi);
       sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", irun, ilumi-nlumi_, ilumi);
       if (fileFormat_ == ROOT)
         saveForOnline(irun, suffix, rewrite);
       else if (fileFormat_ == PB)
         saveForOnlinePB(irun, suffix);
       else
         throw cms::Exception("DQMFileSaver")
           << "Internal error, can save files"
           << " only in ROOT or ProtocolBuffer format.";
     }
 
     // Store at every lumi section end only if some events have been processed.
     // Caveat: if faking FilterUnit, i.e. not accessing DAQ2 services,
     // we cannot ask FastMonitoringService the processed events, so we are forced
     // to save the file at every lumisection, even with no statistics.
     // Here, we protect the call to get the processed events in a lumi section
     // by testing the pointer to FastMonitoringService: if not null, i.e. in real FU mode,
     // we check that the events are not 0; otherwise, we skip the test, so we store at every lumi transition. 
     // TODO(diguida): allow fake FU mode to skip file creation at empty lumi sections.
     if (convention_ == FilterUnit && (fms_ ? fms_->shouldWriteFiles(ilumi) : !fms_))
     {
       char rewrite[128];
       sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", irun, ilumi, ilumi);
       saveForFilterUnit(rewrite, irun, ilumi, fileFormat_);
     }
     if (convention_ == Offline)
     {
       if (fileFormat_ == ROOT)
         saveForOffline(workflow_, irun, ilumi);
       else
       // TODO(diguida): do we need to support lumisection saving in Offline for PB?
       // In this case, for ROOT, we only save EventInfo folders: we can filter them...
         throw cms::Exception("DQMFileSaver")
           << "Internal error, can save files"
           << " only in ROOT format.";
     }
 
     // after saving per LS, delete the old LS global histograms.
     dbe_->deleteUnusedLumiHistograms(enableMultiThread_ ? irun : 0, ilumi);
   }
 }

void DQMFileSaver::globalEndRun	(	const edm::Run &	iRun,
		const edm::EventSetup &
	)		const

overrideprotectedvirtual

Definition at line 752 of file DQMFileSaver.cc.

References convention_, data, dirName_, Exception, fakeFilterUnitMode_, FrontierConditions_GlobalTag_cff::file, fileFormat_, cmsPerfStripChart::format, edm::RunBase::id(), irun_, nlumi_, nrun_, Offline, Online, PB, EnergyCorrector::pt, edm::RunID::run(), saveByRun_, saveForOffline(), saveForOfflinePB(), saveForOnline(), saveForOnlinePB(), harvestTrackValidationPlots::str, AlCaHLTBitMon_QueryRunRegistry::string, createPayload::suffix, and workflow_.

 {
   int irun     = iRun.id().run();
   irun_        = irun;
   if (irun > 0 && saveByRun_ > 0 && (nrun_ % saveByRun_) == 0)
     {
       if (convention_ == Online)
     {
       char suffix[64];
       sprintf(suffix, "_R%09d", irun);
       char rewrite[64];
       sprintf(rewrite, "\\1Run %d/\\2/Run summary", irun);
       if (fileFormat_ == ROOT)
         saveForOnline(irun, suffix, rewrite);
       else if (fileFormat_ == PB)
         saveForOnlinePB(irun, suffix);
       else
         throw cms::Exception("DQMFileSaver")
           << "Internal error, can save files"
           << " only in ROOT or ProtocolBuffer format.";
     }
       else if (convention_ == Offline && fileFormat_ == ROOT)
     saveForOffline(workflow_, irun, 0);
       else if (convention_ == Offline && fileFormat_ == PB)
     saveForOfflinePB(workflow_, irun);
       else
     throw cms::Exception("DQMFileSaver")
       << "Internal error.  Can only save files in endRun()"
       << " in Online and Offline modes.";
     }
 
   // create a fake EoR file for testing purposes.
   if (fakeFilterUnitMode_) {
     edm::LogInfo("DQMFileSaver")
       << "Producing fake EoR file.\n";
 
     std::string runDir = str(boost::format("%s/run%06d") % dirName_ % irun);
     std::string jsonFilePathName = str(boost::format("%s/run%06d_ls0000_EoR.jsn") % runDir % irun);
     std::string openJsonFilePathName = jsonFilePathName + ".open";
 
     boost::filesystem::create_directories(runDir);
 
     using namespace boost::property_tree;
     ptree pt;
     ptree data;
 
     ptree child1, child2, child3;
 
     child1.put("", -1);    // Processed
     child2.put("", -1);    // Accepted
     child3.put("", nlumi_);  // number of lumi
 
     data.push_back(std::make_pair("", child1));
     data.push_back(std::make_pair("", child2));
     data.push_back(std::make_pair("", child3));
 
     pt.add_child("data", data);
     pt.put("definition", "/non-existant/");
     pt.put("source", "--hostname--");
 
     std::ofstream file(jsonFilePathName);
     write_json(file, pt, true);
     file.close();
 
     rename(openJsonFilePathName.c_str(), jsonFilePathName.c_str());
   }
 }

void DQMFileSaver::saveForFilterUnit	(	const std::string &	rewrite,
		int	run,
		int	lumi,
		const FileFormat	fileFormat
	)		const

private

Definition at line 344 of file DQMFileSaver.cc.

References dataFileExtension(), dbe_, dirName_, enableMultiThread_, Exception, fakeFilterUnitMode_, fileUpdate_, fillJson(), filterName_, fms_, cmsPerfStripChart::format, evf::FastMonitoringService::getEventsProcessedForLumi(), mergeType_, PB, EnergyCorrector::pt, DQMStore::save(), DQMStore::savePB(), saveReference_, saveReferenceQMin_, harvestTrackValidationPlots::str, stream_label_, AlCaHLTBitMon_QueryRunRegistry::string, and transferDestination_.

Referenced by globalEndLuminosityBlock().

 {
   // get from DAQ2 services where to store the files according to their format
   namespace bpt = boost::property_tree;
 
   std::string openJsonFilePathName;
   std::string jsonFilePathName;
   std::string openHistoFilePathName;
   std::string histoFilePathName;
 
   // create the files names
   if (fakeFilterUnitMode_) {
     std::string runDir = str(boost::format("%s/run%06d") % dirName_ % run);
     std::string baseName = str(boost::format("%s/run%06d_ls%04d_%s") % runDir % run % lumi % stream_label_ );
 
     boost::filesystem::create_directories(runDir);
 
     jsonFilePathName = baseName + ".jsn";
     openJsonFilePathName = jsonFilePathName + ".open";
 
     histoFilePathName = baseName + dataFileExtension(fileFormat);
     openHistoFilePathName = histoFilePathName + ".open";
   } else {
     openJsonFilePathName = edm::Service<evf::EvFDaqDirector>()->getOpenOutputJsonFilePath(lumi, stream_label_);
     jsonFilePathName = edm::Service<evf::EvFDaqDirector>()->getOutputJsonFilePath(lumi, stream_label_);
 
     if (fileFormat == ROOT) {
       openHistoFilePathName = edm::Service<evf::EvFDaqDirector>()->getOpenRootHistogramFilePath(lumi, stream_label_);
       histoFilePathName = edm::Service<evf::EvFDaqDirector>()->getRootHistogramFilePath(lumi, stream_label_);
     } else if (fileFormat == PB) {
       openHistoFilePathName = edm::Service<evf::EvFDaqDirector>()->getOpenProtocolBufferHistogramFilePath(lumi, stream_label_);
       histoFilePathName = edm::Service<evf::EvFDaqDirector>()->getProtocolBufferHistogramFilePath(lumi, stream_label_);
     }
   }
 
   if (fms_ ? fms_->getEventsProcessedForLumi(lumi) : true) {
     if (fileFormat == ROOT)
     {
       // Save the file with the full directory tree,
       // modifying it according to @a rewrite,
       // but not looking for MEs inside the DQMStore, as in the online case,
       // nor filling new MEs, as in the offline case.
       dbe_->save(openHistoFilePathName,
              "",
              "^(Reference/)?([^/]+)",
              rewrite,
              enableMultiThread_ ? run : 0,
              lumi,
              (DQMStore::SaveReferenceTag) saveReference_,
              saveReferenceQMin_,
              fileUpdate_ ? "UPDATE" : "RECREATE",
              true);
     }
     else if (fileFormat == PB)
     {
       // Save the file in the open directory.
       dbe_->savePB(openHistoFilePathName,
         filterName_,
         enableMultiThread_ ? run : 0,
         lumi,
         true);
     }
     else
       throw cms::Exception("DQMFileSaver")
         << "Internal error, can save files"
         << " only in ROOT or ProtocolBuffer format.";
 
     // Now move the the data and json files into the output directory.
     rename(openHistoFilePathName.c_str(), histoFilePathName.c_str());
   }
 
   // Write the json file in the open directory.
   bpt::ptree pt = fillJson(run, lumi, histoFilePathName, transferDestination_, mergeType_, fms_);
   write_json(openJsonFilePathName, pt);
   rename(openJsonFilePathName.c_str(), jsonFilePathName.c_str());
 }

void DQMFileSaver::saveForOffline	(	const std::string &	workflow,
		int	run,
		int	lumi
	)		const

private

Definition at line 86 of file DQMFileSaver.cc.

References DQMStore::bookFloat(), DQMStore::bookString(), DQMStore::cd(), child_, dbe_, MillePedeFileConverter_cfg::e, enableMultiThread_, fileBaseName_, corrVsCorr::filename, fileUpdate_, MonitorElement::Fill(), DQMStore::get(), edm::getReleaseVersion(), DQMStore::getSubdirs(), mps_fire::i, mps_alisetup::msg, onlineOfflineFileName(), runIsComplete_, DQMStore::save(), saveJobReport(), saveReference_, saveReferenceQMin_, DQMStore::SaveWithoutReference, DQMStore::setCurrentFolder(), dqm::qstatus::STATUS_OK, AlCaHLTBitMon_QueryRunRegistry::string, and createPayload::suffix.

Referenced by endJob(), globalEndLuminosityBlock(), and globalEndRun().

 {
   char suffix[64];
   sprintf(suffix, "R%09d", run);
 
   char rewrite[128];
   if (lumi == 0) // save for run
     sprintf(rewrite, "\\1Run %d/\\2/Run summary", run);
   else
     sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", run, lumi, lumi);
 
   std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, ROOT);
 
   if (lumi == 0) // save for run
   {
     // set run end flag
     dbe_->cd();
     dbe_->setCurrentFolder("Info/ProvInfo");
     
     // do this, because ProvInfo is not yet run in offline DQM
     MonitorElement* me = dbe_->get("Info/ProvInfo/CMSSW"); 
     if (!me) me = dbe_->bookString("CMSSW", edm::getReleaseVersion().c_str() );
     
     me = dbe_->get("Info/ProvInfo/runIsComplete");
     if (!me) me = dbe_->bookFloat("runIsComplete");
     
     if (me)
       { 
     if (runIsComplete_)
       me->Fill(1.);
     else
       me->Fill(0.);
       }
     
     dbe_->save(filename,
                "",
                "^(Reference/)?([^/]+)",
                rewrite,
                enableMultiThread_ ? run : 0,
                lumi,
                (DQMStore::SaveReferenceTag) saveReference_,
                saveReferenceQMin_,
                fileUpdate_ ? "UPDATE" : "RECREATE");
 
     // save the JobReport
     saveJobReport(filename);
   }
   else // save EventInfo folders for luminosity sections
   {
     std::vector<std::string> systems = (dbe_->cd(), dbe_->getSubdirs());
 
     edm::LogAbsolute msg("fileAction");
     msg << "DQMFileSaver: storing EventInfo folders for Run: "
               << run << ", Lumi Section: " << lumi << ", Subsystems: " ;
 
     for (size_t i = 0, e = systems.size(); i != e; ++i) {
       if (systems[i] != "Reference") {
         dbe_->cd();
         msg << systems[i] << "  " ;
 
         dbe_->save(filename,
                    systems[i]+"/EventInfo", "^(Reference/)?([^/]+)",
                    rewrite,
                    enableMultiThread_ ? run : 0,
                    lumi,
                    DQMStore::SaveWithoutReference,
                    dqm::qstatus::STATUS_OK,
                    fileUpdate_ ? "UPDATE" : "RECREATE");
 
         // from now on update newly created file
         if (fileUpdate_.load() == 0) fileUpdate_ = 1;
       }
     }
   }
 
 }

void DQMFileSaver::saveForOfflinePB	(	const std::string &	workflow,
		int	run
	)		const

private

Definition at line 77 of file DQMFileSaver.cc.

References child_, dbe_, fileBaseName_, corrVsCorr::filename, filterName_, onlineOfflineFileName(), PB, DQMStore::savePB(), AlCaHLTBitMon_QueryRunRegistry::string, and createPayload::suffix.

Referenced by globalEndRun().

 {
   char suffix[64];
   sprintf(suffix, "R%09d", run);
   std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, PB);
   dbe_->savePB(filename, filterName_);
 }

void DQMFileSaver::saveForOnline	(	int	run,
		const std::string &	suffix,
		const std::string &	rewrite
	)		const

private

Definition at line 211 of file DQMFileSaver.cc.

References DQMStore::cd(), child_, dbe_, doSaveForOnline(), MillePedeFileConverter_cfg::e, enableMultiThread_, fileBaseName_, DQMStore::get(), DQMStore::getMatchingContents(), DQMStore::getSubdirs(), mps_fire::i, saveReference_, and saveReferenceQMin_.

Referenced by globalEndLuminosityBlock(), and globalEndRun().

 {
   std::vector<std::string> systems = (dbe_->cd(), dbe_->getSubdirs());
 
   for (size_t i = 0, e = systems.size(); i != e; ++i)
   {
     if (systems[i] != "Reference")
     {
       dbe_->cd();
       if (MonitorElement* me = dbe_->get(systems[i] + "/EventInfo/processName"))
       {
     doSaveForOnline(dbe_, run, enableMultiThread_,
             fileBaseName_ + me->getStringValue() + suffix + child_ + ".root",
             "", "^(Reference/)?([^/]+)", rewrite,
                     (DQMStore::SaveReferenceTag) saveReference_,
                     saveReferenceQMin_,
             "", ROOT);
         return;
       }
     }
   }
 
   // look for EventInfo folder in an unorthodox location
   for (size_t i = 0, e = systems.size(); i != e; ++i)
     if (systems[i] != "Reference")
     {
       dbe_->cd();
       std::vector<MonitorElement*> pNamesVector = dbe_->getMatchingContents("^" + systems[i] + "/.*/EventInfo/processName",lat::Regexp::Perl);
       if (pNamesVector.size() > 0){
         doSaveForOnline(dbe_, run, enableMultiThread_,
                         fileBaseName_ + systems[i] + suffix + child_ + ".root",
                         "", "^(Reference/)?([^/]+)", rewrite,
                         (DQMStore::SaveReferenceTag) saveReference_,
                         saveReferenceQMin_,
             "", ROOT);
         pNamesVector.clear();
         return;
       }
     }
 
   // if no EventInfo Folder is found, then store subsystem wise
   for (size_t i = 0, e = systems.size(); i != e; ++i)
     if (systems[i] != "Reference")
       doSaveForOnline(dbe_, run, enableMultiThread_,
                       fileBaseName_ + systems[i] + suffix + child_ + ".root",
                   systems[i], "^(Reference/)?([^/]+)", rewrite,
                   (DQMStore::SaveReferenceTag) saveReference_,
                       saveReferenceQMin_,
                       "", ROOT);
 }

void DQMFileSaver::saveForOnlinePB	(	int	run,
		const std::string &	suffix
	)		const

private

Definition at line 194 of file DQMFileSaver.cc.

References child_, dbe_, doSaveForOnline(), enableMultiThread_, fileBaseName_, corrVsCorr::filename, filterName_, onlineOfflineFileName(), PB, saveReference_, saveReferenceQMin_, AlCaHLTBitMon_QueryRunRegistry::string, and workflow_.

Referenced by globalEndLuminosityBlock(), and globalEndRun().

 {
   // The file name contains the Online workflow name,
   // as we do not want to look inside the DQMStore,
   // and the @a suffix, defined in the run/lumi transitions.
   // TODO(diguida): add the possibility to change the dir structure with rewrite.
   std::string filename = onlineOfflineFileName(fileBaseName_, suffix, workflow_, child_, PB);
   doSaveForOnline(dbe_, run, enableMultiThread_,
           filename,
           "", "^(Reference/)?([^/]+)", "\\1\\2",
           (DQMStore::SaveReferenceTag) saveReference_,
           saveReferenceQMin_,
           filterName_,
           PB);
 }

void DQMFileSaver::saveJobReport ( const std::string & filename ) const

private

Definition at line 422 of file DQMFileSaver.cc.

References info(), edm::Service< T >::isAvailable(), and edm::JobReport::reportAnalysisFile().

Referenced by saveForOffline().

 {
 
   // Report the file to job report service.
   edm::Service<edm::JobReport> jr;
   if (jr.isAvailable())
   {
     std::map<std::string, std::string> info;
     info["Source"] = "DQMStore";
     info["FileClass"] = "DQM";
     jr->reportAnalysisFile(filename, info);
   }
 
 }