#include <DQMFileSaver.h>

Inheritance diagram for DQMFileSaver:

Public Member Functions
	DQMFileSaver (const edm::ParameterSet &ps)

Public Member Functions inherited from edm::EDAnalyzer
	EDAnalyzer ()

std::string	workerType () const

virtual	~EDAnalyzer ()

Protected Member Functions
virtual void	analyze (const edm::Event &e, const edm::EventSetup &)

virtual void	beginJob (void)

virtual void	beginLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &)

virtual void	beginRun (const edm::Run &, const edm::EventSetup &)

virtual void	endJob (void)

virtual void	endLuminosityBlock (const edm::LuminosityBlock &, const edm::EventSetup &)

virtual void	endRun (const edm::Run &, const edm::EventSetup &)

Protected Member Functions inherited from edm::EDAnalyzer
CurrentProcessingContext const *	currentContext () const

Private Types
enum	Convention { Online, Offline }

Private Member Functions
void	saveForOffline (const std::string &workflow, int run, int lumi)

void	saveForOnline (const std::string &suffix, const std::string &rewrite)

void	saveJobReport (const std::string &filename)

Private Attributes
Convention	convention_

DQMStore *	dbe_

std::string	dirName_

std::string	fileBaseName_

std::string	fileUpdate_

MonitorElement *	fileVersion_

int	forceRunNumber_

MonitorElement *	hostName_

int	ievent_

int	ilumi_

int	ilumiprev_

int	irun_

MonitorElement *	isComplete_
	The PID associated with this job. More...

int	nevent_

int	nlumi_

int	nrun_

int	numKeepSavedFiles_

std::list< std::string >	pastSavedFiles_

MonitorElement *	processId_
	Current working directory of the job. More...

MonitorElement *	processName_
	Hostname of the local machine. More...

std::string	producer_

bool	runIsComplete_

bool	saveAtJobEnd_

int	saveByEvent_

int	saveByLumiSection_

int	saveByMinute_

int	saveByRun_

int	saveByTime_

timeval	saved_

int	saveReference_

int	saveReferenceQMin_

timeval	start_

MonitorElement *	versCMSSW_

MonitorElement *	versDataset_

MonitorElement *	versGlobaltag_

int	version_

MonitorElement *	versTaglist_

std::string	workflow_

MonitorElement *	workingDir_
	DQM "name" of the job (eg, Hcal or DT) More...

Additional Inherited Members
Public Types inherited from edm::EDAnalyzer
typedef EDAnalyzer	ModuleType

typedef WorkerT< EDAnalyzer >	WorkerType

Static Public Member Functions inherited from edm::EDAnalyzer
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

Detailed Description

Definition at line 10 of file DQMFileSaver.h.

Member Enumeration Documentation

enum DQMFileSaver::Convention

private

Enumerator
Online
Offline

Definition at line 29 of file DQMFileSaver.h.

   {
     Online,
     Offline
   };

Constructor & Destructor Documentation

DQMFileSaver::DQMFileSaver ( const edm::ParameterSet & ps )

Definition at line 184 of file DQMFileSaver.cc.

References convention_, prof2calltree::count, dirName_, edm::hlt::Exception, fileBaseName_, forceRunNumber_, getAnInt(), edm::ParameterSet::getUntrackedParameter(), numKeepSavedFiles_, Offline, Online, producer_, runIsComplete_, asciidump::s, saveAtJobEnd_, saveByEvent_, saveByLumiSection_, saveByMinute_, saveByRun_, saveByTime_, saved_, saveReference_, saveReferenceQMin_, DQMStore::SaveWithoutReference, DQMStore::SaveWithReference, DQMStore::SaveWithReferenceForQTest, start_, BeamSplash_cfg::version, version_, and workflow_.

   : convention_ (Offline),
     workflow_ (""),
     producer_ ("DQM"),
     dirName_ ("."),
     version_ (1),
     runIsComplete_ (false),
     saveByLumiSection_ (-1),
     saveByEvent_ (-1),
     saveByMinute_ (-1),
     saveByTime_ (-1),
     saveByRun_ (1),
     saveAtJobEnd_ (false),
     saveReference_ (DQMStore::SaveWithReference),
     saveReferenceQMin_ (dqm::qstatus::STATUS_OK),
     forceRunNumber_ (-1),
     fileBaseName_ (""),
     fileUpdate_ ("RECREATE"),
     dbe_ (&*edm::Service<DQMStore>()),
     irun_ (-1),
     ilumi_ (-1),
     ilumiprev_ (-1),
     ievent_ (-1),
     nrun_ (0),
     nlumi_ (0),
     nevent_ (0),
     numKeepSavedFiles_ (5)
 {
   // Determine the file saving convention, and adjust defaults accordingly.
   std::string convention = ps.getUntrackedParameter<std::string>("convention", "Offline");
   if (convention == "Offline")
     convention_ = Offline;
   else if (convention == "Online")
     convention_ = Online;
   else
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'convention' parameter '" << convention << "'."
       << "  Expected one of 'Online' or 'Offline'.";
 
   // If this isn't online convention, check workflow.
   if (convention_ != Online)
   {
     workflow_ = ps.getUntrackedParameter<std::string>("workflow", workflow_);
     if (workflow_.empty()
     || workflow_[0] != '/'
     || *workflow_.rbegin() == '/'
     || std::count(workflow_.begin(), workflow_.end(), '/') != 3
         || workflow_.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                        "abcdefghijklmnopqrstuvwxyz"
                        "0123456789"
                        "-_/") != std::string::npos)
       throw cms::Exception("DQMFileSaver")
     << "Invalid 'workflow' parameter '" << workflow_
     << "'.  Expected '/A/B/C'.";
   }
   else if (! ps.getUntrackedParameter<std::string>("workflow", "").empty())
     throw cms::Exception("DQMFileSaver")
       << "The 'workflow' parameter must be empty in 'Online' convention.";
   else // for online set parameters
   {
     workflow_="/Global/Online/P5";
   }
     
   // Allow file producer to be set to specific values in certain conditions.
   producer_ = ps.getUntrackedParameter<std::string>("producer", producer_);
   if (convention_ == Online
       && producer_ != "DQM"
       && producer_ != "HLTDQM"
       && producer_ != "Playback")
   {
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'producer' parameter '" << producer_
       << "'.  Expected 'DQM', 'HLTDQM' or 'Playback'.";
   }
   else if (convention_ != Online && producer_ != "DQM")
   {
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'producer' parameter '" << producer_
       << "'.  Expected 'DQM'.";
   }
 
   // version number to be used in filename
   version_ = ps.getUntrackedParameter<int>("version", version_);
   // flag to signal that file contains data from complete run
   runIsComplete_ = ps.getUntrackedParameter<bool>("runIsComplete", runIsComplete_);
 
   // Check how we should save the references.
   std::string refsave = ps.getUntrackedParameter<std::string>("referenceHandling", "default");
   if (refsave == "default")
     ;
   else if (refsave == "skip") 
   {
     saveReference_ = DQMStore::SaveWithoutReference;
   //  std::cout << "skip saving all references" << std::endl;
   }
   else if (refsave == "all")
   {
     saveReference_ = DQMStore::SaveWithReference;
   //  std::cout << "saving all references" << std::endl;
   }
   else if (refsave == "qtests")
   {
     saveReference_ = DQMStore::SaveWithReferenceForQTest;
   //  std::cout << "saving qtest references" << std::endl;
   }
   else
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'referenceHandling' parameter '" << refsave
       << "'.  Expected 'default', 'skip', 'all' or 'qtests'.";
 
   // Check minimum required quality test result for which reference is saved.
   saveReferenceQMin_ = ps.getUntrackedParameter<int>("referenceRequireStatus", saveReferenceQMin_);
 
   // Get and check the output directory.
   struct stat s;
   dirName_ = ps.getUntrackedParameter<std::string>("dirName", dirName_);
   if (dirName_.empty() || stat(dirName_.c_str(), &s) == -1)
     throw cms::Exception("DQMFileSaver")
       << "Invalid 'dirName' parameter '" << dirName_ << "'.";
 
   // Find out when and how to save files.  The following contraints apply:
   // - For online, allow files to be saved at event and time intervals.
   // - For online and offline, allow files to be saved per run, lumi and job end
   // - For offline allow run number to be overridden (for mc data).
   if (convention_ == Online)
   {
     getAnInt(ps, saveByEvent_, "saveByEvent");
     getAnInt(ps, saveByMinute_, "saveByMinute");
     getAnInt(ps, saveByTime_, "saveByTime");
     getAnInt(ps, numKeepSavedFiles_, "maxSavedFilesCount");
   }
 
   if (convention_ == Online || convention_ == Offline)
   {
     getAnInt(ps, saveByRun_, "saveByRun");
     getAnInt(ps, saveByLumiSection_, "saveByLumiSection");
   }
 
   if (convention_ != Online)
   {
     getAnInt(ps, forceRunNumber_, "forceRunNumber");
     saveAtJobEnd_ = ps.getUntrackedParameter<bool>("saveAtJobEnd", saveAtJobEnd_);
   }
 
   if (saveAtJobEnd_ && forceRunNumber_ < 1)
     throw cms::Exception("DQMFileSaver")
       << "If saving at the end of the job, the run number must be"
       << " overridden to a specific value using 'forceRunNumber'.";
 
   
   // Set up base file name and determine the start time.
   char version[8];
   sprintf(version, "_V%04d_", int(version_));
   version[7]='\0';
   fileBaseName_ = dirName_ + "/" + producer_ + version;
   gettimeofday(&start_, 0);
   saved_ = start_;
 
   // Log some information what we will do.
   edm::LogInfo("DQMFileSaver")
     << "DQM file saving settings:\n"
     << " using base file name '" << fileBaseName_ << "'\n"
     << " forcing run number " << forceRunNumber_ << "\n"
     << " saving every " << saveByLumiSection_ << " lumi section(s)\n"
     << " saving every " << saveByEvent_ << " event(s)\n"
     << " saving every " << saveByMinute_ << " minute(s)\n"
     << " saving every 2^n*" << saveByTime_ << " minutes \n"
     << " saving every " << saveByRun_ << " run(s)\n"
     << " saving at job end: " << (saveAtJobEnd_ ? "yes" : "no") << "\n"
     << " keeping at most " << numKeepSavedFiles_ << " files\n";
 }

Member Function Documentation

void DQMFileSaver::analyze	(	const edm::Event &	e,
		const edm::EventSetup &
	)

protectedvirtual

Implements edm::EDAnalyzer.

Definition at line 379 of file DQMFileSaver.cc.

References convention_, edm::EventID::event(), edm::hlt::Exception, edm::EventBase::id(), ievent_, irun_, nevent_, Online, saveByEvent_, saveByMinute_, saveByTime_, saved_, saveForOnline(), start_, and createPayload::suffix.

 {
   ++nevent_;
 
   ievent_   = e.id().event();
 
   // Check if we should save for this event.
   char suffix[64];
   if (ievent_ > 0 && saveByEvent_ > 0 && nevent_ == saveByEvent_)
   {
     if (convention_ != Online)
       throw cms::Exception("DQMFileSaver")
     << "Internal error, can save files by event"
     << " only in Online mode.";
 
     sprintf(suffix, "_R%09d_E%08d", irun_, ievent_);
     saveForOnline(suffix, "\\1\\2");
     nevent_ = 0;
   }
 
   // Check if we should save due to elapsed time.
   if ( ievent_ > 0 && ( saveByMinute_ > 0 || saveByTime_ > 0 ) )
   {
     if (convention_ != Online)
       throw cms::Exception("DQMFileSaver")
     << "Internal error, can save files by time"
     << " only in Online mode.";
 
     // Compute elapsed time in minutes.
     struct timeval tv;
     gettimeofday(&tv, 0);
 
     double totalelapsed = ((tv.tv_sec + tv.tv_usec*1e-6)
          - (start_.tv_sec + start_.tv_usec*1e-6)) / 60;
     double elapsed = ((tv.tv_sec + tv.tv_usec*1e-6)
               - (saved_.tv_sec + saved_.tv_usec*1e-6)) / 60;
 
     // Save if enough time has elapsed since the last save.
     if ( (saveByMinute_ > 0 && elapsed > saveByMinute_ ) ||
          (saveByTime_ > 0   && totalelapsed > saveByTime_ ) )
     {
       if ( saveByTime_ > 0 ) saveByTime_ *= 2;
       saved_ = tv;
       sprintf(suffix, "_R%09d_T%08d", irun_, int(totalelapsed));
       char rewrite[64]; sprintf(rewrite, "\\1Run %d/\\2/Run summary", irun_);
       saveForOnline(suffix, rewrite);
     }
   }
 }

void DQMFileSaver::beginJob ( void )

protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 358 of file DQMFileSaver.cc.

References ievent_, ilumi_, ilumiprev_, irun_, nevent_, nlumi_, and nrun_.

 {
   irun_ = ilumi_ = ilumiprev_ = ievent_ = -1;
   nrun_ = nlumi_ = nevent_ = 0;
 }

void DQMFileSaver::beginLuminosityBlock	(	const edm::LuminosityBlock &	l,
		const edm::EventSetup &
	)

protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 372 of file DQMFileSaver.cc.

References edm::LuminosityBlockBase::id(), ilumi_, ilumiprev_, edm::LuminosityBlockID::luminosityBlock(), and nlumi_.

 {
   ilumi_    = l.id().luminosityBlock();
   if (ilumiprev_ == -1) ilumiprev_ = ilumi_;
   ++nlumi_;
 }

void DQMFileSaver::beginRun	(	const edm::Run &	r,
		const edm::EventSetup &
	)

protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 365 of file DQMFileSaver.cc.

References forceRunNumber_, edm::RunBase::id(), irun_, nrun_, and edm::RunID::run().

 {
   irun_     = (forceRunNumber_ == -1 ? r.id().run() : forceRunNumber_);
   ++nrun_;
 }

void DQMFileSaver::endJob ( void )

protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 478 of file DQMFileSaver.cc.

References convention_, edm::hlt::Exception, forceRunNumber_, Offline, saveAtJobEnd_, saveForOffline(), and workflow_.

 { 
   if (saveAtJobEnd_)
   {
     if (convention_ == Offline && forceRunNumber_ > 0)
       saveForOffline(workflow_, forceRunNumber_, 0);
     else
       throw cms::Exception("DQMFileSaver")
     << "Internal error.  Can only save files at the end of the"
     << " job in Offline mode with run number overridden.";
   }
     
 }

void DQMFileSaver::endLuminosityBlock	(	const edm::LuminosityBlock &	,
		const edm::EventSetup &
	)

protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 430 of file DQMFileSaver.cc.

References convention_, edm::hlt::Exception, ilumi_, ilumiprev_, irun_, nlumi_, Offline, Online, saveByLumiSection_, saveForOffline(), saveForOnline(), createPayload::suffix, and workflow_.

 {
 
   if (ilumi_ > 0 && saveByLumiSection_ > 0 )
   {
     if (convention_ != Online && convention_ != Offline )
       throw cms::Exception("DQMFileSaver")
     << "Internal error, can save files at end of lumi block"
     << " only in Online or Offline mode.";
 
     if (convention_ == Online && nlumi_ == saveByLumiSection_) // insist on lumi section ordering
     {
       char suffix[64];
       char rewrite[128];
       sprintf(suffix, "_R%09d_L%06d", irun_, ilumi_);
       sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", irun_, ilumiprev_, ilumi_);
       saveForOnline(suffix, rewrite);
       ilumiprev_ = -1;
       nlumi_ = 0;
     }
     if (convention_ == Offline)
       saveForOffline(workflow_, irun_, ilumi_);
   }
 }

void DQMFileSaver::endRun	(	const edm::Run &	,
		const edm::EventSetup &
	)

protectedvirtual

Reimplemented from edm::EDAnalyzer.

Definition at line 456 of file DQMFileSaver.cc.

References convention_, edm::hlt::Exception, irun_, nrun_, Offline, Online, saveByRun_, saveForOffline(), saveForOnline(), createPayload::suffix, and workflow_.

 {
   if (irun_ > 0 && saveByRun_ > 0 && nrun_ == saveByRun_)
   {
     if (convention_ == Online)
     {
       char suffix[64]; sprintf(suffix, "_R%09d", irun_);
       char rewrite[64]; sprintf(rewrite, "\\1Run %d/\\2/Run summary", irun_);
       saveForOnline(suffix, rewrite);
     }
     else if (convention_ == Offline)
       saveForOffline(workflow_, irun_, 0);
     else
       throw cms::Exception("DQMFileSaver")
     << "Internal error.  Can only save files in endRun()"
     << " in Online and Offline modes.";
 
     nrun_ = 0;
   }
 }

void DQMFileSaver::saveForOffline	(	const std::string &	workflow,
		int	run,
		int	lumi
	)

private

Definition at line 32 of file DQMFileSaver.cc.

References DQMStore::bookFloat(), DQMStore::bookString(), DQMStore::cd(), gather_cfg::cout, dbe_, fileBaseName_, lut2db_cfg::filename, fileUpdate_, MonitorElement::Fill(), DQMStore::get(), edm::getReleaseVersion(), DQMStore::getSubdirs(), i, ilumi_, irun_, pastSavedFiles_, pos, runIsComplete_, DQMStore::save(), saveJobReport(), saveReference_, saveReferenceQMin_, DQMStore::SaveWithoutReference, DQMStore::setCurrentFolder(), dqm::qstatus::STATUS_OK, createPayload::suffix, and reco_calib_source_client_cfg::workflow.

Referenced by endJob(), endLuminosityBlock(), and endRun().

 {
 
   char suffix[64];
   sprintf(suffix, "R%09d", run);
 
   char rewrite[128];
   if (lumi == 0) // save for run
     sprintf(rewrite, "\\1Run %d/\\2/Run summary", run);
   else
     sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", irun_, ilumi_, ilumi_);
 
   size_t pos = 0;
   std::string wflow;
   wflow.reserve(workflow.size() + 3);
   wflow = workflow;
   while ((pos = wflow.find('/', pos)) != std::string::npos)
     wflow.replace(pos++, 1, "__");
     
   std::string filename = fileBaseName_ + suffix + wflow + ".root";
 
   if (lumi == 0) // save for run
   {
     // set run end flag
     dbe_->cd();
     dbe_->setCurrentFolder("Info/ProvInfo");
 
     // do this, because ProvInfo is not yet run in offline DQM
     MonitorElement* me = dbe_->get("Info/ProvInfo/CMSSW"); 
     if (!me) me = dbe_->bookString("CMSSW",edm::getReleaseVersion().c_str() );
     
     me = dbe_->get("Info/ProvInfo/runIsComplete");
     if (!me) me = dbe_->bookFloat("runIsComplete");
 
     if (me)
     { 
       if (runIsComplete_)
         me->Fill(1.);
       else
         me->Fill(0.);
     }
 
     dbe_->save(filename,
              "", 
          "^(Reference/)?([^/]+)", 
          rewrite,
          (DQMStore::SaveReferenceTag) saveReference_,
          saveReferenceQMin_,
          fileUpdate_);
   }
   else // save EventInfo folders for luminosity sections
   {
     std::vector<std::string> systems = (dbe_->cd(), dbe_->getSubdirs());
 
     std::cout << " DQMFileSaver: storing EventInfo folders for Run: " 
               << irun_ << ", Lumi Section: " << ilumi_ << ", Subsystems: " ;
           
     for (size_t i = 0, e = systems.size(); i != e; ++i) {
       if (systems[i] != "Reference") {
         dbe_->cd();
     std::cout << systems[i] << "  " ;
         dbe_->save(filename,
        systems[i]+"/EventInfo", "^(Reference/)?([^/]+)", rewrite,
        DQMStore::SaveWithoutReference,
        dqm::qstatus::STATUS_OK,
        fileUpdate_);
     // from now on update newly created file
     if (fileUpdate_=="RECREATE") fileUpdate_="UPDATE";
       }
     }
     std::cout << "\n";
   }
 
   if (pastSavedFiles_.size() == 0)
   {
     // save JobReport upon creation of file (once per job)
     saveJobReport(filename);
     pastSavedFiles_.push_back(filename);
   }
   
 }

void DQMFileSaver::saveForOnline	(	const std::string &	suffix,
		const std::string &	rewrite
	)

private

Definition at line 136 of file DQMFileSaver.cc.

References DQMStore::cd(), dbe_, doSaveForOnline(), fileBaseName_, DQMStore::get(), DQMStore::getSubdirs(), i, numKeepSavedFiles_, pastSavedFiles_, saveReference_, and saveReferenceQMin_.

Referenced by analyze(), endLuminosityBlock(), and endRun().

 {
   std::vector<std::string> systems = (dbe_->cd(), dbe_->getSubdirs());
 
   for (size_t i = 0, e = systems.size(); i != e; ++i)
   {
     if (systems[i] != "Reference")
     {
       dbe_->cd();
       if (MonitorElement* me = dbe_->get(systems[i] + "/EventInfo/processName"))
       {
     doSaveForOnline(pastSavedFiles_, numKeepSavedFiles_, dbe_,
             fileBaseName_ + me->getStringValue() + suffix + ".root",
             "", "^(Reference/)?([^/]+)", rewrite,
                     (DQMStore::SaveReferenceTag) saveReference_,
                     saveReferenceQMin_);
         return;
       }
     }
   }
 
   // if no EventInfo Folder is found, then store subsystem wise
   for (size_t i = 0, e = systems.size(); i != e; ++i)
     if (systems[i] != "Reference")
       doSaveForOnline(pastSavedFiles_, numKeepSavedFiles_, dbe_,
                       fileBaseName_ + systems[i] + suffix + ".root",
                   systems[i], "^(Reference/)?([^/]+)", rewrite,
                   (DQMStore::SaveReferenceTag) saveReference_,
                   saveReferenceQMin_);
 }

void DQMFileSaver::saveJobReport ( const std::string & filename )

private

Definition at line 168 of file DQMFileSaver.cc.

References info, edm::Service< T >::isAvailable(), and edm::JobReport::reportAnalysisFile().

Referenced by saveForOffline().

 {
 
   // Report the file to job report service.
   edm::Service<edm::JobReport> jr;
   if (jr.isAvailable())
   {
     std::map<std::string, std::string> info;
     info["Source"] = "DQMStore";
     info["FileClass"] = "DQM";
     jr->reportAnalysisFile(filename, info);
   }
 
 }