ThroughputService Class Reference

#include <ThroughputService.h>

Public Types
typedef dqm::reco::DQMStore	DQMStore

Public Member Functions
	ThroughputService (const edm::ParameterSet &, edm::ActivityRegistry &)
	~ThroughputService ()=default

Static Public Member Functions
static void	fillDescriptions (edm::ConfigurationDescriptions &descriptions)

Private Member Functions
void	postEndJob ()
void	postEvent (edm::StreamContext const &sc)
void	preallocate (edm::service::SystemBounds const &bounds)
void	preGlobalBeginRun (edm::GlobalContext const &gc)
void	preSourceEvent (edm::StreamID sid)

Private Attributes
std::atomic< uint32_t >	m_counter
const bool	m_dqm_bynproc
std::string	m_dqm_path
bool	m_enable_dqm
tbb::concurrent_vector< std::chrono::system_clock::time_point >	m_events
bool	m_print_event_summary
const uint32_t	m_resolution
dqm::reco::MonitorElement *	m_retired_events
dqm::reco::MonitorElement *	m_sourced_events
std::chrono::system_clock::time_point	m_startup
const double	m_time_range
const double	m_time_resolution

Detailed Description

Definition at line 30 of file ThroughputService.h.

Member Typedef Documentation

DQMStore

typedef dqm::reco::DQMStore ThroughputService::DQMStore

Definition at line 32 of file ThroughputService.h.

Constructor & Destructor Documentation

ThroughputService()

ThroughputService::ThroughputService	(	const edm::ParameterSet &	config,
		edm::ActivityRegistry &	registry
	)

Definition at line 35 of file ThroughputService.cc.

References m_events, postEndJob(), postEvent(), preallocate(), preGlobalBeginRun(), preSourceEvent(), edm::ActivityRegistry::watchPostEndJob(), edm::ActivityRegistry::watchPostEvent(), edm::ActivityRegistry::watchPreallocate(), edm::ActivityRegistry::watchPreGlobalBeginRun(), and edm::ActivityRegistry::watchPreSourceEvent().

    :  // startup time
      m_startup(std::chrono::system_clock::now()),
      // configuration
      m_resolution(config.getUntrackedParameter<uint32_t>("eventResolution")),
      m_counter(0),
      m_events(config.getUntrackedParameter<uint32_t>("eventRange") / m_resolution),  // allocate initial size
      m_print_event_summary(config.getUntrackedParameter<bool>("printEventSummary")),
      m_enable_dqm(config.getUntrackedParameter<bool>("enableDQM")),
      m_dqm_bynproc(m_enable_dqm ? config.getUntrackedParameter<bool>("dqmPathByProcesses") : false),
      m_dqm_path(m_enable_dqm ? config.getUntrackedParameter<std::string>("dqmPath") : ""),
      m_time_range(m_enable_dqm ? config.getUntrackedParameter<double>("timeRange") : 0.),
      m_time_resolution(m_enable_dqm ? config.getUntrackedParameter<double>("timeResolution") : 0.) {
  m_events.clear();  // erases all elements, but does not free internal arrays
  registry.watchPreallocate(this, &ThroughputService::preallocate);
  registry.watchPreGlobalBeginRun(this, &ThroughputService::preGlobalBeginRun);
  registry.watchPreSourceEvent(this, &ThroughputService::preSourceEvent);
  registry.watchPostEvent(this, &ThroughputService::postEvent);
  registry.watchPostEndJob(this, &ThroughputService::postEndJob);
}

~ThroughputService()

ThroughputService::~ThroughputService ( )

default

Member Function Documentation

fillDescriptions()

void ThroughputService::fillDescriptions ( edm::ConfigurationDescriptions &  descriptions )

static

Definition at line 19 of file ThroughputService.cc.

References edm::ConfigurationDescriptions::add(), submitPVResolutionJobs::desc, and or.

                                                                                   {
  edm::ParameterSetDescription desc;
  desc.addUntracked<uint32_t>("eventRange", 10000)->setComment("Preallocate a buffer for N events");
  desc.addUntracked<uint32_t>("eventResolution", 1)->setComment("Sample the processing time every N events");
  desc.addUntracked<bool>("printEventSummary", false);
  desc.ifValue(edm::ParameterDescription<bool>("enableDQM", true, false),  // "false" means untracked
               // parameters if "enableDQM" is "true"
               true >> (edm::ParameterDescription<bool>("dqmPathByProcesses", false, false) and
                        edm::ParameterDescription<std::string>("dqmPath", "HLT/Throughput", false) and
                        edm::ParameterDescription<double>("timeRange", 60000.0, false) and
                        edm::ParameterDescription<double>("timeResolution", 10.0, false)) or
                   // parameters if "enableDQM" is "false"
                   false >> edm::EmptyGroupDescription());
  descriptions.add("ThroughputService", desc);
}

postEndJob()

void ThroughputService::postEndJob ( )

private

Definition at line 125 of file ThroughputService.cc.

References gather_cfg::blocks, submitPVResolutionJobs::count, dumpMFGeometry_cfg::delta, mps_fire::i, info(), m_counter, m_events, m_print_event_summary, m_resolution, Mean, and TkAlMap::StdDev().

Referenced by ThroughputService().

                                   {
  if (m_counter < 2 * m_resolution) {
    // not enough mesurements to estimate the throughput
    edm::LogWarning("ThroughputService") << "Not enough events to measure the throughput with a resolution of "
                                         << m_resolution << " events";
    return;
  }

  edm::LogInfo info("ThroughputService");

  if (m_print_event_summary) {
    for (uint32_t i = 0; i < m_events.size(); ++i) {
      info << std::setw(8) << (i + 1) * m_resolution << ", " << std::setprecision(6) << edm::TimeOfDay(m_events[i])
           << "\n";
    }
    info << '\n';
  }

  // measure the time to process each block of m_resolution events
  uint32_t blocks = m_counter / m_resolution - 1;
  std::vector<double> delta(blocks);
  for (uint32_t i = 0; i < blocks; ++i) {
    delta[i] = std::chrono::duration_cast<std::chrono::duration<double>>(m_events[i + 1] - m_events[i]).count();
  }
  // measure the average and standard deviation of the time to process m_resolution
  double time_avg = TMath::Mean(delta.begin(), delta.begin() + blocks);
  double time_dev = TMath::StdDev(delta.begin(), delta.begin() + blocks);
  // compute the throughput and its standard deviation across the job
  double throughput_avg = double(m_resolution) / time_avg;
  double throughput_dev = double(m_resolution) * time_dev / time_avg / time_avg;

  info << "Average throughput: " << throughput_avg << " ± " << throughput_dev << " ev/s";
}

postEvent()

void ThroughputService::postEvent ( edm::StreamContext const &  sc )

private

Definition at line 113 of file ThroughputService.cc.

References submitPVResolutionJobs::count, dqm::impl::MonitorElement::Fill(), writeEcalDQMStatus::interval, m_counter, m_enable_dqm, m_events, m_resolution, m_retired_events, m_startup, submitPVValidationJobs::now, and cond::timestamp.

Referenced by ThroughputService().

                                                            {
  auto timestamp = std::chrono::system_clock::now();
  auto interval = std::chrono::duration_cast<std::chrono::duration<double>>(timestamp - m_startup).count();
  if (m_enable_dqm) {
    m_retired_events->Fill(interval);
  }
  ++m_counter;
  if (m_counter % m_resolution == 0) {
    m_events.push_back(timestamp);
  }
}

preallocate()

void ThroughputService::preallocate ( edm::service::SystemBounds const &  bounds )

private

Definition at line 56 of file ThroughputService.cc.

References m_dqm_bynproc, m_dqm_path, m_enable_dqm, and processor_model.

Referenced by ThroughputService().

                                                                        {
  auto concurrent_streams = bounds.maxNumberOfStreams();
  auto concurrent_threads = bounds.maxNumberOfThreads();

  if (m_enable_dqm and m_dqm_bynproc)
    m_dqm_path += fmt::sprintf(
        "/Running on %s with %d streams on %d threads", processor_model, concurrent_streams, concurrent_threads);
}

preGlobalBeginRun()

void ThroughputService::preGlobalBeginRun ( edm::GlobalContext const &  gc )

private

Definition at line 65 of file ThroughputService.cc.

References trigObjTnPSource_cfi::bins, dqm::implementation::IBooker::book1D(), c, m_dqm_path, m_enable_dqm, m_retired_events, m_sourced_events, m_time_range, m_time_resolution, FastTimerService_cff::range, alignCSCRings::s, dqm::implementation::NavigatorBase::setCurrentFolder(), dqm::impl::MonitorElement::setXTitle(), dqm::impl::MonitorElement::setYTitle(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by ThroughputService().

                                                                    {
  // if the DQMStore is available, book the DQM histograms
  // check that the DQMStore service is available
  if (m_enable_dqm and not edm::Service<dqm::legacy::DQMStore>().isAvailable()) {
    // the DQMStore is not available, disable all DQM plots
    m_enable_dqm = false;
    edm::LogWarning("ThroughputService") << "The DQMStore is not avalable, the DQM plots will not be generated";
  }

  if (m_enable_dqm) {
    std::string y_axis_title = fmt::sprintf("events / %g s", m_time_resolution);
    unsigned int bins = std::round(m_time_range / m_time_resolution);
    double range = bins * m_time_resolution;

    // clean characters that are deemed unsafe for DQM
    // see the definition of `s_safe` in DQMServices/Core/src/DQMStore.cc
    auto safe_for_dqm = "/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-+=_()# "s;
    for (auto& c : m_dqm_path)
      if (safe_for_dqm.find(c) == std::string::npos)
        c = '_';

    // define a callback that can book the histograms
    auto bookTransactionCallback = [&, this](DQMStore::IBooker& booker, DQMStore::IGetter&) {
      booker.setCurrentFolder(m_dqm_path);
      m_sourced_events = booker.book1D("throughput_sourced", "Throughput (sourced events)", bins, 0., range);
      m_sourced_events->setXTitle("time [s]");
      m_sourced_events->setYTitle(y_axis_title);
      m_retired_events = booker.book1D("throughput_retired", "Throughput (retired events)", bins, 0., range);
      m_retired_events->setXTitle("time [s]");
      m_retired_events->setYTitle(y_axis_title);
    };

    // book MonitorElement's for this run
    edm::Service<DQMStore>()->meBookerGetter(bookTransactionCallback);
  } else {
    m_sourced_events = nullptr;
    m_retired_events = nullptr;
  }
}

preSourceEvent()

void ThroughputService::preSourceEvent ( edm::StreamID  sid )

private

Definition at line 105 of file ThroughputService.cc.

References submitPVResolutionJobs::count, dqm::impl::MonitorElement::Fill(), writeEcalDQMStatus::interval, m_enable_dqm, m_sourced_events, m_startup, submitPVValidationJobs::now, and cond::timestamp.

Referenced by ThroughputService().

                                                      {
  auto timestamp = std::chrono::system_clock::now();
  auto interval = std::chrono::duration_cast<std::chrono::duration<double>>(timestamp - m_startup).count();
  if (m_enable_dqm) {
    m_sourced_events->Fill(interval);
  }
}

Member Data Documentation

m_counter

std::atomic<uint32_t> ThroughputService::m_counter

private

Definition at line 55 of file ThroughputService.h.

Referenced by postEndJob(), and postEvent().

m_dqm_bynproc

const bool ThroughputService::m_dqm_bynproc

private

Definition at line 61 of file ThroughputService.h.

Referenced by preallocate().

m_dqm_path

std::string ThroughputService::m_dqm_path

private

Definition at line 62 of file ThroughputService.h.

Referenced by preallocate(), and preGlobalBeginRun().

m_enable_dqm

bool ThroughputService::m_enable_dqm

private

Definition at line 60 of file ThroughputService.h.

Referenced by postEvent(), preallocate(), preGlobalBeginRun(), and preSourceEvent().

m_events

tbb::concurrent_vector<std::chrono::system_clock::time_point> ThroughputService::m_events

private

Definition at line 56 of file ThroughputService.h.

Referenced by postEndJob(), postEvent(), and ThroughputService().

m_print_event_summary

bool ThroughputService::m_print_event_summary

private

Definition at line 57 of file ThroughputService.h.

Referenced by postEndJob().

m_resolution

const uint32_t ThroughputService::m_resolution

private

Definition at line 54 of file ThroughputService.h.

Referenced by postEndJob(), and postEvent().

m_retired_events

dqm::reco::MonitorElement* ThroughputService::m_retired_events

private

Definition at line 49 of file ThroughputService.h.

Referenced by postEvent(), and preGlobalBeginRun().

m_sourced_events

dqm::reco::MonitorElement* ThroughputService::m_sourced_events

private

Definition at line 48 of file ThroughputService.h.

Referenced by preGlobalBeginRun(), and preSourceEvent().

m_startup

std::chrono::system_clock::time_point ThroughputService::m_startup

private

Definition at line 51 of file ThroughputService.h.

Referenced by postEvent(), and preSourceEvent().

m_time_range

const double ThroughputService::m_time_range

private

Definition at line 63 of file ThroughputService.h.

Referenced by preGlobalBeginRun().

m_time_resolution

const double ThroughputService::m_time_resolution

private

Definition at line 64 of file ThroughputService.h.

Referenced by preGlobalBeginRun().