edm::ZombieKillerService Class Reference

Public Member Functions
	ZombieKillerService (edm::ParameterSet const &, edm::ActivityRegistry &)

Static Public Member Functions
static void	fillDescriptions (ConfigurationDescriptions &descriptions)

Private Member Functions
void	checkForZombie ()
void	notAZombieYet ()
void	startThread ()
void	stopThread ()

Private Attributes
const unsigned int	m_checkThreshold
bool	m_jobDone
std::condition_variable	m_jobDoneCondition
std::mutex	m_jobDoneMutex
std::atomic< unsigned int >	m_numberChecksWhenNotAlive
const unsigned int	m_secsBetweenChecks
std::atomic< bool >	m_stillAlive
std::thread	m_watchingThread

Detailed Description

Definition at line 28 of file ZombieKillerService.cc.

Constructor & Destructor Documentation

ZombieKillerService()

ZombieKillerService::ZombieKillerService	(	edm::ParameterSet const &	iPSet,
		edm::ActivityRegistry &	iRegistry
	)

Definition at line 59 of file ZombieKillerService.cc.

References notAZombieYet(), startThread(), stopThread(), edm::ActivityRegistry::watchPostBeginJob(), edm::ActivityRegistry::watchPostEndJob(), edm::ActivityRegistry::watchPostModuleBeginStream(), edm::ActivityRegistry::watchPostModuleEndJob(), edm::ActivityRegistry::watchPostModuleEndStream(), edm::ActivityRegistry::watchPostModuleEvent(), edm::ActivityRegistry::watchPostModuleGlobalBeginLumi(), edm::ActivityRegistry::watchPostModuleGlobalBeginRun(), edm::ActivityRegistry::watchPostModuleGlobalEndLumi(), edm::ActivityRegistry::watchPostModuleGlobalEndRun(), edm::ActivityRegistry::watchPostModuleStreamBeginLumi(), edm::ActivityRegistry::watchPostModuleStreamBeginRun(), edm::ActivityRegistry::watchPostModuleStreamEndLumi(), edm::ActivityRegistry::watchPostModuleStreamEndRun(), edm::ActivityRegistry::watchPostSourceEvent(), edm::ActivityRegistry::watchPostSourceLumi(), edm::ActivityRegistry::watchPostSourceRun(), edm::ActivityRegistry::watchPreModuleBeginStream(), edm::ActivityRegistry::watchPreModuleEndJob(), edm::ActivityRegistry::watchPreModuleEndStream(), edm::ActivityRegistry::watchPreModuleEvent(), edm::ActivityRegistry::watchPreModuleGlobalBeginLumi(), edm::ActivityRegistry::watchPreModuleGlobalBeginRun(), edm::ActivityRegistry::watchPreModuleGlobalEndLumi(), edm::ActivityRegistry::watchPreModuleGlobalEndRun(), edm::ActivityRegistry::watchPreModuleStreamBeginLumi(), edm::ActivityRegistry::watchPreModuleStreamBeginRun(), edm::ActivityRegistry::watchPreModuleStreamEndLumi(), edm::ActivityRegistry::watchPreModuleStreamEndRun(), edm::ActivityRegistry::watchPreSourceEvent(), edm::ActivityRegistry::watchPreSourceLumi(), and edm::ActivityRegistry::watchPreSourceRun().

    : m_checkThreshold(iPSet.getUntrackedParameter<unsigned int>("numberOfAllowedFailedChecksInARow")),
      m_secsBetweenChecks(iPSet.getUntrackedParameter<unsigned int>("secondsBetweenChecks")),
      m_jobDone(false),
      m_stillAlive(true),
      m_numberChecksWhenNotAlive(0) {
  iRegistry.watchPostBeginJob([this]() { startThread(); });
  iRegistry.watchPostEndJob([this]() { stopThread(); });

  iRegistry.watchPreSourceRun([this](RunIndex) { notAZombieYet(); });
  iRegistry.watchPostSourceRun([this](RunIndex) { notAZombieYet(); });

  iRegistry.watchPreSourceLumi([this](LuminosityBlockIndex) { notAZombieYet(); });
  iRegistry.watchPostSourceLumi([this](LuminosityBlockIndex) { notAZombieYet(); });

  iRegistry.watchPreSourceEvent([this](StreamID) { notAZombieYet(); });
  iRegistry.watchPostSourceEvent([this](StreamID) { notAZombieYet(); });

  iRegistry.watchPreModuleBeginStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleBeginStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleEndStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleEndStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleEndJob([this](ModuleDescription const&) { notAZombieYet(); });
  iRegistry.watchPostModuleEndJob([this](ModuleDescription const&) { notAZombieYet(); });
  iRegistry.watchPreModuleEvent([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleEvent([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleStreamBeginRun(
      [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleStreamBeginRun(
      [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleStreamEndRun([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleStreamEndRun([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleStreamBeginLumi(
      [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleStreamBeginLumi(
      [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleStreamEndLumi([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleStreamEndLumi(
      [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleGlobalBeginRun(
      [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleGlobalBeginRun(
      [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleGlobalEndRun([this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleGlobalEndRun([this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleGlobalBeginLumi(
      [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleGlobalBeginLumi(
      [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });

  iRegistry.watchPreModuleGlobalEndLumi([this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
  iRegistry.watchPostModuleGlobalEndLumi(
      [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
}

Member Function Documentation

checkForZombie()

void ZombieKillerService::checkForZombie ( )

private

Definition at line 152 of file ZombieKillerService.cc.

References m_checkThreshold, m_numberChecksWhenNotAlive, m_secsBetweenChecks, and m_stillAlive.

Referenced by startThread().

                                         {
  if (not m_stillAlive) {
    ++m_numberChecksWhenNotAlive;
    if (m_numberChecksWhenNotAlive > m_checkThreshold) {
      edm::LogError("JobStuck") << "Too long since the job has last made progress.";
      std::terminate();
    } else {
      edm::LogWarning("JobProgressing") << "It has been " << m_numberChecksWhenNotAlive * m_secsBetweenChecks
                                        << " seconds since job seen progressing";
    }
  }
  m_stillAlive = false;
}

fillDescriptions()

void ZombieKillerService::fillDescriptions ( ConfigurationDescriptions &  descriptions )

static

Definition at line 186 of file ZombieKillerService.cc.

References edm::ConfigurationDescriptions::add(), and submitPVResolutionJobs::desc.

                                                                                  {
  ParameterSetDescription desc;
  desc.addUntracked<unsigned int>("secondsBetweenChecks", 60)
      ->setComment("Number of seconds to wait between checking if progress has been made.");
  desc.addUntracked<unsigned int>("numberOfAllowedFailedChecksInARow", 3)
      ->setComment("Number of allowed checks in a row with no progress.");
  descriptions.add("ZombieKillerService", desc);
}

notAZombieYet()

void ZombieKillerService::notAZombieYet ( )

private

Definition at line 147 of file ZombieKillerService.cc.

References m_numberChecksWhenNotAlive, and m_stillAlive.

Referenced by ZombieKillerService().

                                        {
  m_numberChecksWhenNotAlive = 0;
  m_stillAlive = true;
}

startThread()

void ZombieKillerService::startThread ( )

private

Definition at line 166 of file ZombieKillerService.cc.

References checkForZombie(), CommonMethods::lock(), m_jobDoneCondition, m_jobDoneMutex, m_secsBetweenChecks, m_watchingThread, and seconds().

Referenced by ZombieKillerService().

                                      {
  m_watchingThread = std::thread([this]() {
    std::unique_lock<std::mutex> lock(m_jobDoneMutex);
    while (not m_jobDoneCondition.wait_for(
        lock, std::chrono::seconds(m_secsBetweenChecks), [this]() -> bool { return m_jobDone; })) {
      //we timed out
      checkForZombie();
    }
  });
}

stopThread()

void ZombieKillerService::stopThread ( )

private

Definition at line 177 of file ZombieKillerService.cc.

References m_jobDone, m_jobDoneCondition, m_jobDoneMutex, and m_watchingThread.

Referenced by ZombieKillerService().

                                     {
  {
    std::lock_guard<std::mutex> guard(m_jobDoneMutex);
    m_jobDone = true;
  }
  m_jobDoneCondition.notify_all();
  m_watchingThread.join();
}

Member Data Documentation

m_checkThreshold

const unsigned int edm::ZombieKillerService::m_checkThreshold

private

Definition at line 35 of file ZombieKillerService.cc.

Referenced by checkForZombie().

m_jobDone

bool edm::ZombieKillerService::m_jobDone

private

Definition at line 40 of file ZombieKillerService.cc.

Referenced by stopThread().

m_jobDoneCondition

std::condition_variable edm::ZombieKillerService::m_jobDoneCondition

private

Definition at line 38 of file ZombieKillerService.cc.

Referenced by startThread(), and stopThread().

m_jobDoneMutex

std::mutex edm::ZombieKillerService::m_jobDoneMutex

private

Definition at line 39 of file ZombieKillerService.cc.

Referenced by startThread(), and stopThread().

m_numberChecksWhenNotAlive

std::atomic<unsigned int> edm::ZombieKillerService::m_numberChecksWhenNotAlive

private

Definition at line 42 of file ZombieKillerService.cc.

Referenced by checkForZombie(), and notAZombieYet().

m_secsBetweenChecks

const unsigned int edm::ZombieKillerService::m_secsBetweenChecks

private

Definition at line 36 of file ZombieKillerService.cc.

Referenced by checkForZombie(), and startThread().

m_stillAlive

std::atomic<bool> edm::ZombieKillerService::m_stillAlive

private

Definition at line 41 of file ZombieKillerService.cc.

Referenced by checkForZombie(), and notAZombieYet().

m_watchingThread

std::thread edm::ZombieKillerService::m_watchingThread

private

Definition at line 37 of file ZombieKillerService.cc.

Referenced by startThread(), and stopThread().