Public Member Functions
	ZombieKillerService (edm::ParameterSet const &, edm::ActivityRegistry &)

Static Public Member Functions
static void	fillDescriptions (ConfigurationDescriptions &descriptions)

Private Member Functions
void	checkForZombie ()

void	notAZombieYet ()

void	startThread ()

void	stopThread ()

Private Attributes
const unsigned int	m_checkThreshold

bool	m_jobDone

std::condition_variable	m_jobDoneCondition

std::mutex	m_jobDoneMutex

std::atomic< unsigned int >	m_numberChecksWhenNotAlive

const unsigned int	m_secsBetweenChecks

std::atomic< bool >	m_stillAlive

std::thread	m_watchingThread

Detailed Description

Definition at line 27 of file ZombieKillerService.cc.

Constructor & Destructor Documentation

ZombieKillerService::ZombieKillerService	(	edm::ParameterSet const &	iPSet,
		edm::ActivityRegistry &	iRegistry
	)

Definition at line 65 of file ZombieKillerService.cc.

     : m_checkThreshold(iPSet.getUntrackedParameter<unsigned int>("numberOfAllowedFailedChecksInARow")),
       m_secsBetweenChecks(iPSet.getUntrackedParameter<unsigned int>("secondsBetweenChecks")),
       m_jobDone(false),
       m_stillAlive(true),
       m_numberChecksWhenNotAlive(0) {
   iRegistry.watchPostBeginJob([this]() { startThread(); });
   iRegistry.watchPostEndJob([this]() { stopThread(); });
 
   iRegistry.watchPreSourceRun([this](RunIndex) { notAZombieYet(); });
   iRegistry.watchPostSourceRun([this](RunIndex) { notAZombieYet(); });
 
   iRegistry.watchPreSourceLumi([this](LuminosityBlockIndex) { notAZombieYet(); });
   iRegistry.watchPostSourceLumi([this](LuminosityBlockIndex) { notAZombieYet(); });
 
   iRegistry.watchPreSourceEvent([this](StreamID) { notAZombieYet(); });
   iRegistry.watchPostSourceEvent([this](StreamID) { notAZombieYet(); });
 
   iRegistry.watchPreModuleBeginStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleBeginStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleEndStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleEndStream([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleEndJob([this](ModuleDescription const&) { notAZombieYet(); });
   iRegistry.watchPostModuleEndJob([this](ModuleDescription const&) { notAZombieYet(); });
   iRegistry.watchPreModuleEvent([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleEvent([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleStreamBeginRun(
       [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleStreamBeginRun(
       [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleStreamEndRun([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleStreamEndRun([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleStreamBeginLumi(
       [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleStreamBeginLumi(
       [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleStreamEndLumi([this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleStreamEndLumi(
       [this](StreamContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleGlobalBeginRun(
       [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleGlobalBeginRun(
       [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleGlobalEndRun([this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleGlobalEndRun([this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleGlobalBeginLumi(
       [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleGlobalBeginLumi(
       [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 
   iRegistry.watchPreModuleGlobalEndLumi([this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
   iRegistry.watchPostModuleGlobalEndLumi(
       [this](GlobalContext const&, ModuleCallingContext const&) { notAZombieYet(); });
 }

Member Function Documentation

void ZombieKillerService::checkForZombie ( )

private

Definition at line 158 of file ZombieKillerService.cc.

References m_checkThreshold, m_numberChecksWhenNotAlive, m_secsBetweenChecks, and m_stillAlive.

Referenced by startThread().

                                          {
   if (not m_stillAlive) {
     ++m_numberChecksWhenNotAlive;
     if (m_numberChecksWhenNotAlive > m_checkThreshold) {
       edm::LogError("JobStuck") << "Too long since the job has last made progress.";
       std::terminate();
     } else {
       edm::LogWarning("JobProgressing") << "It has been " << m_numberChecksWhenNotAlive * m_secsBetweenChecks
                                         << " seconds since job seen progressing";
     }
   }
   m_stillAlive = false;
 }

void ZombieKillerService::fillDescriptions ( ConfigurationDescriptions & descriptions )

static

Definition at line 192 of file ZombieKillerService.cc.

References edm::ConfigurationDescriptions::add(), edm::ParameterSetDescription::addUntracked(), and DEFINE_FWK_SERVICE.

                                                                                   {
   ParameterSetDescription desc;
   desc.addUntracked<unsigned int>("secondsBetweenChecks", 60)
       ->setComment("Number of seconds to wait between checking if progress has been made.");
   desc.addUntracked<unsigned int>("numberOfAllowedFailedChecksInARow", 3)
       ->setComment("Number of allowed checks in a row with no progress.");
   descriptions.add("ZombieKillerService", desc);
 }

void ZombieKillerService::notAZombieYet ( )

private

Definition at line 153 of file ZombieKillerService.cc.

References m_numberChecksWhenNotAlive, and m_stillAlive.

Referenced by ZombieKillerService().

                                         {
   m_numberChecksWhenNotAlive = 0;
   m_stillAlive = true;
 }

void ZombieKillerService::startThread ( )

private

Definition at line 172 of file ZombieKillerService.cc.

References checkForZombie(), CommonMethods::lock(), m_jobDoneCondition, m_jobDoneMutex, m_secsBetweenChecks, m_watchingThread, and seconds().

Referenced by ZombieKillerService().

                                       {
   m_watchingThread = std::thread([this]() {
     std::unique_lock<std::mutex> lock(m_jobDoneMutex);
     while (not m_jobDoneCondition.wait_for(
         lock, std::chrono::seconds(m_secsBetweenChecks), [this]() -> bool { return m_jobDone; })) {
       //we timed out
       checkForZombie();
     }
   });
 }

void ZombieKillerService::stopThread ( )

private

Definition at line 183 of file ZombieKillerService.cc.

References m_jobDone, m_jobDoneCondition, m_jobDoneMutex, and m_watchingThread.

Referenced by ZombieKillerService().

                                      {
   {
     std::lock_guard<std::mutex> guard(m_jobDoneMutex);
     m_jobDone = true;
   }
   m_jobDoneCondition.notify_all();
   m_watchingThread.join();
 }