Public Member Functions
	ZombieKillerService (edm::ParameterSet const &, edm::ActivityRegistry &)

Static Public Member Functions
static void	fillDescriptions (ConfigurationDescriptions &descriptions)

Private Member Functions
void	checkForZombie ()

void	notAZombieYet ()

void	startThread ()

void	stopThread ()

Private Attributes
const unsigned int	m_checkThreshold

bool	m_jobDone

std::condition_variable	m_jobDoneCondition

std::mutex	m_jobDoneMutex

std::atomic< unsigned int >	m_numberChecksWhenNotAlive

const unsigned int	m_secsBetweenChecks

std::atomic< bool >	m_stillAlive

std::thread	m_watchingThread

Detailed Description

Definition at line 28 of file ZombieKillerService.cc.

Constructor & Destructor Documentation

ZombieKillerService::ZombieKillerService	(	edm::ParameterSet const &	iPSet,
		edm::ActivityRegistry &	iRegistry
	)

Definition at line 71 of file ZombieKillerService.cc.

                                                                                                     :
 m_checkThreshold(iPSet.getUntrackedParameter<unsigned int>("numberOfAllowedFailedChecksInARow")),
 m_secsBetweenChecks(iPSet.getUntrackedParameter<unsigned int>("secondsBetweenChecks")),
 m_jobDone(false),
 m_stillAlive(true),
 m_numberChecksWhenNotAlive(0)
 {
   iRegistry.watchPostBeginJob([this](){ startThread(); } );
   iRegistry.watchPostEndJob([this]() {stopThread(); } );
   
   iRegistry.watchPreSourceRun([this](RunIndex){notAZombieYet();});
   iRegistry.watchPostSourceRun([this](RunIndex){notAZombieYet();});
   
   iRegistry.watchPreSourceLumi([this](LuminosityBlockIndex){notAZombieYet();});
   iRegistry.watchPostSourceLumi([this](LuminosityBlockIndex){notAZombieYet();});
 
   iRegistry.watchPreSourceEvent([this](StreamID){notAZombieYet();});
   iRegistry.watchPostSourceEvent([this](StreamID){notAZombieYet();});
   
   iRegistry.watchPreModuleBeginStream([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleBeginStream([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
 
   iRegistry.watchPreModuleEndStream([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleEndStream([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   
   iRegistry.watchPreModuleEndJob([this](ModuleDescription const&) {notAZombieYet();});
   iRegistry.watchPostModuleEndJob([this](ModuleDescription const&) {notAZombieYet();});
   iRegistry.watchPreModuleEvent([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleEvent([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
 
   iRegistry.watchPreModuleStreamBeginRun([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleStreamBeginRun([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
 
   iRegistry.watchPreModuleStreamEndRun([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleStreamEndRun([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
 
   iRegistry.watchPreModuleStreamBeginLumi([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleStreamBeginLumi([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   
   iRegistry.watchPreModuleStreamEndLumi([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleStreamEndLumi([this](StreamContext const&, ModuleCallingContext const&){notAZombieYet();});
 
   iRegistry.watchPreModuleGlobalBeginRun([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleGlobalBeginRun([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
   
   iRegistry.watchPreModuleGlobalEndRun([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleGlobalEndRun([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
   
   iRegistry.watchPreModuleGlobalBeginLumi([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleGlobalBeginLumi([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
   
   iRegistry.watchPreModuleGlobalEndLumi([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
   iRegistry.watchPostModuleGlobalEndLumi([this](GlobalContext const&, ModuleCallingContext const&){notAZombieYet();});
 
   
 }

Member Function Documentation

void ZombieKillerService::checkForZombie ( )

private

Definition at line 159 of file ZombieKillerService.cc.

References m_checkThreshold, m_numberChecksWhenNotAlive, m_secsBetweenChecks, and m_stillAlive.

Referenced by startThread().

                                     {
   if (not m_stillAlive) {
     ++m_numberChecksWhenNotAlive;
     if(m_numberChecksWhenNotAlive > m_checkThreshold) {
       edm::LogError("JobStuck")<<"Too long since the job has last made progress.";
       std::terminate();
     } else {
       edm::LogWarning("JobProgressing")<<"It has been "<<m_numberChecksWhenNotAlive*m_secsBetweenChecks<<" seconds since job seen progressing";
     }
   }
   m_stillAlive = false;
 }

void ZombieKillerService::fillDescriptions ( ConfigurationDescriptions & descriptions )

static

Definition at line 201 of file ZombieKillerService.cc.

References edm::ConfigurationDescriptions::add(), edm::ParameterSetDescription::addUntracked(), and DEFINE_FWK_SERVICE.

                                                                              {
   ParameterSetDescription desc;
   desc.addUntracked<unsigned int>("secondsBetweenChecks", 60)->setComment("Number of seconds to wait between checking if progress has been made.");
   desc.addUntracked<unsigned int>("numberOfAllowedFailedChecksInARow", 3)->setComment("Number of allowed checks in a row with no progress.");
   descriptions.add("ZombieKillerService", desc);
 }

void ZombieKillerService::notAZombieYet ( )

private

Definition at line 153 of file ZombieKillerService.cc.

References m_numberChecksWhenNotAlive, and m_stillAlive.

Referenced by ZombieKillerService().

                                    {
   m_numberChecksWhenNotAlive = 0;
   m_stillAlive = true;
 }

void ZombieKillerService::startThread ( )

private

Definition at line 173 of file ZombieKillerService.cc.

References checkForZombie(), CommonMethods::lock(), m_jobDoneCondition, m_jobDoneMutex, m_secsBetweenChecks, m_watchingThread, and seconds().

Referenced by ZombieKillerService().

                                  {
   m_watchingThread = std::thread([this]() {
 
     std::unique_lock<std::mutex> lock(m_jobDoneMutex);
     while(not m_jobDoneCondition.wait_for(lock,
                                           std::chrono::seconds(m_secsBetweenChecks),
                                           [this]()->bool
                                           {
                                             return m_jobDone;
                                           }))
     {
       //we timed out
       checkForZombie();
     }
   });
 }

void ZombieKillerService::stopThread ( )

private

Definition at line 191 of file ZombieKillerService.cc.

References m_jobDone, m_jobDoneCondition, m_jobDoneMutex, and m_watchingThread.

Referenced by ZombieKillerService().

                                 {
   {
     std::lock_guard<std::mutex> guard(m_jobDoneMutex);
     m_jobDone=true;
   }
   m_jobDoneCondition.notify_all();
   m_watchingThread.join();
 }