edm::service::InitRootHandlers Class Reference

Inheritance diagram for edm::service::InitRootHandlers:

Classes
class	ThreadTracker

Public Member Functions
	InitRootHandlers (ParameterSet const &pset, ActivityRegistry &iReg)
	~InitRootHandlers () override
Public Member Functions inherited from edm::RootHandlers
template<typename F >
void	ignoreWarningsWhileDoing (F iFunc, SeverityLevel level=SeverityLevel::kWarning)
virtual	~RootHandlers ()

Static Public Member Functions
static void	fillDescriptions (ConfigurationDescriptions &descriptions)
static void	stacktraceFromThread ()
static int	stackTracePause ()
static const ThreadTracker::Container_type &	threadIDs ()

Static Public Attributes
static std::atomic< std::size_t >	doneModules_
static std::vector< std::array< char, moduleBufferSize > >	moduleListBuffers_
static std::atomic< std::size_t >	nextModule_

Private Member Functions
void	cachePidInfo ()
void	enableWarnings_ () override
void	ignoreWarnings_ (edm::RootHandlers::SeverityLevel level) override
void	willBeUsingThreads () override

Static Private Member Functions
static char const *const *	getPstackArgv ()
static void	stacktraceHelperThread ()

Private Attributes
bool	autoClassParser_
bool	autoLibraryLoader_
bool	interactiveDebug_
bool	loadAllDictionaries_
bool	resetErrHandler_
std::shared_ptr< const void >	sigAbrtHandler_
std::shared_ptr< const void >	sigBusHandler_
std::shared_ptr< const void >	sigFpeHandler_
std::shared_ptr< const void >	sigIllHandler_
std::shared_ptr< const void >	sigSegvHandler_
std::shared_ptr< const void >	sigTermHandler_
bool	unloadSigHandler_

Static Private Attributes
static int	childToParent_ [2] = {-1, -1}
static std::unique_ptr< std::thread >	helperThread_
static int	parentToChild_ [2] = {-1, -1}
static char	pidString_ [pidStringLength_] = {}
static constexpr int	pidStringLength_ = 200
static char const *const	pstackArgv_ [] = {pstackName, dashC, InitRootHandlers::pidString_, nullptr}
static int	stackTracePause_ = 300
static std::unique_ptr< ThreadTracker >	threadTracker_

Friends
int	cmssw_stacktrace (void *)

Additional Inherited Members
Public Types inherited from edm::RootHandlers
enum	SeverityLevel {   SeverityLevel::kInfo, SeverityLevel::kWarning, SeverityLevel::kError, SeverityLevel::kSysError,   SeverityLevel::kFatal }

Detailed Description

Definition at line 69 of file InitRootHandlers.cc.

Constructor & Destructor Documentation

InitRootHandlers()

edm::service::InitRootHandlers::InitRootHandlers ( ParameterSet const &  pset, ActivityRegistry &  iReg  )

explicit

Definition at line 771 of file InitRootHandlers.cc.

References autoClassParser_, autoLibraryLoader_, edm::TypeWithDict::byName(), cachePidInfo(), HLT_2023v12_cff::debugLevel, edm::hasDictionary(), edm::installCustomHandler(), edm::service::SystemBounds::maxNumberOfThreads(), moduleListBuffers_, muonDTDigis_cfi::pset, resetErrHandler_, edm::setRefCoreStreamerInTClass(), sigAbrtHandler_, sigBusHandler_, sigFpeHandler_, sigIllHandler_, sigSegvHandler_, watchdog::SIGTERM, sigTermHandler_, stackTracePause_, threadTracker_, unloadSigHandler_, trackerHitRTTI::vector, edm::ActivityRegistry::watchPostEndJob(), edm::ActivityRegistry::watchPostModuleConstruction(), edm::ActivityRegistry::watchPreallocate(), and edm::ActivityRegistry::watchPreModuleConstruction().

        : RootHandlers(),
          unloadSigHandler_(pset.getUntrackedParameter<bool>("UnloadRootSigHandler")),
          resetErrHandler_(pset.getUntrackedParameter<bool>("ResetRootErrHandler")),
          loadAllDictionaries_(pset.getUntrackedParameter<bool>("LoadAllDictionaries")),
          autoLibraryLoader_(loadAllDictionaries_ or pset.getUntrackedParameter<bool>("AutoLibraryLoader")),
          autoClassParser_(pset.getUntrackedParameter<bool>("AutoClassParser")),
          interactiveDebug_(pset.getUntrackedParameter<bool>("InteractiveDebug")) {
      stackTracePause_ = pset.getUntrackedParameter<int>("StackTracePauseTime");

      if (not threadTracker_) {
        threadTracker_ = std::make_unique<ThreadTracker>();
        iReg.watchPostEndJob([]() {
          if (threadTracker_) {
            threadTracker_->observe(false);
          }
        });
      }

      if (unloadSigHandler_) {
        // Deactivate all the Root signal handlers and restore the system defaults
        gSystem->ResetSignal(kSigChild);
        gSystem->ResetSignal(kSigBus);
        gSystem->ResetSignal(kSigSegmentationViolation);
        gSystem->ResetSignal(kSigIllegalInstruction);
        gSystem->ResetSignal(kSigSystem);
        gSystem->ResetSignal(kSigPipe);
        gSystem->ResetSignal(kSigAlarm);
        gSystem->ResetSignal(kSigUrgent);
        gSystem->ResetSignal(kSigFloatingException);
        gSystem->ResetSignal(kSigWindowChanged);
      } else if (pset.getUntrackedParameter<bool>("AbortOnSignal")) {
        cachePidInfo();

        //NOTE: ROOT can also be told to abort on these kinds of problems BUT
        // it requires an TApplication to be instantiated which causes problems
        gSystem->ResetSignal(kSigBus);
        gSystem->ResetSignal(kSigSegmentationViolation);
        gSystem->ResetSignal(kSigIllegalInstruction);
        gSystem->ResetSignal(kSigFloatingException);
        installCustomHandler(SIGBUS, sig_dostack_then_abort);
        sigBusHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGBUS, sig_abort); });
        installCustomHandler(SIGSEGV, sig_dostack_then_abort);
        sigSegvHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGSEGV, sig_abort); });
        installCustomHandler(SIGILL, sig_dostack_then_abort);
        sigIllHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGILL, sig_abort); });
        installCustomHandler(SIGTERM, sig_dostack_then_abort);
        sigTermHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGTERM, sig_abort); });
        installCustomHandler(SIGFPE, sig_dostack_then_abort);
        sigFpeHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGFPE, sig_abort); });
        installCustomHandler(SIGABRT, sig_dostack_then_abort);
        sigAbrtHandler_ = std::shared_ptr<const void>(nullptr, [](void*) {
          signal(SIGABRT, SIG_DFL);  // release SIGABRT to default
        });
      }

      iReg.watchPreallocate([](edm::service::SystemBounds const& iBounds) {
        if (iBounds.maxNumberOfThreads() > moduleListBuffers_.size()) {
          moduleListBuffers_.resize(iBounds.maxNumberOfThreads());
        }
      });

      if (resetErrHandler_) {
        // Replace the Root error handler with one that uses the MessageLogger
        SetErrorHandler(RootErrorHandler);
      }

      // Enable automatic Root library loading.
      if (autoLibraryLoader_) {
        gInterpreter->SetClassAutoloading(1);
      }

      // Enable/disable automatic parsing of headers
      if (not autoClassParser_) {
        // Disable automatic parsing of headers during module construction
        iReg.watchPreModuleConstruction(
            [](edm::ModuleDescription const&) { gInterpreter->SetClassAutoparsing(false); });
        iReg.watchPostModuleConstruction(
            [](edm::ModuleDescription const&) { gInterpreter->SetClassAutoparsing(true); });
      }

      // Set ROOT parameters.
      TTree::SetMaxTreeSize(kMaxLong64);
      TH1::AddDirectory(kFALSE);
      //G__SetCatchException(0);

      // Set custom streamers
      setRefCoreStreamerInTClass();

      // Load the library containing dictionaries for std:: classes, if not already loaded.
      if (!hasDictionary(typeid(std::vector<std::vector<unsigned int>>))) {
        TypeWithDict::byName("std::vector<std::vector<unsigned int> >");
      }

      int debugLevel = pset.getUntrackedParameter<int>("DebugLevel");
      if (debugLevel > 0) {
        gDebug = debugLevel;
      }

      // Enable Root implicit multi-threading
      bool imt = pset.getUntrackedParameter<bool>("EnableIMT");
      if (imt && not ROOT::IsImplicitMTEnabled()) {
        //cmsRun uses global_control to set the number of allowed threads to use
        // we need to tell ROOT the same value in order to avoid unnecessary warnings
        ROOT::EnableImplicitMT(
            oneapi::tbb::global_control::active_value(oneapi::tbb::global_control::max_allowed_parallelism));
      }
    }

~InitRootHandlers()

edm::service::InitRootHandlers::~InitRootHandlers ( )

override

Definition at line 880 of file InitRootHandlers.cc.

References f, getGTfromDQMFile::obj, and threadTracker_.

                                        {
      // close all open ROOT files
      TIter iter(gROOT->GetListOfFiles());
      TObject* obj = nullptr;
      while (nullptr != (obj = iter.Next())) {
        TFile* f = dynamic_cast<TFile*>(obj);
        if (f) {
          // We get a new iterator each time,
          // because closing a file can invalidate the iterator
          f->Close();
          iter = TIter(gROOT->GetListOfFiles());
        }
      }
      //disengage from TBB to avoid possible at exit problems
      threadTracker_.reset();
    }

Member Function Documentation

cachePidInfo()

void edm::service::InitRootHandlers::cachePidInfo ( )

private

Definition at line 946 of file InitRootHandlers.cc.

References childToParent_, helperThread_, interactiveDebug_, edm::errors::OtherCMS, parentToChild_, pidString_, pidStringLength_, stacktraceHelperThread(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by InitRootHandlers().

                                        {
      if (helperThread_) {
        //Another InitRootHandlers was initialized in this job, possibly
        // because multiple EventProcessors are being used.
        //In that case, we are already all setup
        return;
      }
      std::string gdbcmd{"date; gdb -quiet -p %d"};
      if (!interactiveDebug_) {
        gdbcmd +=
            " 2>&1 <<EOF |\n"
            "set width 0\n"
            "set height 0\n"
            "set pagination no\n"
            "thread apply all bt\n"
            "EOF\n"
            "/bin/sed -n -e 's/^\\((gdb) \\)*//' -e '/^#/p' -e '/^Thread/p'";
      }
      if (snprintf(pidString_, pidStringLength_ - 1, gdbcmd.c_str(), getpid()) >= pidStringLength_) {
        std::ostringstream sstr;
        sstr << "Unable to pre-allocate stacktrace handler information";
        edm::Exception except(edm::errors::OtherCMS, sstr.str());
        throw except;
      }

      // These are initialized to -1; harmless to close an invalid FD.
      // If this is called post-fork, we don't want to be communicating on
      // these FDs as they are used internally by the parent.
      close(childToParent_[0]);
      close(childToParent_[1]);
      childToParent_[0] = -1;
      childToParent_[1] = -1;
      close(parentToChild_[0]);
      close(parentToChild_[1]);
      parentToChild_[0] = -1;
      parentToChild_[1] = -1;

      if (-1 == pipe2(childToParent_, O_CLOEXEC)) {
        std::ostringstream sstr;
        sstr << "Failed to create child-to-parent pipes (errno=" << errno << "): " << strerror(errno);
        edm::Exception except(edm::errors::OtherCMS, sstr.str());
        throw except;
      }

      if (-1 == pipe2(parentToChild_, O_CLOEXEC)) {
        close(childToParent_[0]);
        close(childToParent_[1]);
        childToParent_[0] = -1;
        childToParent_[1] = -1;
        std::ostringstream sstr;
        sstr << "Failed to create child-to-parent pipes (errno=" << errno << "): " << strerror(errno);
        edm::Exception except(edm::errors::OtherCMS, sstr.str());
        throw except;
      }

      helperThread_ = std::make_unique<std::thread>(stacktraceHelperThread);
      helperThread_->detach();
    }

enableWarnings_()

void edm::service::InitRootHandlers::enableWarnings_ ( )

overrideprivatevirtual

Implements edm::RootHandlers.

Definition at line 942 of file InitRootHandlers.cc.

References edm::RootHandlers::kInfo.

{ s_ignoreWarnings = edm::RootHandlers::SeverityLevel::kInfo; }

fillDescriptions()

void edm::service::InitRootHandlers::fillDescriptions ( ConfigurationDescriptions &  descriptions )

static

Definition at line 908 of file InitRootHandlers.cc.

References edm::ConfigurationDescriptions::add(), and submitPVResolutionJobs::desc.

                                                                                   {
      ParameterSetDescription desc;
      desc.setComment("Centralized interface to ROOT.");
      desc.addUntracked<bool>("UnloadRootSigHandler", false)
          ->setComment("If True, signals are handled by this service, rather than by ROOT.");
      desc.addUntracked<bool>("ResetRootErrHandler", true)
          ->setComment(
              "If True, ROOT messages (e.g. errors, warnings) are handled by this service, rather than by ROOT.");
      desc.addUntracked<bool>("AutoLibraryLoader", true)
          ->setComment("If True, enables automatic loading of data dictionaries.");
      desc.addUntracked<bool>("AutoClassParser", true)
          ->setComment(
              "If False, the automatic parsing of class headers for dictionaries when pre-built dictionaries are "
              "missing is disable during module construction. The current implementation of disabling the parsing is "
              "fragile, and may work only in a single-thread job that does not use reco::parser::cutParser() or "
              "reco::parser::expressionParser() (and it certainly does not work on multiple threads).");
      desc.addUntracked<bool>("LoadAllDictionaries", false)->setComment("If True, loads all ROOT dictionaries.");
      desc.addUntracked<bool>("EnableIMT", true)->setComment("If True, calls ROOT::EnableImplicitMT().");
      desc.addUntracked<bool>("AbortOnSignal", true)
          ->setComment(
              "If True, do an abort when a signal occurs that causes a crash. If False, ROOT will do an exit which "
              "attempts to do a clean shutdown.");
      desc.addUntracked<bool>("InteractiveDebug", false)
          ->setComment(
              "If True, leave gdb attached to cmsRun after a crash; "
              "if False, attach gdb, print a stack trace, and quit gdb");
      desc.addUntracked<int>("DebugLevel", 0)->setComment("Sets ROOT's gDebug value.");
      desc.addUntracked<int>("StackTracePauseTime", 300)
          ->setComment("Seconds to pause other threads during stack trace.");
      descriptions.add("InitRootHandlers", desc);
    }

getPstackArgv()

char const *const * edm::service::InitRootHandlers::getPstackArgv ( )

staticprivate

Definition at line 940 of file InitRootHandlers.cc.

References pstackArgv_.

Referenced by edm::service::cmssw_stacktrace().

{ return pstackArgv_; }

ignoreWarnings_()

void edm::service::InitRootHandlers::ignoreWarnings_ ( edm::RootHandlers::SeverityLevel  level )

overrideprivatevirtual

Implements edm::RootHandlers.

Definition at line 944 of file InitRootHandlers.cc.

References personalPlayback::level.

{ s_ignoreWarnings = level; }

stacktraceFromThread()

void edm::service::InitRootHandlers::stacktraceFromThread ( )

static

Definition at line 690 of file InitRootHandlers.cc.

References visDQMUpload::buf, childToParent_, parentToChild_, and mps_fire::result.

                                                {
      int result = full_write(parentToChild_[1], "1");
      if (result < 0) {
        full_cerr_write("\n\nAttempt to request stacktrace failed: ");
        full_cerr_write(strerror(-result));
        full_cerr_write("\n");
        return;
      }
      char buf[2];
      buf[1] = '\0';
      if ((result = full_read(childToParent_[0], buf, 1, 5 * 60)) < 0) {
        full_cerr_write("\n\nWaiting for stacktrace completion failed: ");
        if (result == -ETIMEDOUT) {
          full_cerr_write("timed out waiting for GDB to complete.");
        } else {
          full_cerr_write(strerror(-result));
        }
        full_cerr_write("\n");
        return;
      }
    }

stacktraceHelperThread()

void edm::service::InitRootHandlers::stacktraceHelperThread ( )

staticprivate

Definition at line 647 of file InitRootHandlers.cc.

References visDQMUpload::buf, childToParent_, edm::service::cmssw_stacktrace_fork(), parentToChild_, and mps_fire::result.

Referenced by cachePidInfo().

                                                  {
      int toParent = childToParent_[1];
      int fromParent = parentToChild_[0];
      char buf[2];
      buf[1] = '\0';

      while (true) {
        int result = full_read(fromParent, buf, 1);
        if (result < 0) {
          // To avoid a deadlock (this function is NOT re-entrant), reset signals
          // We never set them back to the CMSSW handler because we assume the parent
          // thread will abort for us.
          set_default_signals();
          close(toParent);
          full_cerr_write("\n\nTraceback helper thread failed to read from parent: ");
          full_cerr_write(strerror(-result));
          full_cerr_write("\n");
          ::abort();
        }
        if (buf[0] == '1') {
          set_default_signals();
          cmssw_stacktrace_fork();
          full_write(toParent, buf);
        } else if (buf[0] == '2') {
          // We have just finished forking.  Reload the file descriptors for thread
          // communication.
          close(toParent);
          close(fromParent);
          toParent = childToParent_[1];
          fromParent = parentToChild_[0];
        } else if (buf[0] == '3') {
          break;
        } else {
          set_default_signals();
          close(toParent);
          full_cerr_write("\n\nTraceback helper thread got unknown command from parent: ");
          full_cerr_write(buf);
          full_cerr_write("\n");
          ::abort();
        }
      }
    }

stackTracePause()

static int edm::service::InitRootHandlers::stackTracePause ( )

inlinestatic

Definition at line 108 of file InitRootHandlers.cc.

References stackTracePause_.

{ return stackTracePause_; }

threadIDs()

static const ThreadTracker::Container_type& edm::service::InitRootHandlers::threadIDs ( )

inlinestatic

Definition at line 101 of file InitRootHandlers.cc.

References relativeConstraints::empty, and threadTracker_.

                                                            {
        static const ThreadTracker::Container_type empty;
        if (threadTracker_) {
          return threadTracker_->IDs();
        }
        return empty;
      }

willBeUsingThreads()

void edm::service::InitRootHandlers::willBeUsingThreads ( )

overrideprivatevirtual

Implements edm::RootHandlers.

Definition at line 897 of file InitRootHandlers.cc.

                                              {
      //Tell Root we want to be multi-threaded
      ROOT::EnableThreadSafety();

      //When threading, also have to keep ROOT from logging all TObjects into a list
      TObject::SetObjectStat(false);

      //Have to avoid having Streamers modify themselves after they have been used
      TVirtualStreamerInfo::Optimize(false);
    }

Friends And Related Function Documentation

cmssw_stacktrace

int cmssw_stacktrace ( void *  )

friend

Definition at line 743 of file InitRootHandlers.cc.

                                  {
      set_default_signals();

      char const* const* argv = edm::service::InitRootHandlers::getPstackArgv();
      // NOTE: this is NOT async-signal-safe at CERN's lxplus service.
      // CERN uses LD_PRELOAD to replace execv with a function from libsnoopy which
      // calls dlsym.
#ifdef __linux__
      syscall(SYS_execve, "/bin/sh", argv, __environ);
#else
      execv("/bin/sh", argv);
#endif
      ::abort();
      return 1;
    }

Member Data Documentation

autoClassParser_

bool edm::service::InitRootHandlers::autoClassParser_

private

Definition at line 135 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

autoLibraryLoader_

bool edm::service::InitRootHandlers::autoLibraryLoader_

private

Definition at line 134 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

childToParent_

int edm::service::InitRootHandlers::childToParent_ = {-1, -1}

staticprivate

Definition at line 126 of file InitRootHandlers.cc.

Referenced by cachePidInfo(), stacktraceFromThread(), and stacktraceHelperThread().

doneModules_

std::atomic< std::size_t > edm::service::InitRootHandlers::doneModules_

static

Definition at line 111 of file InitRootHandlers.cc.

helperThread_

std::unique_ptr< std::thread > edm::service::InitRootHandlers::helperThread_

staticprivate

Definition at line 127 of file InitRootHandlers.cc.

Referenced by cachePidInfo().

interactiveDebug_

bool edm::service::InitRootHandlers::interactiveDebug_

private

Definition at line 136 of file InitRootHandlers.cc.

Referenced by cachePidInfo().

loadAllDictionaries_

bool edm::service::InitRootHandlers::loadAllDictionaries_

private

Definition at line 133 of file InitRootHandlers.cc.

moduleListBuffers_

std::vector< std::array< char, moduleBufferSize > > edm::service::InitRootHandlers::moduleListBuffers_

static

Definition at line 110 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

nextModule_

std::atomic< std::size_t > edm::service::InitRootHandlers::nextModule_

static

Definition at line 111 of file InitRootHandlers.cc.

parentToChild_

int edm::service::InitRootHandlers::parentToChild_ = {-1, -1}

staticprivate

Definition at line 125 of file InitRootHandlers.cc.

Referenced by cachePidInfo(), stacktraceFromThread(), and stacktraceHelperThread().

pidString_

char edm::service::InitRootHandlers::pidString_ = {}

staticprivate

Definition at line 123 of file InitRootHandlers.cc.

Referenced by cachePidInfo().

pidStringLength_

constexpr int edm::service::InitRootHandlers::pidStringLength_ = 200

staticprivate

Definition at line 122 of file InitRootHandlers.cc.

Referenced by cachePidInfo().

pstackArgv_

char const *const edm::service::InitRootHandlers::pstackArgv_ = {pstackName, dashC, InitRootHandlers::pidString_, nullptr}

staticprivate

Definition at line 124 of file InitRootHandlers.cc.

Referenced by getPstackArgv().

resetErrHandler_

bool edm::service::InitRootHandlers::resetErrHandler_

private

Definition at line 132 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

sigAbrtHandler_

std::shared_ptr<const void> edm::service::InitRootHandlers::sigAbrtHandler_

private

Definition at line 141 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

sigBusHandler_

std::shared_ptr<const void> edm::service::InitRootHandlers::sigBusHandler_

private

Definition at line 137 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

sigFpeHandler_

std::shared_ptr<const void> edm::service::InitRootHandlers::sigFpeHandler_

private

Definition at line 142 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

sigIllHandler_

std::shared_ptr<const void> edm::service::InitRootHandlers::sigIllHandler_

private

Definition at line 139 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

sigSegvHandler_

std::shared_ptr<const void> edm::service::InitRootHandlers::sigSegvHandler_

private

Definition at line 138 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

sigTermHandler_

std::shared_ptr<const void> edm::service::InitRootHandlers::sigTermHandler_

private

Definition at line 140 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().

stackTracePause_

int edm::service::InitRootHandlers::stackTracePause_ = 300

staticprivate

Definition at line 129 of file InitRootHandlers.cc.

Referenced by InitRootHandlers(), and stackTracePause().

threadTracker_

std::unique_ptr< InitRootHandlers::ThreadTracker > edm::service::InitRootHandlers::threadTracker_

staticprivate

Definition at line 128 of file InitRootHandlers.cc.

Referenced by InitRootHandlers(), threadIDs(), and ~InitRootHandlers().

unloadSigHandler_

bool edm::service::InitRootHandlers::unloadSigHandler_

private

Definition at line 131 of file InitRootHandlers.cc.

Referenced by InitRootHandlers().