#include <ResourceMonitorCollection.h>

Inheritance diagram for stor::ResourceMonitorCollection:

Classes
struct	DiskUsage

struct	DiskUsageStats

struct	Stats

Public Types
typedef boost::shared_ptr < DiskUsageStats >	DiskUsageStatsPtr

typedef std::vector < DiskUsageStatsPtr >	DiskUsageStatsPtrList

Public Types inherited from stor::MonitorCollection
typedef std::vector< std::pair < std::string, xdata::Serializable * > >	InfoSpaceItems

Public Member Functions
void	configureAlarms (AlarmParams const &)

void	configureDisks (DiskWritingParams const &)

void	configureResources (ResourceMonitorParams const &)

void	getStats (Stats &) const

	ResourceMonitorCollection (const utils::Duration_t &updateInterval, AlarmHandlerPtr)

Public Member Functions inherited from stor::MonitorCollection
void	appendInfoSpaceItems (InfoSpaceItems &)

void	calculateStatistics (const utils::TimePoint_t &now)

	MonitorCollection (const utils::Duration_t &updateInterval)

void	reset (const utils::TimePoint_t &now)

void	updateInfoSpaceItems ()

virtual	~MonitorCollection ()

Private Types
typedef boost::shared_ptr < DiskUsage >	DiskUsagePtr

typedef std::vector< DiskUsagePtr >	DiskUsagePtrList

typedef std::set< std::string >	SATABeasts

Private Member Functions
void	addDisk (const std::string &)

void	addOtherDisks ()

void	calcDiskUsage ()

void	calcNumberOfCopyWorkers ()

void	calcNumberOfInjectWorkers ()

void	checkNumberOfCopyWorkers ()

void	checkNumberOfInjectWorkers ()

void	checkSataBeast (const std::string &sataBeast)

void	checkSataBeasts ()

bool	checkSataDisks (const std::string &sataBeast, const std::string &hostSuffix)

virtual void	do_appendInfoSpaceItems (InfoSpaceItems &)

virtual void	do_calculateStatistics ()

virtual void	do_reset ()

virtual void	do_updateInfoSpaceItems ()

void	doStatFs (DiskUsagePtr)

void	emitDiskAlarm (DiskUsagePtr)

void	emitDiskSpaceAlarm (DiskUsagePtr)

void	getDiskStats (Stats &) const

int	getProcessCount (const std::string &processName, const int &uid=-1)

bool	getSataBeasts (SATABeasts &sataBeasts)

bool	isImportantDisk (const std::string &)

ResourceMonitorCollection &	operator= (ResourceMonitorCollection const &)

	ResourceMonitorCollection (ResourceMonitorCollection const &)

void	retrieveDiskSize (DiskUsagePtr)

void	revokeDiskAlarm (DiskUsagePtr)

void	updateSataBeastStatus (const std::string &sataBeast, const std::string &content)

Private Attributes
AlarmHandlerPtr	alarmHandler_

AlarmParams	alarmParams_

xdata::UnsignedInteger32	copyWorkers_

xdata::Vector< xdata::String >	diskPaths_

DiskUsagePtrList	diskUsageList_

boost::mutex	diskUsageListMutex_

DiskWritingParams	dwParams_

xdata::UnsignedInteger32	injectWorkers_

int	latchedSataBeastStatus_

unsigned int	nLogicalDisks_

int	numberOfCopyWorkers_

xdata::UnsignedInteger32	numberOfDisks_

int	numberOfInjectWorkers_

ResourceMonitorParams	rmParams_

xdata::Integer32	sataBeastStatus_

xdata::Vector < xdata::UnsignedInteger32 >	totalDiskSpace_

const utils::Duration_t	updateInterval_

xdata::Vector < xdata::UnsignedInteger32 >	usedDiskSpace_

Friends
class	testResourceMonitorCollection

Additional Inherited Members

Detailed Description

A collection of MonitoredQuantities related to resource usages

Author:: mommsen

Revision:: 1.30

Date:: 2011/11/10 10:56:37

Definition at line 44 of file ResourceMonitorCollection.h.

Member Typedef Documentation

typedef boost::shared_ptr<DiskUsage> stor::ResourceMonitorCollection::DiskUsagePtr

private

Definition at line 121 of file ResourceMonitorCollection.h.

typedef std::vector<DiskUsagePtr> stor::ResourceMonitorCollection::DiskUsagePtrList

private

Definition at line 122 of file ResourceMonitorCollection.h.

typedef boost::shared_ptr<DiskUsageStats> stor::ResourceMonitorCollection::DiskUsageStatsPtr

Definition at line 59 of file ResourceMonitorCollection.h.

typedef std::vector<DiskUsageStatsPtr> stor::ResourceMonitorCollection::DiskUsageStatsPtrList

Definition at line 60 of file ResourceMonitorCollection.h.

typedef std::set<std::string> stor::ResourceMonitorCollection::SATABeasts

private

Definition at line 156 of file ResourceMonitorCollection.h.

Constructor & Destructor Documentation

stor::ResourceMonitorCollection::ResourceMonitorCollection	(	const utils::Duration_t &	updateInterval,
		AlarmHandlerPtr	ah
	)

Constructor.

Definition at line 30 of file ResourceMonitorCollection.cc.

     :
   MonitorCollection(updateInterval),
   updateInterval_(updateInterval),
   alarmHandler_(ah),
   numberOfCopyWorkers_(-1),
   numberOfInjectWorkers_(-1),
   nLogicalDisks_(0),
   latchedSataBeastStatus_(-1)
   {}

stor::ResourceMonitorCollection::ResourceMonitorCollection ( ResourceMonitorCollection const & )

private

Member Function Documentation

void stor::ResourceMonitorCollection::addDisk ( const std::string & pathname )

private

Definition at line 72 of file ResourceMonitorCollection.cc.

References diskUsageList_, and retrieveDiskSize().

Referenced by addOtherDisks(), and configureDisks().

   {
     if ( pathname.empty() ) return;
     
     DiskUsagePtr diskUsage( new DiskUsage(pathname) );
     retrieveDiskSize(diskUsage);
     diskUsageList_.push_back(diskUsage);
   }

void stor::ResourceMonitorCollection::addOtherDisks ( )

private

Definition at line 82 of file ResourceMonitorCollection.cc.

References addDisk(), dwParams_, and stor::DiskWritingParams::otherDiskPaths_.

Referenced by configureDisks().

   {
     for ( DiskWritingParams::OtherDiskPaths::const_iterator
             it = dwParams_.otherDiskPaths_.begin(),
             itEnd =  dwParams_.otherDiskPaths_.end();
           it != itEnd;
           ++it)
     {
       addDisk(*it);
     }
   }

void stor::ResourceMonitorCollection::calcDiskUsage ( )

private

Definition at line 242 of file ResourceMonitorCollection.cc.

References diskUsageList_, diskUsageListMutex_, and retrieveDiskSize().

Referenced by do_calculateStatistics(), and do_updateInfoSpaceItems().

   {
     boost::mutex::scoped_lock sl(diskUsageListMutex_);
     
     for ( DiskUsagePtrList::iterator it = diskUsageList_.begin(),
             itEnd = diskUsageList_.end();
           it != itEnd;
           ++it)
     {
       retrieveDiskSize(*it);
     }
   }

void stor::ResourceMonitorCollection::calcNumberOfCopyWorkers ( )

private

Definition at line 370 of file ResourceMonitorCollection.cc.

References alarmParams_, checkNumberOfCopyWorkers(), stor::ResourceMonitorParams::WorkerParams::command_, stor::ResourceMonitorParams::copyWorkers_, stor::ResourceMonitorParams::WorkerParams::expectedCount_, getProcessCount(), stor::AlarmParams::isProductionSystem_, numberOfCopyWorkers_, rmParams_, and stor::ResourceMonitorParams::WorkerParams::user_.

Referenced by do_calculateStatistics().

   {
     struct passwd* passwd = getpwnam(rmParams_.copyWorkers_.user_.c_str());
     if (passwd)
     {
       numberOfCopyWorkers_ =
         getProcessCount(rmParams_.copyWorkers_.command_, passwd->pw_uid);
     }
     else
     {
       numberOfCopyWorkers_ = 0;
     }
     
     if ( alarmParams_.isProductionSystem_ && rmParams_.copyWorkers_.expectedCount_ >= 0 )
     {
       checkNumberOfCopyWorkers();
     }
   }

void stor::ResourceMonitorCollection::calcNumberOfInjectWorkers ( )

private

Definition at line 410 of file ResourceMonitorCollection.cc.

References alarmParams_, checkNumberOfInjectWorkers(), stor::ResourceMonitorParams::WorkerParams::command_, stor::ResourceMonitorParams::WorkerParams::expectedCount_, getProcessCount(), stor::ResourceMonitorParams::injectWorkers_, stor::AlarmParams::isProductionSystem_, numberOfInjectWorkers_, rmParams_, and stor::ResourceMonitorParams::WorkerParams::user_.

Referenced by do_calculateStatistics().

   {
     struct passwd* passwd = getpwnam(rmParams_.injectWorkers_.user_.c_str());
     if (passwd)
     {
       numberOfInjectWorkers_ = getProcessCount(rmParams_.injectWorkers_.command_, passwd->pw_uid);
     }
     else
     {
       numberOfInjectWorkers_ = 0;
     }
     
     if (
       alarmParams_.isProductionSystem_ &&
       rmParams_.injectWorkers_.expectedCount_ >= 0
     )
     {
       checkNumberOfInjectWorkers();
     }
   }

void stor::ResourceMonitorCollection::checkNumberOfCopyWorkers ( )

private

Definition at line 390 of file ResourceMonitorCollection.cc.

References alarmHandler_, stor::ResourceMonitorParams::copyWorkers_, stor::ResourceMonitorParams::WorkerParams::expectedCount_, lumiQueryAPI::msg, numberOfCopyWorkers_, rmParams_, AlCaHLTBitMon_QueryRunRegistry::string, and stor::AlarmHandler::WARNING.

Referenced by calcNumberOfCopyWorkers().

   {
     const std::string alarmName = "CopyWorkers";
     
     if ( numberOfCopyWorkers_ < rmParams_.copyWorkers_.expectedCount_ )
     {
       std::ostringstream msg;
       msg << "Expected " << rmParams_.copyWorkers_.expectedCount_ <<
         " running CopyWorkers, but found " <<
         numberOfCopyWorkers_ << ".";
       XCEPT_DECLARE(stor::exception::CopyWorkers, ex, msg.str());
       alarmHandler_->raiseAlarm(alarmName, AlarmHandler::WARNING, ex);
     }
     else
     {
       alarmHandler_->revokeAlarm(alarmName);
     }
   }

void stor::ResourceMonitorCollection::checkNumberOfInjectWorkers ( )

private

Definition at line 432 of file ResourceMonitorCollection.cc.

References alarmHandler_, stor::ResourceMonitorParams::WorkerParams::expectedCount_, stor::ResourceMonitorParams::injectWorkers_, lumiQueryAPI::msg, numberOfInjectWorkers_, rmParams_, AlCaHLTBitMon_QueryRunRegistry::string, and stor::AlarmHandler::WARNING.

Referenced by calcNumberOfInjectWorkers().

   {
     const std::string alarmName = "InjectWorkers";
     
     if ( numberOfInjectWorkers_ != rmParams_.injectWorkers_.expectedCount_ )
     {
       std::ostringstream msg;
       msg << "Expected " << rmParams_.injectWorkers_.expectedCount_ <<
         " running InjectWorkers, but found " <<
         numberOfInjectWorkers_ << ".";
       XCEPT_DECLARE(stor::exception::InjectWorkers, ex, msg.str());
       alarmHandler_->raiseAlarm(alarmName, AlarmHandler::WARNING, ex);
     }
     else
     {
       alarmHandler_->revokeAlarm(alarmName);
     }
   }

void stor::ResourceMonitorCollection::checkSataBeast ( const std::string & sataBeast )

private

Definition at line 503 of file ResourceMonitorCollection.cc.

References alarmHandler_, checkSataDisks(), stor::AlarmHandler::ERROR, and latchedSataBeastStatus_.

Referenced by checkSataBeasts().

   {
     if ( ! (checkSataDisks(sataBeast,"-00.cms") || checkSataDisks(sataBeast,"-10.cms")) )
     {
       XCEPT_DECLARE(stor::exception::SataBeast, ex, 
         "Failed to connect to SATA beast " + sataBeast);
       alarmHandler_->raiseAlarm(sataBeast, AlarmHandler::ERROR, ex);
       
       latchedSataBeastStatus_ = 99999;
     }
   }

void stor::ResourceMonitorCollection::checkSataBeasts ( )

private

Definition at line 452 of file ResourceMonitorCollection.cc.

References checkSataBeast(), getSataBeasts(), and latchedSataBeastStatus_.

Referenced by do_calculateStatistics().

   {
     SATABeasts sataBeasts;
     if ( getSataBeasts(sataBeasts) )
     {
       for (
         SATABeasts::const_iterator it = sataBeasts.begin(),
           itEnd= sataBeasts.end();
         it != itEnd;
         ++it
       )
       {
         checkSataBeast(*it);
       }
     }
     else
     {
       latchedSataBeastStatus_ = -1;
     }
   }

bool stor::ResourceMonitorCollection::checkSataDisks	(	const std::string &	sataBeast,
		const std::string &	hostSuffix
	)

private

Definition at line 517 of file ResourceMonitorCollection.cc.

References ws_sso_content_reader::content, stor::CurlInterface::getInterface(), lumiQueryAPI::msg, runEdmFileComparison::returnCode, AlCaHLTBitMon_QueryRunRegistry::string, and stor::AlarmHandler::WARNING.

Referenced by checkSataBeast().

   {
     CurlInterfacePtr curlInterface = CurlInterface::getInterface();
     CurlInterface::Content content;
     
     // Do not try to connect if we have no user name
     if ( rmParams_.sataUser_.empty() ) return true;
     
     const CURLcode returnCode =
       curlInterface->getContent(
         "http://" + sataBeast + hostSuffix + "/status.asp",rmParams_.sataUser_,
         content
       );
     
     if (returnCode == CURLE_OK)
     {
       updateSataBeastStatus(sataBeast, std::string(&content[0]));
       return true;
     }
     else
     {
       std::ostringstream msg;
       msg << "Failed to connect to SATA controller "
         << sataBeast << hostSuffix 
         << ": " << std::string(&content[0]);
       XCEPT_DECLARE(stor::exception::SataBeast, ex, msg.str());
       alarmHandler_->notifySentinel(AlarmHandler::WARNING, ex);
       
       return false;
     }
   }

void stor::ResourceMonitorCollection::configureAlarms ( AlarmParams const & alarmParams )

Configures the alarms

Definition at line 105 of file ResourceMonitorCollection.cc.

Referenced by stor::Ready::do_entryActionWork().

   {
     alarmParams_ = alarmParams;
   }

void stor::ResourceMonitorCollection::configureDisks ( DiskWritingParams const & dwParams )

Configures the disks used to write events

Definition at line 44 of file ResourceMonitorCollection.cc.

References addDisk(), addOtherDisks(), alarmParams_, stor::DiskWritingParams::dbFilePath_, diskUsageList_, diskUsageListMutex_, dwParams_, stor::DiskWritingParams::filePath_, i, stor::AlarmParams::isProductionSystem_, max(), stor::DiskWritingParams::nLogicalDisk_, nLogicalDisks_, stor::DiskWritingParams::otherDiskPaths_, and EgammaValidation_cff::pathName.

Referenced by stor::Ready::do_entryActionWork().

   {
     boost::mutex::scoped_lock sl(diskUsageListMutex_);
     
     dwParams_ = dwParams;
     
     nLogicalDisks_ = std::max(dwParams.nLogicalDisk_, 1);
     diskUsageList_.clear();
     diskUsageList_.reserve(nLogicalDisks_+dwParams.otherDiskPaths_.size()+1);
     
     for (unsigned int i=0; i<nLogicalDisks_; ++i) {
       
       std::ostringstream pathName;
       pathName << dwParams.filePath_;
       if( dwParams.nLogicalDisk_ > 0 ) {
         pathName << "/" << std::setfill('0') << std::setw(2) << i; 
       }
       addDisk(pathName.str());
     }
     addDisk(dwParams.dbFilePath_);
     
     if ( alarmParams_.isProductionSystem_ )
     {
       addOtherDisks();
     }
   }

void stor::ResourceMonitorCollection::configureResources ( ResourceMonitorParams const & rmParams )

Configures the resources to be monitored

Definition at line 96 of file ResourceMonitorCollection.cc.

Referenced by stor::Ready::do_entryActionWork().

   {
     rmParams_ = rmParams;
   }

void stor::ResourceMonitorCollection::do_appendInfoSpaceItems ( InfoSpaceItems & infoSpaceItems )

privatevirtual

Reimplemented from stor::MonitorCollection.

Definition at line 179 of file ResourceMonitorCollection.cc.

References copyWorkers_, diskPaths_, injectWorkers_, numberOfDisks_, sataBeastStatus_, totalDiskSpace_, and usedDiskSpace_.

   {
     infoSpaceItems.push_back(std::make_pair("copyWorkers", &copyWorkers_));
     infoSpaceItems.push_back(std::make_pair("injectWorkers", &injectWorkers_));
     infoSpaceItems.push_back(std::make_pair("sataBeastStatus", &sataBeastStatus_));
     infoSpaceItems.push_back(std::make_pair("numberOfDisks", &numberOfDisks_));
     infoSpaceItems.push_back(std::make_pair("diskPaths", &diskPaths_));
     infoSpaceItems.push_back(std::make_pair("totalDiskSpace", &totalDiskSpace_));
     infoSpaceItems.push_back(std::make_pair("usedDiskSpace", &usedDiskSpace_));
   }

void stor::ResourceMonitorCollection::do_calculateStatistics ( )

privatevirtual

Implements stor::MonitorCollection.

Definition at line 146 of file ResourceMonitorCollection.cc.

References calcDiskUsage(), calcNumberOfCopyWorkers(), calcNumberOfInjectWorkers(), and checkSataBeasts().

   {
     calcDiskUsage();
     calcNumberOfCopyWorkers();
     calcNumberOfInjectWorkers();
     checkSataBeasts();
   }

void stor::ResourceMonitorCollection::do_reset ( )

privatevirtual

Implements stor::MonitorCollection.

Definition at line 155 of file ResourceMonitorCollection.cc.

References diskUsageList_, diskUsageListMutex_, latchedSataBeastStatus_, numberOfCopyWorkers_, numberOfInjectWorkers_, and stor::AlarmHandler::OKAY.

   {
     numberOfCopyWorkers_ = -1;
     numberOfInjectWorkers_ = -1;
     latchedSataBeastStatus_ = -1;
     
     boost::mutex::scoped_lock sl(diskUsageListMutex_);
     for ( DiskUsagePtrList::const_iterator it = diskUsageList_.begin(),
             itEnd = diskUsageList_.end();
           it != itEnd;
           ++it)
     {
       if ( ! (*it)->retrievingDiskSize_ )
       {
         (*it)->diskSize_ = -1;
         (*it)->absDiskUsage_ = -1;
         (*it)->relDiskUsage_ = -1;
         (*it)->retVal_ = 0;
         (*it)->alarmState_ = AlarmHandler::OKAY;
       }
     }
   }

void stor::ResourceMonitorCollection::do_updateInfoSpaceItems ( )

privatevirtual

Reimplemented from stor::MonitorCollection.

Definition at line 191 of file ResourceMonitorCollection.cc.

References calcDiskUsage(), copyWorkers_, diskPaths_, stor::ResourceMonitorCollection::Stats::diskUsageStatsList, getStats(), injectWorkers_, nLogicalDisks_, stor::ResourceMonitorCollection::Stats::numberOfCopyWorkers, numberOfDisks_, stor::ResourceMonitorCollection::Stats::numberOfInjectWorkers, stor::ResourceMonitorCollection::Stats::sataBeastStatus, sataBeastStatus_, totalDiskSpace_, and usedDiskSpace_.

   {
     Stats stats;
     getStats(stats);
     
     if (stats.numberOfCopyWorkers > 0)
       copyWorkers_ = static_cast<xdata::UnsignedInteger32>(stats.numberOfCopyWorkers);
     else
       copyWorkers_ = 0;
 
     if (stats.numberOfInjectWorkers > 0)
       injectWorkers_ = static_cast<xdata::UnsignedInteger32>(stats.numberOfInjectWorkers);
     else
       injectWorkers_ = 0;
     
     sataBeastStatus_ = stats.sataBeastStatus;
     numberOfDisks_ = nLogicalDisks_;
     
     diskPaths_.clear();
     totalDiskSpace_.clear();
     usedDiskSpace_.clear();
     
     diskPaths_.reserve(stats.diskUsageStatsList.size());
     totalDiskSpace_.reserve(stats.diskUsageStatsList.size());
     usedDiskSpace_.reserve(stats.diskUsageStatsList.size());
     
     for (DiskUsageStatsPtrList::const_iterator
            it = stats.diskUsageStatsList.begin(),
            itEnd = stats.diskUsageStatsList.end();
          it != itEnd;
          ++it)
     {
       diskPaths_.push_back(
         static_cast<xdata::String>( (*it)->pathName )
       );
       totalDiskSpace_.push_back(
         static_cast<xdata::UnsignedInteger32>(
           static_cast<unsigned int>( (*it)->diskSize * 1024 )
         )
       );
       usedDiskSpace_.push_back(
         static_cast<xdata::UnsignedInteger32>( 
           static_cast<unsigned int>( (*it)->absDiskUsage * 1024 )
         )
       );
     }
     
     calcDiskUsage();
   }

void stor::ResourceMonitorCollection::doStatFs ( DiskUsagePtr diskUsage )

private

Definition at line 294 of file ResourceMonitorCollection.cc.

References stor::utils::sleep().

Referenced by retrieveDiskSize().

   {
     diskUsage->retrievingDiskSize_ = true;
 
     #if __APPLE__
     diskUsage->retVal_ = statfs(diskUsage->pathName_.c_str(), &(diskUsage->statfs_));
     #else
     diskUsage->retVal_ = statfs64(diskUsage->pathName_.c_str(), &(diskUsage->statfs_));
     #endif
     if (diskUsage->pathName_ == "/aSlowDiskForUnitTests") ::sleep(5);
 
     diskUsage->retrievingDiskSize_ = false;
   }

void stor::ResourceMonitorCollection::emitDiskAlarm ( DiskUsagePtr diskUsage )

private

Definition at line 309 of file ResourceMonitorCollection.cc.

References alarmHandler_, stor::AlarmHandler::ERROR, stor::AlarmHandler::FATAL, isImportantDisk(), lumiQueryAPI::msg, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by retrieveDiskSize().

   {
     const std::string msg = "Cannot access " + diskUsage->pathName_ + ". Is it mounted?";
 
     diskUsage->diskSize_ = -1;
     diskUsage->absDiskUsage_ = -1;
     diskUsage->relDiskUsage_ = -1;
 
     if ( isImportantDisk(diskUsage->pathName_) )
     {
       diskUsage->alarmState_ = AlarmHandler::FATAL;
       XCEPT_DECLARE(stor::exception::DiskSpaceAlarm, ex, msg);
       alarmHandler_->moveToFailedState(ex);
     }
     else
     {    
       diskUsage->alarmState_ = AlarmHandler::ERROR;
       XCEPT_DECLARE(stor::exception::DiskSpaceAlarm, ex, msg);
       alarmHandler_->raiseAlarm(diskUsage->pathName_, diskUsage->alarmState_, ex);
     }
   }

void stor::ResourceMonitorCollection::emitDiskSpaceAlarm ( DiskUsagePtr diskUsage )

private

Definition at line 332 of file ResourceMonitorCollection.cc.

References alarmHandler_, dwParams_, stor::DiskWritingParams::failHighWaterMark_, stor::AlarmHandler::FATAL, isImportantDisk(), and stor::AlarmHandler::WARNING.

Referenced by retrieveDiskSize().

   {
     if (
       isImportantDisk(diskUsage->pathName_) &&
       (diskUsage->relDiskUsage_ > dwParams_.failHighWaterMark_)
     )
     {
       diskUsage->alarmState_ = AlarmHandler::FATAL;
       XCEPT_DECLARE(stor::exception::DiskSpaceAlarm, ex, diskUsage->toString());
       alarmHandler_->moveToFailedState(ex);
     }
     else
     {    
       diskUsage->alarmState_ = AlarmHandler::WARNING;
       XCEPT_DECLARE(stor::exception::DiskSpaceAlarm, ex, diskUsage->toString());
       alarmHandler_->raiseAlarm(diskUsage->pathName_, diskUsage->alarmState_, ex);
     }
   }

void stor::ResourceMonitorCollection::getDiskStats ( Stats & stats ) const

private

Definition at line 124 of file ResourceMonitorCollection.cc.

References diskUsageList_, diskUsageListMutex_, and stor::ResourceMonitorCollection::Stats::diskUsageStatsList.

Referenced by getStats().

   {
     boost::mutex::scoped_lock sl(diskUsageListMutex_);
     
     stats.diskUsageStatsList.clear();
     stats.diskUsageStatsList.reserve(diskUsageList_.size());
     for ( DiskUsagePtrList::const_iterator it = diskUsageList_.begin(),
             itEnd = diskUsageList_.end();
           it != itEnd;
           ++it)
     {
       DiskUsageStatsPtr diskUsageStats(new DiskUsageStats);
       diskUsageStats->diskSize = (*it)->diskSize_;
       diskUsageStats->absDiskUsage = (*it)->absDiskUsage_;
       diskUsageStats->relDiskUsage = (*it)->relDiskUsage_;
       diskUsageStats->pathName = (*it)->pathName_;
       diskUsageStats->alarmState = (*it)->alarmState_;
       stats.diskUsageStatsList.push_back(diskUsageStats);
     }
   }

int stor::ResourceMonitorCollection::getProcessCount	(	const std::string &	processName,
		const int &	uid = `-1`
	)

private

Definition at line 665 of file ResourceMonitorCollection.cc.

References prof2calltree::count, alcazmumu_cfi::filter, and n.

Referenced by calcNumberOfCopyWorkers(), and calcNumberOfInjectWorkers().

   {
     int count(0);
     struct dirent **namelist;
     int n;
     
     #if __APPLE__
     return -1;
     #else 
     n = scandir("/proc", &namelist, filter, 0);
     #endif
     if (n < 0) return -1;
     
     while(n--)
     {
       std::ostringstream cmdline;
       cmdline << "/proc/" << namelist[n]->d_name << "/cmdline";
       
       if ( grep(cmdline.str(), processName) &&
         (uid < 0 || matchUid(cmdline.str(), uid)) &&
         isMaster(namelist[n]->d_name) )
       {
         ++count;
       }
       free(namelist[n]);
     }
     free(namelist);
     
     return count;
   }

bool stor::ResourceMonitorCollection::getSataBeasts ( SATABeasts & sataBeasts )

private

Definition at line 474 of file ResourceMonitorCollection.cc.

References alarmParams_, query::host, recoMuon::in, stor::AlarmParams::isProductionSystem_, geometryCSVtoXML::line, pos, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by checkSataBeasts().

   {
     if (! alarmParams_.isProductionSystem_) return false;
     
     std::ifstream in;
     in.open( "/proc/mounts" );
     
     if ( ! in.is_open() ) return false;
     
     std::string line;
     while( getline(in,line) )
     {
       size_t pos = line.find("sata");
       if ( pos != std::string::npos )
       {
         std::ostringstream host;
         host << "satab-c2c"
           << std::setw(2) << std::setfill('0')
           << line.substr(pos+4,1)
           << "-"
           << std::setw(2) << std::setfill('0')
           << line.substr(pos+5,1);
         sataBeasts.insert(host.str());
       }
     }
     return !sataBeasts.empty();
   }

void stor::ResourceMonitorCollection::getStats ( Stats & stats ) const

Write all our collected statistics into the given Stats struct.

Definition at line 113 of file ResourceMonitorCollection.cc.

References getDiskStats(), latchedSataBeastStatus_, stor::ResourceMonitorCollection::Stats::numberOfCopyWorkers, numberOfCopyWorkers_, stor::ResourceMonitorCollection::Stats::numberOfInjectWorkers, numberOfInjectWorkers_, and stor::ResourceMonitorCollection::Stats::sataBeastStatus.

Referenced by stor::SMWebPageHelper::addDOMforResourceUsage(), and do_updateInfoSpaceItems().

   {
     getDiskStats(stats);
     
     stats.numberOfCopyWorkers = numberOfCopyWorkers_;
     stats.numberOfInjectWorkers = numberOfInjectWorkers_;
     
     stats.sataBeastStatus = latchedSataBeastStatus_;
   }

bool stor::ResourceMonitorCollection::isImportantDisk ( const std::string & pathName )

private

Definition at line 352 of file ResourceMonitorCollection.cc.

References begin, dwParams_, end, spr::find(), and stor::DiskWritingParams::otherDiskPaths_.

Referenced by emitDiskAlarm(), and emitDiskSpaceAlarm().

   {
     DiskWritingParams::OtherDiskPaths::const_iterator begin =
       dwParams_.otherDiskPaths_.begin();
     DiskWritingParams::OtherDiskPaths::const_iterator end =
       dwParams_.otherDiskPaths_.end();
     return ( std::find(begin, end, pathName) == end );
   }

ResourceMonitorCollection& stor::ResourceMonitorCollection::operator= ( ResourceMonitorCollection const & )

private

void stor::ResourceMonitorCollection::retrieveDiskSize ( DiskUsagePtr diskUsage )

private

Definition at line 256 of file ResourceMonitorCollection.cc.

References doStatFs(), dwParams_, emitDiskAlarm(), emitDiskSpaceAlarm(), stor::DiskWritingParams::highWaterMark_, and revokeDiskAlarm().

Referenced by addDisk(), and calcDiskUsage().

   {
     if ( ! diskUsage->retrievingDiskSize_ )
       // don't start another thread if there's already one
     {
       boost::thread thread(
         boost::bind( &ResourceMonitorCollection::doStatFs, this, diskUsage)
       );
       if (
         ( ! thread.timed_join( boost::posix_time::milliseconds(500) ) )
         || (diskUsage->retVal_ != 0)
       )
       {
         emitDiskAlarm(diskUsage);
       }
       else
       {
         const unsigned int blksize = diskUsage->statfs_.f_bsize;
         diskUsage->diskSize_ =
           static_cast<double>(diskUsage->statfs_.f_blocks * blksize) / 1024 / 1024 / 1024;
         diskUsage->absDiskUsage_ =
           diskUsage->diskSize_ -
           static_cast<double>(diskUsage->statfs_.f_bavail * blksize) / 1024 / 1024 / 1024;
         diskUsage->relDiskUsage_ = (100 * (diskUsage->absDiskUsage_ / diskUsage->diskSize_));
         if ( diskUsage->relDiskUsage_ > dwParams_.highWaterMark_ )
         {
           emitDiskSpaceAlarm(diskUsage);
         }
         else if ( diskUsage->relDiskUsage_ < dwParams_.highWaterMark_*0.95 )
           // do not change alarm level if we are close to the high water mark
         {
           revokeDiskAlarm(diskUsage);
         }
       }
     }
   }

void stor::ResourceMonitorCollection::revokeDiskAlarm ( DiskUsagePtr diskUsage )

private

Definition at line 362 of file ResourceMonitorCollection.cc.

References alarmHandler_, and stor::AlarmHandler::OKAY.

Referenced by retrieveDiskSize().

   {
     diskUsage->alarmState_ = AlarmHandler::OKAY;
     
     alarmHandler_->revokeAlarm(diskUsage->pathName_);
   }

void stor::ResourceMonitorCollection::updateSataBeastStatus	(	const std::string &	sataBeast,
		const std::string &	content
	)

private

Definition at line 554 of file ResourceMonitorCollection.cc.

References end, stor::AlarmHandler::ERROR, flags, dqm_diff::start, and AlCaHLTBitMon_QueryRunRegistry::string.

   {
     boost::regex failedEntry(">([^<]* has failed[^<]*)");
     boost::regex failedDisk("Hard disk([[:digit:]]+)");
     boost::regex failedController("RAID controller ([[:digit:]]+)");
     boost::match_results<std::string::const_iterator> matchedEntry, matchedCause;
     boost::match_flag_type flags = boost::match_default;
     
     std::string::const_iterator start = content.begin();
     std::string::const_iterator end = content.end();
     
     unsigned int newSataBeastStatus = 0;
     
     while( regex_search(start, end, matchedEntry, failedEntry, flags) )
     {
       std::string errorMsg = matchedEntry[1];
       XCEPT_DECLARE(stor::exception::SataBeast, ex, sataBeast+": "+errorMsg);
       alarmHandler_->raiseAlarm(sataBeast, AlarmHandler::ERROR, ex);
       
       // find what failed
       if ( regex_search(errorMsg, matchedCause, failedDisk) )
       {
         // Update the number of failed disks
         ++newSataBeastStatus;
       }
       else if ( regex_search(errorMsg, matchedCause, failedController) )
       {
         // Update the number of failed controllers
         newSataBeastStatus += 100;
       }
       else
       {
         // Unknown failure
         newSataBeastStatus += 1000;
       }
       
       // update search position:
       start = matchedEntry[0].second;
       // update flags:
       flags |= boost::match_prev_avail;
       flags |= boost::match_not_bob;
     }
     
     latchedSataBeastStatus_ = newSataBeastStatus;
     
     if (latchedSataBeastStatus_ == 0) // no more problems
       alarmHandler_->revokeAlarm(sataBeast);
     
   }