#include <StatisticsSenderService.h>

Classes
struct	FileInfo

class	FileStatistics

Public Member Functions
void	closedFile (std::string const &lfn, bool usedFallback)

void	openingFile (std::string const &lfn, edm::InputType type, size_t size=-1)

void	setCurrentServer (const std::string &urlOrLfn, const std::string &servername)

void	setSize (const std::string &urlOrLfn, size_t size)

	StatisticsSenderService (edm::ParameterSet const &pset, edm::ActivityRegistry &ar)

Static Public Member Functions
static const char *	getJobID ()

static bool	getX509Subject (std::string &)

Private Member Functions
void	cleanupOldFiles ()

void	determineHostnames ()

void	filePostCloseEvent (std::string const &lfn, bool usedFallback)

void	fillUDP (const std::string &site, const FileInfo &fileinfo, bool, std::string &) const

std::string const *	matchedLfn (std::string const &iURL)

Private Attributes
std::string	m_clientdomain

std::string	m_clienthost

size_t	m_counter

const bool	m_debug

FileStatistics	m_filestats

std::string	m_guid

tbb::concurrent_unordered_map< std::string, FileInfo >	m_lfnToFileInfo

std::mutex	m_servermutex

tbb::concurrent_unordered_map< std::string, std::string >	m_urlToLfn

std::string	m_userdn

Detailed Description

Definition at line 19 of file StatisticsSenderService.h.

Constructor & Destructor Documentation

StatisticsSenderService::StatisticsSenderService	(	edm::ParameterSet const &	pset,
		edm::ActivityRegistry &	ar
	)

Definition at line 168 of file StatisticsSenderService.cc.

References determineHostnames(), filePostCloseEvent(), getX509Subject(), m_userdn, and edm::ActivityRegistry::watchPostCloseFile().

     : m_clienthost("unknown"),
       m_clientdomain("unknown"),
       m_filestats(),
       m_guid(Guid().toString()),
       m_counter(0),
       m_userdn("unknown"),
       m_debug(iPSet.getUntrackedParameter<bool>("debug", false)) {
   determineHostnames();
   ar.watchPostCloseFile(this, &StatisticsSenderService::filePostCloseEvent);
   if (!getX509Subject(m_userdn)) {
     m_userdn = "unknown";
   }
 }

Member Function Documentation

void StatisticsSenderService::cleanupOldFiles ( )

private

Definition at line 312 of file StatisticsSenderService.cc.

References m_lfnToFileInfo, and m_urlToLfn.

Referenced by filePostCloseEvent().

                                               {
   //remove entries with openCount of 0
   bool moreToTest = false;
   do {
     moreToTest = false;
     for (auto it = m_lfnToFileInfo.begin(); it != m_lfnToFileInfo.end(); ++it) {
       if (it->second.m_openCount == 0) {
         auto lfn = it->first;
         bool moreToTest2 = false;
         do {
           moreToTest2 = false;
           for (auto it2 = m_urlToLfn.begin(); it2 != m_urlToLfn.end(); ++it2) {
             if (it2->second == lfn) {
               m_urlToLfn.unsafe_erase(it2);
               moreToTest2 = true;
               break;
             }
           }
         } while (moreToTest2);
 
         m_lfnToFileInfo.unsafe_erase(it);
         moreToTest = true;
         break;
       }
     }
   } while (moreToTest);
 }

void StatisticsSenderService::closedFile	(	std::string const &	lfn,
		bool	usedFallback
	)

Definition at line 259 of file StatisticsSenderService.cc.

References EnergyCorrector::c, fillUDP(), runEdmFileComparison::found, info(), edm::Service< T >::isAvailable(), m_debug, m_lfnToFileInfo, m_userdn, matchedLfn(), edm::SiteLocalConfig::siteName(), edm::SiteLocalConfig::statisticsDestination(), edm::SiteLocalConfig::statisticsInfo(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by edm::RootInputFileSequence::closeFile().

                                                                                 {
   edm::Service<edm::SiteLocalConfig> pSLC;
   if (!pSLC.isAvailable()) {
     return;
   }
 
   const struct addrinfo *addresses = pSLC->statisticsDestination();
   if (!addresses and !m_debug) {
     return;
   }
 
   std::set<std::string> const *info = pSLC->statisticsInfo();
   if (info && !info->empty() && (m_userdn != "unknown") &&
       ((info->find("dn") == info->end()) || (info->find("nodn") != info->end()))) {
     m_userdn = "not reported";
   }
 
   auto lfn = matchedLfn(url);
   if (nullptr != lfn) {
     auto found = m_lfnToFileInfo.find(*lfn);
     assert(found != m_lfnToFileInfo.end());
 
     std::string results;
     fillUDP(pSLC->siteName(), found->second, usedFallback, results);
     if (m_debug) {
       edm::LogSystem("StatisticSenderService") << "\n" << results << "\n";
     }
 
     for (const struct addrinfo *address = addresses; address != nullptr; address = address->ai_next) {
       int sock = socket(address->ai_family, address->ai_socktype, address->ai_protocol);
       if (sock < 0) {
         continue;
       }
       auto close_del = [](int *iSocket) { close(*iSocket); };
       std::unique_ptr<int, decltype(close_del)> guard(&sock, close_del);
       if (sendto(sock, results.c_str(), results.size(), 0, address->ai_addr, address->ai_addrlen) >= 0) {
         break;
       }
     }
 
     auto c = --found->second.m_openCount;
     if (m_debug) {
       if (c == 0) {
         edm::LogWarning("StatisticsSenderService") << "fully closed: " << *lfn << "\n";
       } else {
         edm::LogWarning("StatisticsSenderService") << "partially closed: " << *lfn << "\n";
       }
     }
   } else if (m_debug) {
     edm::LogWarning("StatisticsSenderService") << "closed: unknown url name " << url << "\n";
   }
 }

void StatisticsSenderService::determineHostnames ( void )

private

Definition at line 358 of file StatisticsSenderService.cc.

References HOST_NAME_MAX, m_clientdomain, and m_clienthost.

Referenced by StatisticsSenderService().

                                                      {
   char tmpName[HOST_NAME_MAX];
   if (gethostname(tmpName, HOST_NAME_MAX) != 0) {
     // Sigh, no way to log errors from here.
     m_clienthost = "unknown";
   } else {
     m_clienthost = tmpName;
   }
   size_t dot_pos = m_clienthost.find(".");
   if (dot_pos == std::string::npos) {
     m_clientdomain = "unknown";
   } else {
     m_clientdomain = m_clienthost.substr(dot_pos + 1, m_clienthost.size() - dot_pos - 1);
     m_clienthost = m_clienthost.substr(0, dot_pos);
   }
 }

void StatisticsSenderService::filePostCloseEvent	(	std::string const &	lfn,
		bool	usedFallback
	)

private

Definition at line 352 of file StatisticsSenderService.cc.

References cleanupOldFiles(), m_filestats, and edm::storage::StatisticsSenderService::FileStatistics::update().

Referenced by StatisticsSenderService().

                                                                                         {
   //we are at a sync point in the framwework so no new files are being opened
   cleanupOldFiles();
   m_filestats.update();
 }

void StatisticsSenderService::fillUDP	(	const std::string &	site,
		const FileInfo &	fileinfo,
		bool	usedFallback,
		std::string &	udpinfo
	)		const

private

Definition at line 375 of file StatisticsSenderService.cc.

References edm::storage::StatisticsSenderService::FileStatistics::fillUDP(), getJobID(), m_clientdomain, m_clienthost, edm::storage::StatisticsSenderService::FileInfo::m_filelfn, m_filestats, m_guid, edm::storage::StatisticsSenderService::FileInfo::m_id, edm::storage::StatisticsSenderService::FileInfo::m_serverdomain, edm::storage::StatisticsSenderService::FileInfo::m_serverhost, edm::storage::StatisticsSenderService::FileInfo::m_size, edm::storage::StatisticsSenderService::FileInfo::m_type, m_userdn, edm::Primary, edm::SecondaryFile, and edm::SecondarySource.

Referenced by closedFile().

                                                                 {
   std::ostringstream os;
 
   // Header - same for all IO accesses
   os << "{";
   if (!siteName.empty()) {
     os << "\"site_name\":\"" << siteName << "\", ";
   }
   if (usedFallback) {
     os << "\"fallback\": true, ";
   } else {
     os << "\"fallback\": false, ";
   }
   os << "\"type\": ";
   switch (fileinfo.m_type) {
     case edm::InputType::Primary: {
       os << "\"primary\", ";
       break;
     }
     case edm::InputType::SecondaryFile: {
       os << "\"secondary\", ";
       break;
     }
     case edm::InputType::SecondarySource: {
       os << "\"embedded\", ";
       break;
     }
   }
   auto serverhost = fileinfo.m_serverhost;
   auto serverdomain = fileinfo.m_serverdomain;
 
   os << "\"user_dn\":\"" << m_userdn << "\", ";
   os << "\"client_host\":\"" << m_clienthost << "\", ";
   os << "\"client_domain\":\"" << m_clientdomain << "\", ";
   os << "\"server_host\":\"" << serverhost << "\", ";
   os << "\"server_domain\":\"" << serverdomain << "\", ";
   os << "\"unique_id\":\"" << m_guid << "-" << fileinfo.m_id << "\", ";
   os << "\"file_lfn\":\"" << fileinfo.m_filelfn << "\", ";
   // Dashboard devs requested that we send out no app_info if a job ID
   // is not present in the environment.
   const char *jobId = getJobID();
   if (jobId) {
     os << "\"app_info\":\"" << jobId << "\", ";
   }
 
   if (fileinfo.m_size >= 0) {
     os << "\"file_size\":" << fileinfo.m_size << ", ";
   }
 
   m_filestats.fillUDP(os);
 
   os << "}";
   udpinfo = os.str();
 }

const char * StatisticsSenderService::getJobID ( )

static

Definition at line 183 of file StatisticsSenderService.cc.

References JOB_UNIQUE_ID_ENV, and JOB_UNIQUE_ID_ENV_V2.

Referenced by fillUDP(), and SendMonitoringInfo().

                                               {
   const char *id = getenv(JOB_UNIQUE_ID_ENV);
   // Dashboard developers requested that we migrate to this environment variable.
   return id ? id : getenv(JOB_UNIQUE_ID_ENV_V2);
 }

bool StatisticsSenderService::getX509Subject ( std::string & result )

static

Definition at line 537 of file StatisticsSenderService.cc.

References corrVsCorr::filename, getX509SubjectFromFile(), and mps_fire::result.

Referenced by StatisticsSenderService().

                                                               {
   char *filename = getenv("X509_USER_PROXY");
   if (filename && getX509SubjectFromFile(filename, result)) {
     return true;
   }
   std::stringstream ss;
   ss << "/tmp/x509up_u" << geteuid();
   return getX509SubjectFromFile(ss.str(), result);
 }

std::string const * StatisticsSenderService::matchedLfn ( std::string const & iURL )

private

Definition at line 189 of file StatisticsSenderService.cc.

References runEdmFileComparison::found, m_lfnToFileInfo, m_urlToLfn, and findQualityFiles::v.

Referenced by closedFile(), setCurrentServer(), and setSize().

                                                                           {
   auto found = m_urlToLfn.find(iURL);
   if (found != m_urlToLfn.end()) {
     return &found->second;
   }
   for (auto const &v : m_lfnToFileInfo) {
     if (v.first.size() < iURL.size()) {
       if (v.first == iURL.substr(iURL.size() - v.first.size())) {
         m_urlToLfn.emplace(iURL, v.first);
         return &m_urlToLfn.find(iURL)->second;
       }
     }
   }
   //does the lfn have a protocol and the iURL not?
   if (std::string::npos == iURL.find(':')) {
     for (auto const &v : m_lfnToFileInfo) {
       if ((std::string::npos != v.first.find(':')) and (v.first.size() > iURL.size())) {
         if (iURL == v.first.substr(v.first.size() - iURL.size())) {
           m_urlToLfn.emplace(iURL, v.first);
           return &m_urlToLfn.find(iURL)->second;
         }
       }
     }
   }
 
   return nullptr;
 }

void StatisticsSenderService::openingFile	(	std::string const &	lfn,
		edm::InputType	type,
		size_t	size = `-1`
	)

Definition at line 246 of file StatisticsSenderService.cc.

References m_counter, m_lfnToFileInfo, m_urlToLfn, and findQualityFiles::size.

Referenced by edm::RootInputFileSequence::initTheFile().

                                                                                               {
   m_urlToLfn.emplace(lfn, lfn);
   auto attempt = m_lfnToFileInfo.emplace(lfn, FileInfo{lfn, type});
   if (attempt.second) {
     attempt.first->second.m_size = size;
     attempt.first->second.m_id = m_counter++;
     edm::LogInfo("StatisticsSenderService") << "openingFile: opening " << lfn << "\n";
   } else {
     ++(attempt.first->second.m_openCount);
     edm::LogInfo("StatisticsSenderService") << "openingFile: re-opening" << lfn << "\n";
   }
 }

void StatisticsSenderService::setCurrentServer	(	const std::string &	urlOrLfn,
		const std::string &	servername
	)

Definition at line 217 of file StatisticsSenderService.cc.

References runEdmFileComparison::found, m_debug, m_lfnToFileInfo, m_servermutex, matchedLfn(), eostools::move(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by XrdAdaptor::RequestManager::updateCurrentServer().

                                                                                                 {
   size_t dot_pos = servername.find('.');
   std::string serverhost;
   std::string serverdomain;
   if (dot_pos == std::string::npos) {
     serverhost = servername.substr(0, servername.find(":"));
     serverdomain = "unknown";
   } else {
     serverhost = servername.substr(0, dot_pos);
     serverdomain = servername.substr(dot_pos + 1, servername.find(":") - dot_pos - 1);
     if (serverdomain.empty()) {
       serverdomain = "unknown";
     }
   }
   {
     auto lfn = matchedLfn(url);
     std::lock_guard<std::mutex> sentry(m_servermutex);
     if (nullptr != lfn) {
       auto found = m_lfnToFileInfo.find(*lfn);
       if (found != m_lfnToFileInfo.end()) {
         found->second.m_serverhost = std::move(serverhost);
         found->second.m_serverdomain = std::move(serverdomain);
       }
     } else if (m_debug) {
       edm::LogWarning("StatisticsSenderService") << "setCurrentServer: unknown url name " << url << "\n";
     }
   }
 }

void StatisticsSenderService::setSize	(	const std::string &	urlOrLfn,
		size_t	size
	)

Definition at line 340 of file StatisticsSenderService.cc.

References m_debug, m_lfnToFileInfo, matchedLfn(), and findQualityFiles::size.

Referenced by TStorageFactoryFile::Initialize().

                                                                        {
   auto lfn = matchedLfn(url);
   if (nullptr != lfn) {
     auto itFound = m_lfnToFileInfo.find(*lfn);
     if (itFound != m_lfnToFileInfo.end()) {
       itFound->second.m_size = size;
     }
   } else if (m_debug) {
     edm::LogWarning("StatisticsSenderService") << "setSize: unknown url name " << url << "\n";
   }
 }