CMS 3D CMS Logo

List of all members | Classes | Public Member Functions | Static Public Member Functions | Private Member Functions | Private Attributes
edm::storage::StatisticsSenderService Class Reference

#include <StatisticsSenderService.h>

Classes

struct  FileInfo
 
class  FileStatistics
 

Public Member Functions

void closedFile (std::string const &lfn, bool usedFallback)
 
void openingFile (std::string const &lfn, edm::InputType type, size_t size=-1)
 
void setCurrentServer (const std::string &urlOrLfn, const std::string &servername)
 
void setSize (const std::string &urlOrLfn, size_t size)
 
 StatisticsSenderService (edm::ParameterSet const &pset, edm::ActivityRegistry &ar)
 

Static Public Member Functions

static const char * getJobID ()
 
static bool getX509Subject (std::string &)
 

Private Member Functions

void cleanupOldFiles ()
 
void determineHostnames ()
 
void filePostCloseEvent (std::string const &lfn, bool usedFallback)
 
void fillUDP (const std::string &site, const FileInfo &fileinfo, bool, std::string &) const
 
std::string const * matchedLfn (std::string const &iURL)
 

Private Attributes

std::string m_clientdomain
 
std::string m_clienthost
 
size_t m_counter
 
const bool m_debug
 
FileStatistics m_filestats
 
std::string m_guid
 
tbb::concurrent_unordered_map< std::string, FileInfom_lfnToFileInfo
 
std::mutex m_servermutex
 
tbb::concurrent_unordered_map< std::string, std::string > m_urlToLfn
 
std::string m_userdn
 

Detailed Description

Definition at line 19 of file StatisticsSenderService.h.

Constructor & Destructor Documentation

StatisticsSenderService::StatisticsSenderService ( edm::ParameterSet const &  pset,
edm::ActivityRegistry ar 
)

Definition at line 168 of file StatisticsSenderService.cc.

References determineHostnames(), filePostCloseEvent(), getX509Subject(), m_userdn, and edm::ActivityRegistry::watchPostCloseFile().

169  : m_clienthost("unknown"),
170  m_clientdomain("unknown"),
171  m_filestats(),
172  m_guid(Guid().toString()),
173  m_counter(0),
174  m_userdn("unknown"),
175  m_debug(iPSet.getUntrackedParameter<bool>("debug", false)) {
178  if (!getX509Subject(m_userdn)) {
179  m_userdn = "unknown";
180  }
181 }
void watchPostCloseFile(PostCloseFile::slot_type const &iSlot)
std::string toString(const std::pair< T, T > &aT)
Definition: CaloEllipse.h:72
void filePostCloseEvent(std::string const &lfn, bool usedFallback)

Member Function Documentation

void StatisticsSenderService::cleanupOldFiles ( )
private

Definition at line 312 of file StatisticsSenderService.cc.

References m_lfnToFileInfo, and m_urlToLfn.

Referenced by filePostCloseEvent().

312  {
313  //remove entries with openCount of 0
314  bool moreToTest = false;
315  do {
316  moreToTest = false;
317  for (auto it = m_lfnToFileInfo.begin(); it != m_lfnToFileInfo.end(); ++it) {
318  if (it->second.m_openCount == 0) {
319  auto lfn = it->first;
320  bool moreToTest2 = false;
321  do {
322  moreToTest2 = false;
323  for (auto it2 = m_urlToLfn.begin(); it2 != m_urlToLfn.end(); ++it2) {
324  if (it2->second == lfn) {
325  m_urlToLfn.unsafe_erase(it2);
326  moreToTest2 = true;
327  break;
328  }
329  }
330  } while (moreToTest2);
331 
332  m_lfnToFileInfo.unsafe_erase(it);
333  moreToTest = true;
334  break;
335  }
336  }
337  } while (moreToTest);
338 }
tbb::concurrent_unordered_map< std::string, std::string > m_urlToLfn
tbb::concurrent_unordered_map< std::string, FileInfo > m_lfnToFileInfo
void StatisticsSenderService::closedFile ( std::string const &  lfn,
bool  usedFallback 
)

Definition at line 259 of file StatisticsSenderService.cc.

References EnergyCorrector::c, fillUDP(), runEdmFileComparison::found, info(), edm::Service< T >::isAvailable(), m_debug, m_lfnToFileInfo, m_userdn, matchedLfn(), edm::SiteLocalConfig::siteName(), edm::SiteLocalConfig::statisticsDestination(), edm::SiteLocalConfig::statisticsInfo(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by edm::RootInputFileSequence::closeFile().

259  {
261  if (!pSLC.isAvailable()) {
262  return;
263  }
264 
265  const struct addrinfo *addresses = pSLC->statisticsDestination();
266  if (!addresses and !m_debug) {
267  return;
268  }
269 
270  std::set<std::string> const *info = pSLC->statisticsInfo();
271  if (info && !info->empty() && (m_userdn != "unknown") &&
272  ((info->find("dn") == info->end()) || (info->find("nodn") != info->end()))) {
273  m_userdn = "not reported";
274  }
275 
276  auto lfn = matchedLfn(url);
277  if (nullptr != lfn) {
278  auto found = m_lfnToFileInfo.find(*lfn);
279  assert(found != m_lfnToFileInfo.end());
280 
281  std::string results;
282  fillUDP(pSLC->siteName(), found->second, usedFallback, results);
283  if (m_debug) {
284  edm::LogSystem("StatisticSenderService") << "\n" << results << "\n";
285  }
286 
287  for (const struct addrinfo *address = addresses; address != nullptr; address = address->ai_next) {
288  int sock = socket(address->ai_family, address->ai_socktype, address->ai_protocol);
289  if (sock < 0) {
290  continue;
291  }
292  auto close_del = [](int *iSocket) { close(*iSocket); };
293  std::unique_ptr<int, decltype(close_del)> guard(&sock, close_del);
294  if (sendto(sock, results.c_str(), results.size(), 0, address->ai_addr, address->ai_addrlen) >= 0) {
295  break;
296  }
297  }
298 
299  auto c = --found->second.m_openCount;
300  if (m_debug) {
301  if (c == 0) {
302  edm::LogWarning("StatisticsSenderService") << "fully closed: " << *lfn << "\n";
303  } else {
304  edm::LogWarning("StatisticsSenderService") << "partially closed: " << *lfn << "\n";
305  }
306  }
307  } else if (m_debug) {
308  edm::LogWarning("StatisticsSenderService") << "closed: unknown url name " << url << "\n";
309  }
310 }
virtual struct addrinfo const * statisticsDestination() const =0
static const TGPicture * info(bool iBackgroundIsBlack)
void fillUDP(const std::string &site, const FileInfo &fileinfo, bool, std::string &) const
virtual std::set< std::string > const * statisticsInfo() const =0
virtual std::string const & siteName(void) const =0
std::string const * matchedLfn(std::string const &iURL)
bool isAvailable() const
Definition: Service.h:40
tbb::concurrent_unordered_map< std::string, FileInfo > m_lfnToFileInfo
void StatisticsSenderService::determineHostnames ( void  )
private

Definition at line 358 of file StatisticsSenderService.cc.

References HOST_NAME_MAX, m_clientdomain, and m_clienthost.

Referenced by StatisticsSenderService().

358  {
359  char tmpName[HOST_NAME_MAX];
360  if (gethostname(tmpName, HOST_NAME_MAX) != 0) {
361  // Sigh, no way to log errors from here.
362  m_clienthost = "unknown";
363  } else {
364  m_clienthost = tmpName;
365  }
366  size_t dot_pos = m_clienthost.find(".");
367  if (dot_pos == std::string::npos) {
368  m_clientdomain = "unknown";
369  } else {
370  m_clientdomain = m_clienthost.substr(dot_pos + 1, m_clienthost.size() - dot_pos - 1);
371  m_clienthost = m_clienthost.substr(0, dot_pos);
372  }
373 }
#define HOST_NAME_MAX
void StatisticsSenderService::filePostCloseEvent ( std::string const &  lfn,
bool  usedFallback 
)
private
void StatisticsSenderService::fillUDP ( const std::string &  site,
const FileInfo fileinfo,
bool  usedFallback,
std::string &  udpinfo 
) const
private

Definition at line 375 of file StatisticsSenderService.cc.

References edm::storage::StatisticsSenderService::FileStatistics::fillUDP(), getJobID(), m_clientdomain, m_clienthost, edm::storage::StatisticsSenderService::FileInfo::m_filelfn, m_filestats, m_guid, edm::storage::StatisticsSenderService::FileInfo::m_id, edm::storage::StatisticsSenderService::FileInfo::m_serverdomain, edm::storage::StatisticsSenderService::FileInfo::m_serverhost, edm::storage::StatisticsSenderService::FileInfo::m_size, edm::storage::StatisticsSenderService::FileInfo::m_type, m_userdn, edm::Primary, edm::SecondaryFile, and edm::SecondarySource.

Referenced by closedFile().

378  {
379  std::ostringstream os;
380 
381  // Header - same for all IO accesses
382  os << "{";
383  if (!siteName.empty()) {
384  os << "\"site_name\":\"" << siteName << "\", ";
385  }
386  if (usedFallback) {
387  os << "\"fallback\": true, ";
388  } else {
389  os << "\"fallback\": false, ";
390  }
391  os << "\"type\": ";
392  switch (fileinfo.m_type) {
394  os << "\"primary\", ";
395  break;
396  }
398  os << "\"secondary\", ";
399  break;
400  }
402  os << "\"embedded\", ";
403  break;
404  }
405  }
406  auto serverhost = fileinfo.m_serverhost;
407  auto serverdomain = fileinfo.m_serverdomain;
408 
409  os << "\"user_dn\":\"" << m_userdn << "\", ";
410  os << "\"client_host\":\"" << m_clienthost << "\", ";
411  os << "\"client_domain\":\"" << m_clientdomain << "\", ";
412  os << "\"server_host\":\"" << serverhost << "\", ";
413  os << "\"server_domain\":\"" << serverdomain << "\", ";
414  os << "\"unique_id\":\"" << m_guid << "-" << fileinfo.m_id << "\", ";
415  os << "\"file_lfn\":\"" << fileinfo.m_filelfn << "\", ";
416  // Dashboard devs requested that we send out no app_info if a job ID
417  // is not present in the environment.
418  const char *jobId = getJobID();
419  if (jobId) {
420  os << "\"app_info\":\"" << jobId << "\", ";
421  }
422 
423  if (fileinfo.m_size >= 0) {
424  os << "\"file_size\":" << fileinfo.m_size << ", ";
425  }
426 
427  m_filestats.fillUDP(os);
428 
429  os << "}";
430  udpinfo = os.str();
431 }
const char * StatisticsSenderService::getJobID ( )
static

Definition at line 183 of file StatisticsSenderService.cc.

References JOB_UNIQUE_ID_ENV, and JOB_UNIQUE_ID_ENV_V2.

Referenced by fillUDP(), and SendMonitoringInfo().

183  {
184  const char *id = getenv(JOB_UNIQUE_ID_ENV);
185  // Dashboard developers requested that we migrate to this environment variable.
186  return id ? id : getenv(JOB_UNIQUE_ID_ENV_V2);
187 }
static char const *const JOB_UNIQUE_ID_ENV_V2
static char const *const JOB_UNIQUE_ID_ENV
bool StatisticsSenderService::getX509Subject ( std::string &  result)
static

Definition at line 537 of file StatisticsSenderService.cc.

References corrVsCorr::filename, getX509SubjectFromFile(), and mps_fire::result.

Referenced by StatisticsSenderService().

537  {
538  char *filename = getenv("X509_USER_PROXY");
539  if (filename && getX509SubjectFromFile(filename, result)) {
540  return true;
541  }
542  std::stringstream ss;
543  ss << "/tmp/x509up_u" << geteuid();
544  return getX509SubjectFromFile(ss.str(), result);
545 }
static bool getX509SubjectFromFile(const std::string &filename, std::string &result)
std::string const * StatisticsSenderService::matchedLfn ( std::string const &  iURL)
private

Definition at line 189 of file StatisticsSenderService.cc.

References runEdmFileComparison::found, m_lfnToFileInfo, m_urlToLfn, and findQualityFiles::v.

Referenced by closedFile(), setCurrentServer(), and setSize().

189  {
190  auto found = m_urlToLfn.find(iURL);
191  if (found != m_urlToLfn.end()) {
192  return &found->second;
193  }
194  for (auto const &v : m_lfnToFileInfo) {
195  if (v.first.size() < iURL.size()) {
196  if (v.first == iURL.substr(iURL.size() - v.first.size())) {
197  m_urlToLfn.emplace(iURL, v.first);
198  return &m_urlToLfn.find(iURL)->second;
199  }
200  }
201  }
202  //does the lfn have a protocol and the iURL not?
203  if (std::string::npos == iURL.find(':')) {
204  for (auto const &v : m_lfnToFileInfo) {
205  if ((std::string::npos != v.first.find(':')) and (v.first.size() > iURL.size())) {
206  if (iURL == v.first.substr(v.first.size() - iURL.size())) {
207  m_urlToLfn.emplace(iURL, v.first);
208  return &m_urlToLfn.find(iURL)->second;
209  }
210  }
211  }
212  }
213 
214  return nullptr;
215 }
tbb::concurrent_unordered_map< std::string, std::string > m_urlToLfn
tbb::concurrent_unordered_map< std::string, FileInfo > m_lfnToFileInfo
void StatisticsSenderService::openingFile ( std::string const &  lfn,
edm::InputType  type,
size_t  size = -1 
)

Definition at line 246 of file StatisticsSenderService.cc.

References m_counter, m_lfnToFileInfo, m_urlToLfn, and findQualityFiles::size.

Referenced by edm::RootInputFileSequence::initTheFile().

246  {
247  m_urlToLfn.emplace(lfn, lfn);
248  auto attempt = m_lfnToFileInfo.emplace(lfn, FileInfo{lfn, type});
249  if (attempt.second) {
250  attempt.first->second.m_size = size;
251  attempt.first->second.m_id = m_counter++;
252  edm::LogInfo("StatisticsSenderService") << "openingFile: opening " << lfn << "\n";
253  } else {
254  ++(attempt.first->second.m_openCount);
255  edm::LogInfo("StatisticsSenderService") << "openingFile: re-opening" << lfn << "\n";
256  }
257 }
size
Write out results.
type
Definition: HCALResponse.h:21
tbb::concurrent_unordered_map< std::string, std::string > m_urlToLfn
tbb::concurrent_unordered_map< std::string, FileInfo > m_lfnToFileInfo
void StatisticsSenderService::setCurrentServer ( const std::string &  urlOrLfn,
const std::string &  servername 
)

Definition at line 217 of file StatisticsSenderService.cc.

References runEdmFileComparison::found, m_debug, m_lfnToFileInfo, m_servermutex, matchedLfn(), eostools::move(), and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by XrdAdaptor::RequestManager::updateCurrentServer().

217  {
218  size_t dot_pos = servername.find('.');
219  std::string serverhost;
220  std::string serverdomain;
221  if (dot_pos == std::string::npos) {
222  serverhost = servername.substr(0, servername.find(":"));
223  serverdomain = "unknown";
224  } else {
225  serverhost = servername.substr(0, dot_pos);
226  serverdomain = servername.substr(dot_pos + 1, servername.find(":") - dot_pos - 1);
227  if (serverdomain.empty()) {
228  serverdomain = "unknown";
229  }
230  }
231  {
232  auto lfn = matchedLfn(url);
233  std::lock_guard<std::mutex> sentry(m_servermutex);
234  if (nullptr != lfn) {
235  auto found = m_lfnToFileInfo.find(*lfn);
236  if (found != m_lfnToFileInfo.end()) {
237  found->second.m_serverhost = std::move(serverhost);
238  found->second.m_serverdomain = std::move(serverdomain);
239  }
240  } else if (m_debug) {
241  edm::LogWarning("StatisticsSenderService") << "setCurrentServer: unknown url name " << url << "\n";
242  }
243  }
244 }
std::string const * matchedLfn(std::string const &iURL)
tbb::concurrent_unordered_map< std::string, FileInfo > m_lfnToFileInfo
def move(src, dest)
Definition: eostools.py:511
void StatisticsSenderService::setSize ( const std::string &  urlOrLfn,
size_t  size 
)

Definition at line 340 of file StatisticsSenderService.cc.

References m_debug, m_lfnToFileInfo, matchedLfn(), and findQualityFiles::size.

Referenced by TStorageFactoryFile::Initialize().

340  {
341  auto lfn = matchedLfn(url);
342  if (nullptr != lfn) {
343  auto itFound = m_lfnToFileInfo.find(*lfn);
344  if (itFound != m_lfnToFileInfo.end()) {
345  itFound->second.m_size = size;
346  }
347  } else if (m_debug) {
348  edm::LogWarning("StatisticsSenderService") << "setSize: unknown url name " << url << "\n";
349  }
350 }
size
Write out results.
std::string const * matchedLfn(std::string const &iURL)
tbb::concurrent_unordered_map< std::string, FileInfo > m_lfnToFileInfo

Member Data Documentation

std::string edm::storage::StatisticsSenderService::m_clientdomain
private

Definition at line 78 of file StatisticsSenderService.h.

Referenced by determineHostnames(), and fillUDP().

std::string edm::storage::StatisticsSenderService::m_clienthost
private

Definition at line 77 of file StatisticsSenderService.h.

Referenced by determineHostnames(), and fillUDP().

size_t edm::storage::StatisticsSenderService::m_counter
private

Definition at line 83 of file StatisticsSenderService.h.

Referenced by openingFile().

const bool edm::storage::StatisticsSenderService::m_debug
private

Definition at line 86 of file StatisticsSenderService.h.

Referenced by closedFile(), setCurrentServer(), and setSize().

FileStatistics edm::storage::StatisticsSenderService::m_filestats
private

Definition at line 81 of file StatisticsSenderService.h.

Referenced by filePostCloseEvent(), and fillUDP().

std::string edm::storage::StatisticsSenderService::m_guid
private

Definition at line 82 of file StatisticsSenderService.h.

Referenced by fillUDP().

tbb::concurrent_unordered_map<std::string, FileInfo> edm::storage::StatisticsSenderService::m_lfnToFileInfo
private
std::mutex edm::storage::StatisticsSenderService::m_servermutex
private

Definition at line 85 of file StatisticsSenderService.h.

Referenced by setCurrentServer().

tbb::concurrent_unordered_map<std::string, std::string> edm::storage::StatisticsSenderService::m_urlToLfn
private

Definition at line 80 of file StatisticsSenderService.h.

Referenced by cleanupOldFiles(), matchedLfn(), and openingFile().

std::string edm::storage::StatisticsSenderService::m_userdn
private

Definition at line 84 of file StatisticsSenderService.h.

Referenced by closedFile(), fillUDP(), and StatisticsSenderService().