CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
SiteLocalConfigService.cc
Go to the documentation of this file.
1 //<<<<<< INCLUDES >>>>>>
2 
8 #include "tinyxml2.h"
9 #include <sstream>
10 #include <memory>
11 #include <boost/algorithm/string.hpp>
12 
13 //<<<<<< PRIVATE DEFINES >>>>>>
14 //<<<<<< PRIVATE CONSTANTS >>>>>>
15 //<<<<<< PRIVATE TYPES >>>>>>
16 //<<<<<< PRIVATE VARIABLE DEFINITIONS >>>>>>
17 //<<<<<< PUBLIC VARIABLE DEFINITIONS >>>>>>
18 //<<<<<< CLASS STRUCTURE INITIALIZATION >>>>>>
19 //<<<<<< PRIVATE FUNCTION DEFINITIONS >>>>>>
20 //<<<<<< PUBLIC FUNCTION DEFINITIONS >>>>>>
21 //<<<<<< MEMBER FUNCTION DEFINITIONS >>>>>>
22 
23 namespace {
24 
25  // concatenate all the XML node attribute/value pairs into a
26  // paren-separated string (for use by CORAL and frontier_client)
27  inline std::string _toParenString(tinyxml2::XMLElement const &nodeToConvert) {
28  std::ostringstream oss;
29 
30  for (auto child = nodeToConvert.FirstChildElement(); child != nullptr; child = child->NextSiblingElement()) {
31  for (auto attribute = child->FirstAttribute(); attribute != nullptr; attribute = attribute->Next()) {
32  oss << "(" << child->Name() << attribute->Name() << "=" << attribute->Value() << ")";
33  }
34  }
35  return oss.str();
36  }
37 
38  template <typename T>
39  static void overrideFromPSet(char const *iName, edm::ParameterSet const &iPSet, T &iHolder, T const *&iPointer) {
40  if (iPSet.exists(iName)) {
41  iHolder = iPSet.getUntrackedParameter<T>(iName);
42  iPointer = &iHolder;
43  }
44  }
45 
46  constexpr char const *const kEmptyString = "";
47  const char *safe(const char *iCheck) {
48  if (iCheck == nullptr) {
49  return kEmptyString;
50  }
51  return iCheck;
52  }
53 
54  std::string defaultURL() {
55  std::string returnValue;
56  const char *tmp = std::getenv("CMS_PATH");
57  if (tmp) {
58  returnValue = tmp;
59  }
60  returnValue += "/SITECONF/local/JobConfig/site-local-config.xml";
61  return returnValue;
62  }
63 
64 } // namespace
65 
66 namespace edm {
67  namespace service {
68 
70 
72  : m_url(pset.getUntrackedParameter<std::string>("siteLocalConfigFileUrl", defaultURL())),
73  m_dataCatalog(),
74  m_fallbackDataCatalog(),
75  m_frontierConnect(),
76  m_rfioType("castor"),
77  m_connected(false),
78  m_cacheTempDir(),
79  m_cacheTempDirPtr(nullptr),
80  m_cacheMinFree(),
81  m_cacheMinFreePtr(nullptr),
82  m_cacheHint(),
83  m_cacheHintPtr(nullptr),
84  m_cloneCacheHint(),
85  m_cloneCacheHintPtr(nullptr),
86  m_readHint(),
87  m_readHintPtr(nullptr),
88  m_ttreeCacheSize(0U),
89  m_ttreeCacheSizePtr(nullptr),
90  m_timeout(0U),
91  m_timeoutPtr(nullptr),
92  m_debugLevel(0U),
93  m_enablePrefetching(false),
94  m_enablePrefetchingPtr(nullptr),
95  m_nativeProtocols(),
96  m_nativeProtocolsPtr(nullptr),
97  m_statisticsDestination(),
98  m_statisticsAddrInfo(nullptr),
99  m_statisticsInfoAvail(false),
100  m_siteName() {
101  this->parse(m_url);
102 
103  //apply overrides
104  overrideFromPSet("overrideSourceCacheTempDir", pset, m_cacheTempDir, m_cacheTempDirPtr);
105  overrideFromPSet("overrideSourceCacheMinFree", pset, m_cacheMinFree, m_cacheMinFreePtr);
106  overrideFromPSet("overrideSourceCacheHintDir", pset, m_cacheHint, m_cacheHintPtr);
107  overrideFromPSet("overrideSourceCloneCacheHintDir", pset, m_cloneCacheHint, m_cloneCacheHintPtr);
108  overrideFromPSet("overrideSourceReadHint", pset, m_readHint, m_readHintPtr);
109  overrideFromPSet("overrideSourceNativeProtocols", pset, m_nativeProtocols, m_nativeProtocolsPtr);
110  overrideFromPSet("overrideSourceTTreeCacheSize", pset, m_ttreeCacheSize, m_ttreeCacheSizePtr);
111  overrideFromPSet("overrideSourceTimeout", pset, m_timeout, m_timeoutPtr);
112  overrideFromPSet("overridePrefetching", pset, m_enablePrefetching, m_enablePrefetchingPtr);
113  const std::string *tmpStringPtr = nullptr;
114  overrideFromPSet("overrideStatisticsDestination", pset, m_statisticsDestination, tmpStringPtr);
116  std::vector<std::string> tmpStatisticsInfo;
117  std::vector<std::string> const *tmpStatisticsInfoPtr = nullptr;
118  overrideFromPSet("overrideStatisticsInfo", pset, tmpStatisticsInfo, tmpStatisticsInfoPtr);
119  if (tmpStatisticsInfoPtr) {
120  m_statisticsInfoAvail = true;
121  m_statisticsInfo.clear();
122  for (auto &entry : tmpStatisticsInfo) {
124  }
125  }
126 
127  if (pset.exists("debugLevel")) {
128  m_debugLevel = pset.getUntrackedParameter<unsigned int>("debugLevel");
129  }
130  }
131 
133  if (m_statisticsAddrInfo) {
134  freeaddrinfo(m_statisticsAddrInfo);
135  m_statisticsAddrInfo = nullptr;
136  }
137  }
138 
140  if (!m_connected) {
141  //throw cms::Exception("Incomplete configuration")
142  // << "Valid site-local-config not found at " << m_url;
143  // Return PoolFileCatalog.xml for now
144  return "file:PoolFileCatalog.xml";
145  }
146 
147  if (m_dataCatalog.empty()) {
148  throw cms::Exception("Incomplete configuration") << "Did not find catalog in event-data section in " << m_url;
149  }
150 
151  return m_dataCatalog;
152  }
153 
155  if (!m_connected) {
156  //throw cms::Exception("Incomplete configuration")
157  // << "Valid site-local-config not found at " << m_url;
158  // Return PoolFileCatalog.xml for now
159  return "file:PoolFileCatalog.xml";
160  }
161 
162  // Note: Unlike the dataCatalog, the fallbackDataCatalog may be empty!
163  return m_fallbackDataCatalog;
164  }
165 
167  if (!m_connected) {
168  throw cms::Exception("Incomplete configuration") << "Valid site-local-config not found at " << m_url;
169  }
170 
171  if (m_frontierConnect.empty()) {
172  throw cms::Exception("Incomplete configuration")
173  << "Did not find frontier-connect in calib-data section in " << m_url;
174  }
175 
176  if (servlet.empty()) {
177  return m_frontierConnect;
178  }
179 
180  // Replace the last component of every "serverurl=" piece (up to the
181  // next close-paren) with the servlet
182  std::string::size_type nextparen = 0;
183  std::string::size_type serverurl, lastslash;
184  std::string complexstr = "";
185  while ((serverurl = m_frontierConnect.find("(serverurl=", nextparen)) != std::string::npos) {
186  complexstr.append(m_frontierConnect, nextparen, serverurl - nextparen);
187  nextparen = m_frontierConnect.find(')', serverurl);
188  lastslash = m_frontierConnect.rfind('/', nextparen);
189  complexstr.append(m_frontierConnect, serverurl, lastslash - serverurl + 1);
190  complexstr.append(servlet);
191  }
192  complexstr.append(m_frontierConnect, nextparen, m_frontierConnect.length() - nextparen);
193 
194  return complexstr;
195  }
196 
198  static std::string const proto = "frontier://";
199 
200  if (input.substr(0, proto.length()) == proto) {
201  // Replace the part after the frontier:// and before either an open-
202  // parentheses (which indicates user-supplied options) or the last
203  // slash (which indicates start of the schema) with the complex
204  // parenthesized string returned from frontierConnect() (which
205  // contains all the information needed to connect to frontier),
206  // if that part is a simple servlet name (non-empty and not
207  // containing special characters)
208  // Example connect strings where servlet is replaced:
209  // frontier://cms_conditions_data/CMS_COND_ECAL
210  // frontier://FrontierInt/CMS_COND_ECAL
211  // frontier://FrontierInt(retrieve-ziplevel=0)/CMS_COND_ECAL
212  // Example connect strings left untouched:
213  // frontier://cmsfrontier.cern.ch:8000/FrontierInt/CMS_COND_ECAL
214  // frontier://(serverurl=cmsfrontier.cern.ch:8000/FrontierInt)/CMS_COND_ECAL
215  std::string::size_type startservlet = proto.length();
216  // if user supplied extra parenthesized options, stop servlet there
217  std::string::size_type endservlet = input.find("(", startservlet);
218  if (endservlet == std::string::npos) {
219  endservlet = input.rfind('/', input.length());
220  }
221  std::string servlet = input.substr(startservlet, endservlet - startservlet);
222  if ((!servlet.empty()) && (servlet.find_first_of(":/)[]") == std::string::npos)) {
223  if (servlet == "cms_conditions_data") {
224  // use the default servlet from site-local-config.xml
225  servlet = "";
226  }
227  return proto + frontierConnect(servlet) + input.substr(endservlet);
228  }
229  }
230  return input;
231  }
232 
234 
236 
238 
240 
242 
244 
246 
247  unsigned int const *SiteLocalConfigService::sourceTimeout() const { return m_timeoutPtr; }
248 
251  }
252 
253  unsigned int SiteLocalConfigService::debugLevel() const { return m_debugLevel; }
254 
255  std::vector<std::string> const *SiteLocalConfigService::sourceNativeProtocols() const {
256  return m_nativeProtocolsPtr;
257  }
258 
260  return m_statisticsAddrInfo;
261  }
262 
263  std::set<std::string> const *SiteLocalConfigService::statisticsInfo() const {
264  return m_statisticsInfoAvail ? &m_statisticsInfo : nullptr;
265  }
266 
268 
270  tinyxml2::XMLDocument doc;
271  auto loadErr = doc.LoadFile(url.c_str());
272  if (loadErr != tinyxml2::XML_SUCCESS) {
273  return;
274  }
275 
276  // The Site Config has the following format
277  // <site-local-config>
278  // <site name="FNAL">
279  // <event-data>
280  // <catalog url="trivialcatalog_file:/x/y/z.xml"/>
281  // <rfiotype value="castor"/>
282  // </event-data>
283  // <calib-data>
284  // <catalog url="trivialcatalog_file:/x/y/z.xml"/>
285  // <frontier-connect>
286  // ... frontier-interpreted server/proxy xml ...
287  // </frontier-connect>
288  // </calib-data>
289  // <source-config>
290  // <cache-temp-dir name="/a/b/c"/>
291  // <cache-hint value="..."/>
292  // <read-hint value="..."/>
293  // <ttree-cache-size value="0"/>
294  // <native-protocols>
295  // <protocol prefix="dcache"/>
296  // <protocol prefix="file"/>
297  // </native-protocols>
298  // </source-config>
299  // </site>
300  // </site-local-config>
301  auto rootElement = doc.RootElement();
302 
303  for (auto site = rootElement->FirstChildElement("site"); site != nullptr;
304  site = site->NextSiblingElement("site")) {
305  // Parse the site name
306  m_siteName = safe(site->Attribute("name"));
307 
308  // Parsing of the event data section
309  {
310  auto eventData = site->FirstChildElement("event-data");
311  if (eventData) {
312  auto catalog = eventData->FirstChildElement("catalog");
313  if (catalog) {
314  m_dataCatalog = safe(catalog->Attribute("url"));
315  catalog = catalog->NextSiblingElement("catalog");
316  if (catalog) {
317  m_fallbackDataCatalog = safe(catalog->Attribute("url"));
318  }
319  }
320  auto rfiotype = eventData->FirstChildElement("rfiotype");
321  if (rfiotype) {
322  m_rfioType = safe(rfiotype->Attribute("value"));
323  }
324  }
325  }
326 
327  // Parsing of the calib-data section
328  {
329  auto calibData = site->FirstChildElement("calib-data");
330 
331  if (calibData) {
332  auto frontierConnect = calibData->FirstChildElement("frontier-connect");
333 
334  if (frontierConnect) {
335  m_frontierConnect = _toParenString(*frontierConnect);
336  }
337  }
338  }
339  // Parsing of the source config section
340  {
341  auto sourceConfig = site->FirstChildElement("source-config");
342 
343  if (sourceConfig) {
344  auto cacheTempDir = sourceConfig->FirstChildElement("cache-temp-dir");
345 
346  if (cacheTempDir) {
347  m_cacheTempDir = safe(cacheTempDir->Attribute("name"));
349  }
350 
351  auto cacheMinFree = sourceConfig->FirstChildElement("cache-min-free");
352 
353  if (cacheMinFree) {
354  //TODO what did xerces do if it couldn't convert?
355  m_cacheMinFree = cacheMinFree->DoubleAttribute("value");
357  }
358 
359  auto cacheHint = sourceConfig->FirstChildElement("cache-hint");
360 
361  if (cacheHint) {
362  m_cacheHint = safe(cacheHint->Attribute("value"));
364  }
365 
366  auto cloneCacheHint = sourceConfig->FirstChildElement("clone-cache-hint");
367 
368  if (cloneCacheHint) {
369  m_cloneCacheHint = safe(cloneCacheHint->Attribute("value"));
371  }
372 
373  auto readHint = sourceConfig->FirstChildElement("read-hint");
374 
375  if (readHint) {
376  m_readHint = safe(readHint->Attribute("value"));
378  }
379 
380  auto ttreeCacheSize = sourceConfig->FirstChildElement("ttree-cache-size");
381 
382  if (ttreeCacheSize) {
383  m_ttreeCacheSize = ttreeCacheSize->UnsignedAttribute("value");
385  }
386 
387  auto timeout = sourceConfig->FirstChildElement("timeout-in-seconds");
388 
389  if (timeout) {
390  m_timeout = timeout->UnsignedAttribute("value");
392  }
393 
394  auto statsDest = sourceConfig->FirstChildElement("statistics-destination");
395 
396  if (statsDest) {
397  m_statisticsDestination = safe(statsDest->Attribute("endpoint"));
398  if (m_statisticsDestination.empty()) {
399  m_statisticsDestination = safe(statsDest->Attribute("name"));
400  }
401  std::string tmpStatisticsInfo = safe(statsDest->Attribute("info"));
402  boost::split(m_statisticsInfo, tmpStatisticsInfo, boost::is_any_of("\t ,"));
403  m_statisticsInfoAvail = !tmpStatisticsInfo.empty();
404  }
405 
406  auto prefetching = sourceConfig->FirstChildElement("prefetching");
407 
408  if (prefetching) {
409  m_enablePrefetching = prefetching->BoolAttribute("value");
411  }
412 
413  auto nativeProtocol = sourceConfig->FirstChildElement("native-protocols");
414 
415  if (nativeProtocol) {
416  for (auto child = nativeProtocol->FirstChildElement(); child != nullptr;
417  child = child->NextSiblingElement()) {
418  m_nativeProtocols.push_back(safe(child->Attribute("prefix")));
419  }
421  }
422  }
423  }
424  }
425  m_connected = true;
426  }
427 
429  std::vector<std::string> inputStrings;
430  boost::split(inputStrings, m_statisticsDestination, boost::is_any_of(":"));
431  const std::string &host = inputStrings[0];
432  const std::string &port = (inputStrings.size() > 1) ? inputStrings[1] : m_statisticsDefaultPort;
433  struct addrinfo *res;
434  struct addrinfo hints;
435  memset(&hints, '\0', sizeof(hints));
436  hints.ai_socktype = SOCK_DGRAM;
437  hints.ai_flags = AI_ADDRCONFIG;
438  hints.ai_family = AF_UNSPEC;
439  int e = getaddrinfo(host.c_str(), port.c_str(), &hints, &res);
440  if (e != 0) {
441  // Silent failure - there's no way to report non-fatal failures from here.
442  return;
443  }
444  m_statisticsAddrInfo = res;
445  }
446 
449  desc.setComment("Service to translate logical file names to physical file names.");
450 
451  desc.addOptionalUntracked<std::string>("siteLocalConfigFileUrl", std::string())
452  ->setComment(
453  "Specify the file containing the site local config. Empty string will load from default directory.");
454  desc.addOptionalUntracked<std::string>("overrideSourceCacheTempDir");
455  desc.addOptionalUntracked<double>("overrideSourceCacheMinFree");
456  desc.addOptionalUntracked<std::string>("overrideSourceCacheHintDir");
457  desc.addOptionalUntracked<std::string>("overrideSourceCloneCacheHintDir")
458  ->setComment("Provide an alternate cache hint for fast cloning.");
459  desc.addOptionalUntracked<std::string>("overrideSourceReadHint");
460  desc.addOptionalUntracked<std::vector<std::string> >("overrideSourceNativeProtocols");
461  desc.addOptionalUntracked<unsigned int>("overrideSourceTTreeCacheSize");
462  desc.addOptionalUntracked<unsigned int>("overrideSourceTimeout");
463  desc.addOptionalUntracked<unsigned int>("debugLevel");
464  desc.addOptionalUntracked<bool>("overridePrefetching")
465  ->setComment("Request ROOT to asynchronously prefetch I/O during computation.");
466  desc.addOptionalUntracked<std::string>("overrideStatisticsDestination")
467  ->setComment(
468  "Provide an alternate network destination for I/O statistics (must be in the form of host:port).");
469  desc.addOptionalUntracked<std::vector<std::string> >("overrideStatisticsInfo")
470  ->setComment(
471  "Provide an alternate listing of statistics to send (comma separated list; current options are 'dn' or "
472  "'nodn'). If left blank, all information is snet (including DNs).");
473 
474  descriptions.add("SiteLocalConfigService", desc);
475  }
476  } // namespace service
477 } // namespace edm
std::vector< std::string_view > split(std::string_view, const char *)
T getUntrackedParameter(std::string const &, T const &) const
host
Definition: query.py:115
unsigned int const * sourceTTreeCacheSize() const override
static void fillDescriptions(ConfigurationDescriptions &descriptions)
std::string const fallbackDataCatalog(void) const override
static const std::string m_statisticsDefaultPort
#define nullptr
bool exists(std::string const &parameterName) const
checks if a parameter exists
uint16_t size_type
std::string const dataCatalog(void) const override
port
Definition: query.py:116
Definition: Electron.h:6
std::vector< std::string > const * m_nativeProtocolsPtr
static std::string const input
Definition: EdmProvDump.cc:48
std::set< std::string > const * statisticsInfo() const override
void setComment(std::string const &value)
edm::propagate_const< struct addrinfo * > m_statisticsAddrInfo
unsigned int const * sourceTimeout() const override
std::string const rfioType(void) const override
std::string const * sourceCloneCacheHint() const override
std::string const & siteName() const override
static char kEmptyString[1]
std::vector< std::string > const * sourceNativeProtocols() const override
std::string const * sourceCacheHint() const override
struct addrinfo const * statisticsDestination() const override
std::vector< std::string > m_nativeProtocols
double const * sourceCacheMinFree() const override
std::string const * sourceReadHint() const override
std::string const lookupCalibConnect(std::string const &input) const override
SiteLocalConfigService(ParameterSet const &pset)
void add(std::string const &label, ParameterSetDescription const &psetDescription)
HLT enums.
std::string const * sourceCacheTempDir() const override
ParameterDescriptionBase * addOptionalUntracked(U const &iLabel, T const &value)
std::string const frontierConnect(std::string const &servlet) const
tmp
align.sh
Definition: createJobs.py:716
long double T
def move(src, dest)
Definition: eostools.py:511
#define constexpr
unsigned int debugLevel() const override