#include <FileLocator.h>

Classes
struct	Rule

Public Member Functions
	FileLocator (edm::CatalogAttributes const &catAttr, unsigned iCatalog=0, std::string const &storageDescriptionPath=std::string())

	FileLocator (std::string const &catUrl, unsigned iCatalog=0)

std::string	pfn (std::string const &ilfn, edm::CatalogType catType) const

	~FileLocator ()

Private Types
typedef std::map< std::string, Rules >	ProtocolRules

typedef std::vector< Rule >	Rules

Private Member Functions
std::string	applyRules (ProtocolRules const &protocolRules, std::string const &protocol, std::string const &destination, bool direct, std::string name) const

std::string	convert (std::string const &input, ProtocolRules const &rules, bool direct) const

void	init (edm::CatalogAttributes const &input_dataCatalog, unsigned iCatalog, std::string const &storageDescriptionPath)

void	init_trivialCatalog (std::string const &catUrl, unsigned iCatalog)

void	parseRule (boost::property_tree::ptree::value_type const &storageRule, std::string const &protocol, ProtocolRules &rules)

void	parseRuleTrivialCatalog (tinyxml2::XMLElement *ruleNode, ProtocolRules &rules)

Private Attributes
std::string	m_destination

ProtocolRules	m_directRules

ProtocolRules	m_directRules_trivialCatalog

std::string	m_filename

std::string	m_fileType

ProtocolRules	m_inverseRules

std::string	m_prefix

std::vector< std::string >	m_protocols

Static Private Attributes
static int	s_numberOfInstances

Detailed Description

Definition at line 15 of file FileLocator.h.

Member Typedef Documentation

◆ ProtocolRules

typedef std::map<std::string, Rules> edm::FileLocator::ProtocolRules

private

Definition at line 42 of file FileLocator.h.

◆ Rules

typedef std::vector<Rule> edm::FileLocator::Rules

private

Definition at line 41 of file FileLocator.h.

Constructor & Destructor Documentation

◆ FileLocator() [1/2]

edm::FileLocator::FileLocator	(	edm::CatalogAttributes const &	catAttr,
		unsigned	iCatalog = `0`,
		std::string const &	storageDescriptionPath = `std::string()`
	)

explicit

Definition at line 53 of file FileLocator.cc.

References init().

       : m_destination("any") {
     init(catAttr, iCatalog, storageDescriptionPath);
   }

◆ FileLocator() [2/2]

edm::FileLocator::FileLocator	(	std::string const &	catUrl,
		unsigned	iCatalog = `0`
	)

explicit

Definition at line 49 of file FileLocator.cc.

References init_trivialCatalog().

                                                                      : m_destination("any") {
     init_trivialCatalog(catUrl, iCatalog);
   }

◆ ~FileLocator()

edm::FileLocator::~FileLocator ( )

Definition at line 60 of file FileLocator.cc.

60 {}

Member Function Documentation

◆ applyRules()

std::string edm::FileLocator::applyRules	(	ProtocolRules const &	protocolRules,
		std::string const &	protocol,
		std::string const &	destination,
		bool	direct,
		std::string	name
	)		const

private

Definition at line 316 of file FileLocator.cc.

References HLTMuonOfflineAnalyzer_cff::destination, mps_fire::i, Skims_PA_cff::name, findAndChange::protocol, cmsCodeRulesChecker::rules, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by convert().

                                                             {
     ProtocolRules::const_iterator const rulesIterator = protocolRules.find(protocol);
     if (rulesIterator == protocolRules.end()) {
       return "";
     }
 
     Rules const& rules = (*(rulesIterator)).second;
 
     std::smatch destinationMatches;
     std::smatch nameMatches;
 
     /* Look up for a matching rule*/
     for (Rules::const_iterator i = rules.begin(); i != rules.end(); ++i) {
       if (!std::regex_match(destination, destinationMatches, i->destinationMatch)) {
         continue;
       }
 
       if (!std::regex_match(name, i->pathMatch)) {
         continue;
       }
 
       // std::cerr << "Rule " << i->pathMatch << "matched! " << std::endl;
 
       std::string const chain = i->chain;
       if ((direct == true) && (!chain.empty())) {
         name = applyRules(protocolRules, chain, destination, direct, name);
         if (name.empty()) {
           return "";
         }
       }
 
       std::regex_match(name, nameMatches, i->pathMatch);
       name = replaceWithRegexp(nameMatches, i->result);
 
       if ((direct == false) && (!chain.empty())) {
         name = applyRules(protocolRules, chain, destination, direct, name);
       }
       return name;
     }
     return "";
   }

◆ convert()

std::string edm::FileLocator::convert	(	std::string const &	input,
		ProtocolRules const &	rules,
		bool	direct
	)		const

private

Definition at line 68 of file FileLocator.cc.

References applyRules(), input, m_destination, m_prefix, m_protocols, MillePedeFileConverter_cfg::out, pi, cmsCodeRulesChecker::rules, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by pfn().

                                                                                                       {
     std::string out = "";
     //check if input is an authentic LFN
     if (input.compare(0, 7, kLFNPrefix) != 0)
       return out;
     //use prefix in the protocol
     if (!m_prefix.empty()) {
       out = m_prefix + "/" + input;
       if (input[0] == '/')
         out = m_prefix + input;
       return out;
     }
     //no prefix in the protocol, use rule
     for (size_t pi = 0, pe = m_protocols.size(); pi != pe; ++pi) {
       out = applyRules(rules, m_protocols[pi], m_destination, direct, input);
       if (!out.empty()) {
         return out;
       }
     }
     return out;
   }

◆ init()

void edm::FileLocator::init	(	edm::CatalogAttributes const &	input_dataCatalog,
		unsigned	iCatalog,
		std::string const &	storageDescriptionPath
	)

private

Definition at line 223 of file FileLocator.cc.

References cms::Exception::addContext(), MillePedeFileConverter_cfg::e, edm::CatalogAttributes::empty(), cppFunctionSkipper::exception, edm::Service< T >::isAvailable(), fwlite::kEmptyString, m_directRules, m_prefix, m_protocols, parseRule(), castor_dqm_sourceclient_file_cfg::path, findAndChange::protocol, edm::CatalogAttributes::protocol, cmsCodeRulesChecker::rules, edm::CatalogAttributes::storageSite, AlCaHLTBitMon_QueryRunRegistry::string, and edm::CatalogAttributes::volume.

Referenced by FileLocator().

                                                                   {
     Service<SiteLocalConfig> localconfservice;
     edm::CatalogAttributes aCatalog = input_dataCatalog;
     if (input_dataCatalog.empty()) {
       if (!localconfservice.isAvailable()) {
         cms::Exception ex("FileCatalog");
         ex << "edm::SiteLocalConfigService is not available";
         ex.addContext("Calling edm::FileLocator::init()");
         throw ex;
       }
       if (iCatalog >= localconfservice->dataCatalogs().size()) {
         cms::Exception ex("FileCatalog");
         ex << "Request nonexistence data catalog";
         ex.addContext("Calling edm::FileLocator::init()");
         throw ex;
       }
       aCatalog = localconfservice->dataCatalogs()[iCatalog];
     }
 
     std::filesystem::path filename_storage = localconfservice->storageDescriptionPath(aCatalog);
 
     //use path to storage description from input parameter
     if (!storageDescriptionPath.empty())
       filename_storage = storageDescriptionPath;
 
     //now read json
     pt::ptree json;
     try {
       boost::property_tree::read_json(filename_storage.string(), json);
     } catch (std::exception& e) {
       cms::Exception ex("FileCatalog");
       ex << "Can not open storage.json (" << filename_storage.string()
          << "). Check SITECONFIG_PATH and site-local-config.xml <data-access>";
       ex.addContext("edm::FileLocator:init()");
       throw ex;
     }
     auto found_site = std::find_if(json.begin(), json.end(), [&](pt::ptree::value_type const& site) {
       //get site name
       std::string siteName = site.second.get("site", kEmptyString);
       //get volume name
       std::string volName = site.second.get("volume", kEmptyString);
       return aCatalog.storageSite == siteName && aCatalog.volume == volName;
     });
 
     //let enforce that site-local-config.xml and storage.json contains valid catalogs in <data-access>, in which site defined in site-local-config.xml <data-access> should be found in storage.json
     if (found_site == json.end()) {
       cms::Exception ex("FileCatalog");
       ex << "Can not find storage site \"" << aCatalog.storageSite << "\" and volume \"" << aCatalog.volume
          << "\" in storage.json. Check site-local-config.xml <data-access> and storage.json";
       ex.addContext("edm::FileLocator:init()");
       throw ex;
     }
 
     const pt::ptree& protocols = found_site->second.find("protocols")->second;
     auto found_protocol = std::find_if(protocols.begin(), protocols.end(), [&](pt::ptree::value_type const& protocol) {
       std::string protName = protocol.second.get("protocol", kEmptyString);
       return aCatalog.protocol == protName;
     });
 
     //let enforce that site-local-config.xml and storage.json contains valid catalogs, in which protocol defined in site-local-config.xml <data-access> should be found in storage.json
     if (found_protocol == protocols.end()) {
       cms::Exception ex("FileCatalog");
       ex << "Can not find protocol \"" << aCatalog.protocol << "\" for the storage site \"" << aCatalog.storageSite
          << "\" and volume \"" << aCatalog.volume
          << "\" in storage.json. Check site-local-config.xml <data-access> and storage.json";
       ex.addContext("edm::FileLocator:init()");
       throw ex;
     }
 
     std::string protName = found_protocol->second.get("protocol", kEmptyString);
     m_protocols.push_back(protName);
     m_prefix = found_protocol->second.get("prefix", kEmptyString);
     if (m_prefix == kEmptyString) {
       //get rules
       if (found_protocol->second.find("rules") == found_protocol->second.not_found()) {
         cms::Exception ex("FileCatalog");
         ex << "protocol must contain either a prefix or rules, "
            << "neither found for protocol \"" << aCatalog.protocol << "\" for the storage site \""
            << aCatalog.storageSite << "\" and volume \"" << aCatalog.volume
            << "\" in storage.json. Check site-local-config.xml <data-access> and storage.json";
         ex.addContext("edm::FileLocator:init()");
         throw ex;
       }
       const pt::ptree& rules = found_protocol->second.find("rules")->second;
       //loop over rules
       for (pt::ptree::value_type const& storageRule : rules) {
         parseRule(storageRule, protName, m_directRules);
       }
     }
   }

◆ init_trivialCatalog()

void edm::FileLocator::init_trivialCatalog	(	std::string const &	catUrl,
		unsigned	iCatalog
	)

private

Definition at line 129 of file FileLocator.cc.

References findQualityFiles::comma, GCPpyPlots::configFile, hcalRecHitTable_cff::doc, Exception, edm::Service< T >::isAvailable(), m_destination, m_directRules_trivialCatalog, m_filename, m_inverseRules, m_protocols, fileinputsource_cfi::option, parseRuleTrivialCatalog(), submitPVValidationJobs::split(), AlCaHLTBitMon_QueryRunRegistry::string, and relmon_authenticated_wget::url.

Referenced by FileLocator().

                                                                                   {
     std::string url = catUrl;
     if (url.empty()) {
       Service<SiteLocalConfig> localconfservice;
       if (!localconfservice.isAvailable())
         throw cms::Exception("TrivialFileCatalog", "edm::SiteLocalConfigService is not available");
       if (iCatalog >= localconfservice->trivialDataCatalogs().size())
         throw cms::Exception("TrivialFileCatalog", "edm::FileLocator: Request nonexistence data catalog");
       url = localconfservice->trivialDataCatalogs()[iCatalog];
     }
 
     if (url.find("file:") == std::string::npos) {
       throw cms::Exception("TrivialFileCatalog",
                            "TrivialFileCatalog::connect: Malformed url for file catalog configuration");
     }
 
     url = url.erase(0, url.find(':') + 1);
 
     std::vector<std::string> tokens;
     boost::algorithm::split(tokens, url, boost::is_any_of(std::string("?")));
     m_filename = tokens[0];
 
     if (tokens.size() == 2) {
       std::string const options = tokens[1];
       std::vector<std::string> optionTokens;
       boost::algorithm::split(optionTokens, options, boost::is_any_of(std::string("&")));
 
       std::string const equalSign("=");
       std::string const comma(",");
 
       for (size_t oi = 0, oe = optionTokens.size(); oi != oe; ++oi) {
         std::string const option = optionTokens[oi];
         std::vector<std::string> argTokens;
         boost::algorithm::split(argTokens, option, boost::is_any_of(equalSign));
 
         if (argTokens.size() != 2) {
           throw cms::Exception("TrivialFileCatalog",
                                "TrivialFileCatalog::connect: Malformed url for file catalog configuration");
         }
 
         if (argTokens[0] == "protocol") {
           boost::algorithm::split(m_protocols, argTokens[1], boost::is_any_of(comma));
         } else if (argTokens[0] == "destination") {
           m_destination = argTokens[1];
         }
       }
     }
 
     if (m_protocols.empty()) {
       throw cms::Exception("TrivialFileCatalog",
                            "TrivialFileCatalog::connect: protocol was not supplied in the contact string");
     }
 
     std::ifstream configFile;
     configFile.open(m_filename.c_str());
 
     if (!configFile.good() || !configFile.is_open()) {
       throw cms::Exception("TrivialFileCatalog",
                            "TrivialFileCatalog::connect: Unable to open trivial file catalog " + m_filename);
     }
 
     configFile.close();
 
     tinyxml2::XMLDocument doc;
     auto loadErr = doc.LoadFile(m_filename.c_str());
     if (loadErr != tinyxml2::XML_SUCCESS) {
       throw cms::Exception("TrivialFileCatalog")
           << "tinyxml file load failed with error : " << doc.ErrorStr() << std::endl;
     }
     /* trivialFileCatalog matches the following xml schema
      FIXME: write a proper DTD
      <storage-mapping>
      <lfn-to-pfn protocol="direct" destination-match=".*"
      path-match="lfn/guid match regular expression"
      result="/castor/cern.ch/cms/$1"/>
      <pfn-to-lfn protocol="srm"
      path-match="lfn/guid match regular expression"
      result="$1"/>
      </storage-mapping>
     */
     auto rootElement = doc.RootElement();
     /*first of all do the lfn-to-pfn bit*/
     for (auto el = rootElement->FirstChildElement("lfn-to-pfn"); el != nullptr;
          el = el->NextSiblingElement("lfn-to-pfn")) {
       parseRuleTrivialCatalog(el, m_directRules_trivialCatalog);
     }
 
     /*Then we handle the pfn-to-lfn bit*/
     for (auto el = rootElement->FirstChildElement("pfn-to-lfn"); el != nullptr;
          el = el->NextSiblingElement("pfn-to-lfn")) {
       parseRuleTrivialCatalog(el, m_inverseRules);
     }
   }

◆ parseRule()

void edm::FileLocator::parseRule	(	boost::property_tree::ptree::value_type const &	storageRule,
		std::string const &	protocol,
		ProtocolRules &	rules
	)

private

Definition at line 113 of file FileLocator.cc.

References Exception, eostools::move(), edm::FileLocator::Rule::pathMatch, findAndChange::protocol, mps_fire::result, cmsCodeRulesChecker::rules, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by init().

                                                     {
     if (storageRule.second.empty()) {
       throw cms::Exception("RucioFileCatalog", "edm::FileLocator::parseRule Malformed storage rule");
     }
     auto const pathMatchRegexp = storageRule.second.get<std::string>("lfn");
     auto const result = storageRule.second.get<std::string>("pfn");
     Rule rule;
     rule.pathMatch.assign(pathMatchRegexp);
     rule.destinationMatch.assign(".*");
     rule.result = result;
     rule.chain = "";
     rules[protocol].emplace_back(std::move(rule));
   }

◆ parseRuleTrivialCatalog()

void edm::FileLocator::parseRuleTrivialCatalog	(	tinyxml2::XMLElement *	ruleNode,
		ProtocolRules &	rules
	)

private

Definition at line 90 of file FileLocator.cc.

References Exception, eostools::move(), or, edm::FileLocator::Rule::pathMatch, findAndChange::protocol, mps_fire::result, cmsCodeRulesChecker::rules, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by init_trivialCatalog().

                                                                                                  {
     if (!ruleElement) {
       throw cms::Exception("TrivialFileCatalog", std::string("TrivialFileCatalog::connect: Malformed trivial catalog"));
     }
 
     auto const protocol = safe(ruleElement->Attribute("protocol"));
     auto destinationMatchRegexp = ruleElement->Attribute("destination-match");
     if (destinationMatchRegexp == nullptr or destinationMatchRegexp[0] == 0) {
       destinationMatchRegexp = ".*";
     }
 
     auto const pathMatchRegexp = safe(ruleElement->Attribute("path-match"));
     auto const result = safe(ruleElement->Attribute("result"));
     auto const chain = safe(ruleElement->Attribute("chain"));
 
     Rule rule;
     rule.pathMatch.assign(pathMatchRegexp);
     rule.destinationMatch.assign(destinationMatchRegexp);
     rule.result = result;
     rule.chain = chain;
     rules[protocol].emplace_back(std::move(rule));
   }

◆ pfn()

std::string edm::FileLocator::pfn	(	std::string const &	ilfn,
		edm::CatalogType	catType
	)		const

Definition at line 62 of file FileLocator.cc.

References convert(), m_directRules, m_directRules_trivialCatalog, and edm::TrivialCatalog.

                                                                                 {
     if (catType == edm::CatalogType::TrivialCatalog)
       return convert(ilfn, m_directRules_trivialCatalog, true);
     return convert(ilfn, m_directRules, true);
   }

Member Data Documentation

◆ m_destination

std::string edm::FileLocator::m_destination

private

Definition at line 72 of file FileLocator.h.

Referenced by convert(), and init_trivialCatalog().

◆ m_directRules

ProtocolRules edm::FileLocator::m_directRules

private

Direct rules are used to do the mapping from LFN to PFN taken from storage.json

Definition at line 67 of file FileLocator.h.

Referenced by init(), and pfn().

◆ m_directRules_trivialCatalog

ProtocolRules edm::FileLocator::m_directRules_trivialCatalog

private

Direct rules are used to do the mapping from LFN to PFN.

Definition at line 63 of file FileLocator.h.

Referenced by init_trivialCatalog(), and pfn().

◆ m_filename

std::string edm::FileLocator::m_filename

private

Definition at line 70 of file FileLocator.h.

Referenced by init_trivialCatalog().

◆ m_fileType

std::string edm::FileLocator::m_fileType

private

Definition at line 69 of file FileLocator.h.

◆ m_inverseRules

ProtocolRules edm::FileLocator::m_inverseRules

private

Inverse rules are used to do the mapping from PFN to LFN

Definition at line 65 of file FileLocator.h.

Referenced by init_trivialCatalog().

◆ m_prefix

std::string edm::FileLocator::m_prefix

private

Definition at line 73 of file FileLocator.h.

Referenced by convert(), and init().

◆ m_protocols

std::vector<std::string> edm::FileLocator::m_protocols

private

Definition at line 71 of file FileLocator.h.

Referenced by convert(), init(), and init_trivialCatalog().

◆ s_numberOfInstances

int edm::FileLocator::s_numberOfInstances

staticprivate

For the time being the only allowed configuration item is a prefix to be added to the GUID/LFN.

Definition at line 32 of file FileLocator.h.

Classes

Public Member Functions

Private Types

Private Member Functions

Private Attributes

Static Private Attributes

Detailed Description

Member Typedef Documentation

◆ ProtocolRules

◆ Rules

Constructor & Destructor Documentation

◆ FileLocator() [1/2]

◆ FileLocator() [2/2]

◆ ~FileLocator()

Member Function Documentation

◆ applyRules()

◆ convert()

◆ init()

◆ init_trivialCatalog()

◆ parseRule()

◆ parseRuleTrivialCatalog()

◆ pfn()

Member Data Documentation

◆ m_destination

◆ m_directRules

◆ m_directRules_trivialCatalog

◆ m_filename

◆ m_fileType

◆ m_inverseRules

◆ m_prefix

◆ m_protocols

◆ s_numberOfInstances