StorageFactory Class Reference

#include <StorageFactory.h>

Public Types
enum	CacheHint { CACHE_HINT_APPLICATION, CACHE_HINT_STORAGE, CACHE_HINT_LAZY_DOWNLOAD, CACHE_HINT_AUTO_DETECT }
enum	ReadHint { READ_HINT_UNBUFFERED, READ_HINT_READAHEAD, READ_HINT_AUTO }

Public Member Functions
bool	accounting (void) const
CacheHint	cacheHint (void) const
bool	check (const std::string &url, IOOffset *size=nullptr) const
unsigned int	debugLevel (void) const
bool	enableAccounting (bool enabled)
std::unique_ptr< Storage >	open (const std::string &url, int mode=IOFlags::OpenRead) const
ReadHint	readHint (void) const
void	setCacheHint (CacheHint value)
void	setDebugLevel (unsigned int level)
void	setReadHint (ReadHint value)
void	setTempDir (const std::string &s, double minFreeSpace)
void	setTimeout (unsigned int timeout)
void	stagein (const std::string &url) const
std::string	tempDir (void) const
double	tempMinFree (void) const
std::string	tempPath (void) const
unsigned int	timeout (void) const
std::unique_ptr< Storage >	wrapNonLocalFile (std::unique_ptr< Storage > s, const std::string &proto, const std::string &path, int mode) const
	~StorageFactory (void)

Static Public Member Functions
static const StorageFactory *	get (void)
static StorageFactory *	getToModify (void)

Private Types
typedef tbb::concurrent_unordered_map< std::string, std::shared_ptr< StorageMaker > >	MakerTable

Private Member Functions
StorageMaker *	getMaker (const std::string &proto) const
StorageMaker *	getMaker (const std::string &url, std::string &protocol, std::string &rest) const
	StorageFactory (void)

Private Attributes
bool	m_accounting
CacheHint	m_cacheHint
unsigned int	m_debugLevel
LocalFileSystem	m_lfs
MakerTable	m_makers
ReadHint	m_readHint
std::string	m_tempdir
double	m_tempfree
std::string	m_temppath
unsigned int	m_timeout
std::string	m_unusableDirWarnings

Static Private Attributes
static StorageFactory	s_instance

Detailed Description

Definition at line 13 of file StorageFactory.h.

Member Typedef Documentation

MakerTable

typedef tbb::concurrent_unordered_map<std::string, std::shared_ptr<StorageMaker> > StorageFactory::MakerTable

private

Definition at line 57 of file StorageFactory.h.

Member Enumeration Documentation

CacheHint

enum StorageFactory::CacheHint

Enumerator
CACHE_HINT_APPLICATION
CACHE_HINT_STORAGE
CACHE_HINT_LAZY_DOWNLOAD
CACHE_HINT_AUTO_DETECT

Definition at line 15 of file StorageFactory.h.

{ CACHE_HINT_APPLICATION, CACHE_HINT_STORAGE, CACHE_HINT_LAZY_DOWNLOAD, CACHE_HINT_AUTO_DETECT };

ReadHint

enum StorageFactory::ReadHint

Enumerator
READ_HINT_UNBUFFERED
READ_HINT_READAHEAD
READ_HINT_AUTO

Definition at line 17 of file StorageFactory.h.

{ READ_HINT_UNBUFFERED, READ_HINT_READAHEAD, READ_HINT_AUTO };

Constructor & Destructor Documentation

~StorageFactory()

StorageFactory::~StorageFactory

(

void

)

Definition at line 26 of file StorageFactory.cc.

{}

StorageFactory()

StorageFactory::StorageFactory ( void  )

private

Definition at line 15 of file StorageFactory.cc.

    : m_cacheHint(CACHE_HINT_AUTO_DETECT),
      m_readHint(READ_HINT_AUTO),
      m_accounting(false),
      m_tempfree(4.),  // GB
      m_temppath(".:$TMPDIR"),
      m_timeout(0U),
      m_debugLevel(0U) {
  setTempDir(m_temppath, m_tempfree);
}

References m_tempfree, m_temppath, and setTempDir().

Member Function Documentation

accounting()

bool StorageFactory::accounting

(

void

)

const

Definition at line 38 of file StorageFactory.cc.

{ return m_accounting; }

References m_accounting.

Referenced by wrapNonLocalFile().

cacheHint()

StorageFactory::CacheHint StorageFactory::cacheHint

(

void

)

const

Definition at line 42 of file StorageFactory.cc.

{ return m_cacheHint; }

References m_cacheHint.

Referenced by wrapNonLocalFile().

check()

bool StorageFactory::check	(	const std::string &	url,
		IOOffset *	size = nullptr
	)		const

Definition at line 181 of file StorageFactory.cc.

                                                                       {
  std::string protocol;
  std::string rest;
 
  bool ret = false;
  std::unique_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker(url, protocol, rest)) {
    if (m_accounting) {
      auto token = StorageAccount::tokenForStorageClassName(protocol);
      stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::check));
    }
    try {
      ret = maker->check(
          protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size);
      if (stats)
        stats->tick();
    } catch (cms::Exception &err) {
      edm::LogWarning("StorageFactory::check()")
          << "Existence or size check for the file '" << url << "' failed because:\n"
          << err.explainSelf();
    }
  }
 
  return ret;
}

References StorageAccount::check, StorageAccount::counter(), submitPVResolutionJobs::err, getMaker(), m_accounting, m_debugLevel, m_timeout, mergeAndRegister::rest, runTheMatrix::ret, StorageMaker::AuxSettings::setDebugLevel(), StorageMaker::AuxSettings::setTimeout(), findQualityFiles::size, dqmMemoryStats::stats, AlCaHLTBitMon_QueryRunRegistry::string, unpackBuffers-CaloStage2::token, StorageAccount::tokenForStorageClassName(), and relmon_authenticated_wget::url.

Referenced by TStorageFactorySystem::AccessPathName(), and PixelSLinkDataInputSource::PixelSLinkDataInputSource().

debugLevel()

unsigned int StorageFactory::debugLevel

(

void

)

const

Definition at line 54 of file StorageFactory.cc.

{ return m_debugLevel; }

References m_debugLevel.

enableAccounting()

bool StorageFactory::enableAccounting

(

bool

enabled

)

Definition at line 32 of file StorageFactory.cc.

                                                  {
  bool old = m_accounting;
  m_accounting = enabled;
  return old;
}

References pixel_dqm_sourceclient-live_cfg::enabled, and m_accounting.

Referenced by PixelSLinkDataInputSource::PixelSLinkDataInputSource().

get()

const StorageFactory * StorageFactory::get ( void  )

static

Definition at line 28 of file StorageFactory.cc.

{ return &s_instance; }

References s_instance.

Referenced by Options.Options::__getitem__(), betterConfigParser.BetterConfigParser::__updateDict(), submitPVValidationJobs.BetterConfigParser::__updateDict(), TStorageFactorySystem::AccessPathName(), lhef::LHEReader::FileSource::FileSource(), betterConfigParser.BetterConfigParser::getCompares(), betterConfigParser.BetterConfigParser::getGeneral(), TStorageFactorySystem::GetPathInfo(), betterConfigParser.BetterConfigParser::getResultingSection(), submitPVValidationJobs.BetterConfigParser::getResultingSection(), TStorageFactoryFile::Initialize(), GsiFTPStorageMaker::open(), HttpStorageMaker::open(), DavixStorageMaker::open(), LocalStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), XrdStorageMaker::open(), StormLcgGtStorageMaker::open(), edm::StreamerInputFile::openStreamerFile(), PixelSLinkDataInputSource::PixelSLinkDataInputSource(), TStorageFactoryFile::ReadBufferAsync(), edm::RootPrimaryFileSequence::RootPrimaryFileSequence(), edm::RootSecondaryFileSequence::RootSecondaryFileSequence(), and TStorageFactoryFile::SysOpen().

getMaker() [1/2]

StorageMaker * StorageFactory::getMaker ( const std::string &  proto ) const

private

Definition at line 96 of file StorageFactory.cc.

                                                                   {
  auto itFound = m_makers.find(proto);
  if (itFound != m_makers.end()) {
    return itFound->second.get();
  }
  if (!edmplugin::PluginManager::isAvailable()) {
    edmplugin::PluginManager::configure(edmplugin::standard::config());
  }
  std::shared_ptr<StorageMaker> instance{StorageMakerFactory::get()->tryToCreate(proto)};
  auto insertResult = m_makers.insert(MakerTable::value_type(proto, instance));
  //Can't use instance since it is possible that another thread beat
  // us to the insertion so the map contains a different instance.
  return insertResult.first->second.get();
}

References edmplugin::standard::config(), edmplugin::PluginManager::configure(), get, instance, edmplugin::PluginManager::isAvailable(), and m_makers.

Referenced by check(), getMaker(), open(), and stagein().

getMaker() [2/2]

StorageMaker * StorageFactory::getMaker ( const std::string &  url, std::string &  protocol, std::string &  rest  ) const

private

Definition at line 111 of file StorageFactory.cc.

                                                                                                       {
  size_t p = url.find(':');
  if (p != std::string::npos) {
    protocol = url.substr(0, p);
    rest = url.substr(p + 1);
  } else {
    protocol = "file";
    rest = url;
  }
 
  return getMaker(protocol);
}

References getMaker(), AlCaHLTBitMon_ParallelJobs::p, mergeAndRegister::rest, and relmon_authenticated_wget::url.

getToModify()

StorageFactory * StorageFactory::getToModify ( void  )

static

Definition at line 30 of file StorageFactory.cc.

{ return &s_instance; }

References s_instance.

Referenced by PixelSLinkDataInputSource::PixelSLinkDataInputSource(), and TFileAdaptor::TFileAdaptor().

open()

std::unique_ptr< Storage > StorageFactory::open	(	const std::string &	url,
		int	mode = IOFlags::OpenRead
	)		const

Definition at line 124 of file StorageFactory.cc.

                                                                                  {
  std::string protocol;
  std::string rest;
  std::unique_ptr<Storage> ret;
  std::unique_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker(url, protocol, rest)) {
    if (m_accounting) {
      auto token = StorageAccount::tokenForStorageClassName(protocol);
      stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::open));
    }
    try {
      if (auto storage = maker->open(
              protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout))) {
        if (dynamic_cast<LocalCacheFile *>(storage.get()))
          protocol = "local-cache";
 
        if (m_accounting)
          ret = std::make_unique<StorageAccountProxy>(protocol, std::move(storage));
        else
          ret = std::move(storage);
 
        if (stats)
          stats->tick();
      }
    } catch (cms::Exception &err) {
      err.addContext("Calling StorageFactory::open()");
      err.addAdditionalInfo(err.message());
      err.clearMessage();
      err << "Failed to open the file '" << url << "'";
      throw;
    }
  }
  return ret;
}

References StorageAccount::counter(), submitPVResolutionJobs::err, getMaker(), m_accounting, m_debugLevel, m_timeout, ALCARECOPromptCalibProdSiPixelAli0T_cff::mode, eostools::move(), StorageAccount::open, mergeAndRegister::rest, runTheMatrix::ret, StorageMaker::AuxSettings::setDebugLevel(), StorageMaker::AuxSettings::setTimeout(), dqmMemoryStats::stats, AlCaHLTBitMon_QueryRunRegistry::string, unpackBuffers-CaloStage2::token, StorageAccount::tokenForStorageClassName(), and relmon_authenticated_wget::url.

Referenced by lhef::LHEReader::FileSource::FileSource(), edm::StreamerInputFile::openStreamerFile(), and PixelSLinkDataInputSource::PixelSLinkDataInputSource().

readHint()

StorageFactory::ReadHint StorageFactory::readHint

(

void

)

const

Definition at line 46 of file StorageFactory.cc.

{ return m_readHint; }

References m_readHint.

setCacheHint()

void StorageFactory::setCacheHint

(

CacheHint

value

)

Definition at line 40 of file StorageFactory.cc.

{ m_cacheHint = value; }

References m_cacheHint, and relativeConstraints::value.

setDebugLevel()

void StorageFactory::setDebugLevel

(

unsigned int

level

)

Definition at line 52 of file StorageFactory.cc.

{ m_debugLevel = level; }

References personalPlayback::level, and m_debugLevel.

setReadHint()

void StorageFactory::setReadHint

(

ReadHint

value

)

Definition at line 44 of file StorageFactory.cc.

{ m_readHint = value; }

References m_readHint, and relativeConstraints::value.

setTempDir()

void StorageFactory::setTempDir	(	const std::string &	s,
		double	minFreeSpace
	)

Definition at line 56 of file StorageFactory.cc.

                                                                       {
#if 0
  std::cerr /* edm::LogInfo("StorageFactory") */
    << "Considering path '" << s
    << "', min free space " << minFreeSpace
    << "GB for temp dir" << std::endl;
#endif
 
  size_t begin = 0;
  std::vector<std::string> dirs;
  dirs.reserve(std::count(s.begin(), s.end(), ':') + 1);
 
  while (true) {
    size_t end = s.find(':', begin);
    if (end == std::string::npos) {
      dirs.push_back(s.substr(begin, end));
      break;
    } else {
      dirs.push_back(s.substr(begin, end - begin));
      begin = end + 1;
    }
  }
 
  m_temppath = s;
  m_tempfree = minFreeSpace;
  std::tie(m_tempdir, m_unusableDirWarnings) = m_lfs.findCachePath(dirs, minFreeSpace);
 
#if 0
  std::cerr /* edm::LogInfo("StorageFactory") */
    << "Using '" << m_tempdir << "' for temp dir"
    << std::endl;
#endif
}

References EcnaPython_AdcPeg12_S1_10_R170298_1_0_150_Dee0::cerr, submitPVResolutionJobs::count, heppy_check::dirs, mps_fire::end, LocalFileSystem::findCachePath(), m_lfs, m_tempdir, m_tempfree, m_temppath, m_unusableDirWarnings, and alignCSCRings::s.

Referenced by StorageFactory().

setTimeout()

void StorageFactory::setTimeout

(

unsigned int

timeout

)

Definition at line 48 of file StorageFactory.cc.

{ m_timeout = timeout; }

References m_timeout, and timeout().

stagein()

void StorageFactory::stagein

(

const std::string &

url

)

const

Definition at line 159 of file StorageFactory.cc.

                                                       {
  std::string protocol;
  std::string rest;
 
  std::unique_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker(url, protocol, rest)) {
    if (m_accounting) {
      auto token = StorageAccount::tokenForStorageClassName(protocol);
      stats =
          std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::stagein));
    }
    try {
      maker->stagein(protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
      if (stats)
        stats->tick();
    } catch (cms::Exception &err) {
      edm::LogWarning("StorageFactory::stagein()") << "Failed to stage in file '" << url << "' because:\n"
                                                   << err.explainSelf();
    }
  }
}

References StorageAccount::counter(), submitPVResolutionJobs::err, getMaker(), m_accounting, m_debugLevel, m_timeout, mergeAndRegister::rest, StorageMaker::AuxSettings::setDebugLevel(), StorageMaker::AuxSettings::setTimeout(), StorageAccount::stagein, dqmMemoryStats::stats, AlCaHLTBitMon_QueryRunRegistry::string, unpackBuffers-CaloStage2::token, StorageAccount::tokenForStorageClassName(), and relmon_authenticated_wget::url.

Referenced by edm::RootPrimaryFileSequence::RootPrimaryFileSequence(), and edm::RootSecondaryFileSequence::RootSecondaryFileSequence().

tempDir()

std::string StorageFactory::tempDir

(

void

)

const

Definition at line 90 of file StorageFactory.cc.

{ return m_tempdir; }

References m_tempdir.

tempMinFree()

double StorageFactory::tempMinFree

(

void

)

const

Definition at line 94 of file StorageFactory.cc.

{ return m_tempfree; }

References m_tempfree.

tempPath()

std::string StorageFactory::tempPath

(

void

)

const

Definition at line 92 of file StorageFactory.cc.

{ return m_temppath; }

References m_temppath.

timeout()

unsigned int StorageFactory::timeout

(

void

)

const

Definition at line 50 of file StorageFactory.cc.

{ return m_timeout; }

References m_timeout.

Referenced by setTimeout().

wrapNonLocalFile()

std::unique_ptr< Storage > StorageFactory::wrapNonLocalFile	(	std::unique_ptr< Storage >	s,
		const std::string &	proto,
		const std::string &	path,
		int	mode
	)		const

Definition at line 207 of file StorageFactory.cc.

                                                                          {
  StorageFactory::CacheHint hint = cacheHint();
  if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap)) {
    if (mode & IOFlags::OpenWrite) {
      // For now, issue no warning - otherwise, we'd always warn on output files.
    } else if (m_tempdir.empty()) {
      edm::LogWarning("StorageFactory") << m_unusableDirWarnings;
    } else if ((not path.empty()) and m_lfs.isLocalPath(path)) {
      // For now, issue no warning - otherwise, we'd always warn on local input files.
    } else {
      if (accounting()) {
        s = std::make_unique<StorageAccountProxy>(proto, std::move(s));
      }
      s = std::make_unique<LocalCacheFile>(std::move(s), m_tempdir);
    }
  }
 
  return s;
}

References accounting(), CACHE_HINT_LAZY_DOWNLOAD, cacheHint(), LocalFileSystem::isLocalPath(), m_lfs, m_tempdir, m_unusableDirWarnings, ALCARECOPromptCalibProdSiPixelAli0T_cff::mode, eostools::move(), IOFlags::OpenWrap, IOFlags::OpenWrite, castor_dqm_sourceclient_file_cfg::path, and alignCSCRings::s.

Member Data Documentation

m_accounting

bool StorageFactory::m_accounting

private

Definition at line 66 of file StorageFactory.h.

Referenced by accounting(), check(), enableAccounting(), open(), and stagein().

m_cacheHint

CacheHint StorageFactory::m_cacheHint

private

Definition at line 64 of file StorageFactory.h.

Referenced by cacheHint(), and setCacheHint().

m_debugLevel

unsigned int StorageFactory::m_debugLevel

private

Definition at line 72 of file StorageFactory.h.

Referenced by check(), debugLevel(), open(), setDebugLevel(), and stagein().

m_lfs

LocalFileSystem StorageFactory::m_lfs

private

Definition at line 73 of file StorageFactory.h.

Referenced by setTempDir(), and wrapNonLocalFile().

m_makers

MakerTable StorageFactory::m_makers

mutableprivate

Definition at line 63 of file StorageFactory.h.

Referenced by getMaker().

m_readHint

ReadHint StorageFactory::m_readHint

private

Definition at line 65 of file StorageFactory.h.

Referenced by readHint(), and setReadHint().

m_tempdir

std::string StorageFactory::m_tempdir

private

Definition at line 69 of file StorageFactory.h.

Referenced by setTempDir(), tempDir(), and wrapNonLocalFile().

m_tempfree

double StorageFactory::m_tempfree

private

Definition at line 67 of file StorageFactory.h.

Referenced by setTempDir(), StorageFactory(), and tempMinFree().

m_temppath

std::string StorageFactory::m_temppath

private

Definition at line 68 of file StorageFactory.h.

Referenced by setTempDir(), StorageFactory(), and tempPath().

m_timeout

unsigned int StorageFactory::m_timeout

private

Definition at line 71 of file StorageFactory.h.

Referenced by check(), open(), setTimeout(), stagein(), and timeout().

m_unusableDirWarnings

std::string StorageFactory::m_unusableDirWarnings

private

Definition at line 70 of file StorageFactory.h.

Referenced by setTempDir(), and wrapNonLocalFile().

s_instance

StorageFactory StorageFactory::s_instance

staticprivate

Definition at line 74 of file StorageFactory.h.

Referenced by get(), and getToModify().