StorageFactory Class Reference

#include <StorageFactory.h>

Public Types
enum	CacheHint { CACHE_HINT_APPLICATION, CACHE_HINT_STORAGE, CACHE_HINT_LAZY_DOWNLOAD, CACHE_HINT_AUTO_DETECT }
enum	ReadHint { READ_HINT_UNBUFFERED, READ_HINT_READAHEAD, READ_HINT_AUTO }

Public Member Functions
bool	accounting (void) const
CacheHint	cacheHint (void) const
bool	check (const std::string &url, IOOffset *size=0) const
unsigned int	debugLevel (void) const
bool	enableAccounting (bool enabled)
std::unique_ptr< Storage >	open (const std::string &url, int mode=IOFlags::OpenRead) const
ReadHint	readHint (void) const
void	setCacheHint (CacheHint value)
void	setDebugLevel (unsigned int level)
void	setReadHint (ReadHint value)
void	setTempDir (const std::string &s, double minFreeSpace)
void	setTimeout (unsigned int timeout)
void	stagein (const std::string &url) const
std::string	tempDir (void) const
double	tempMinFree (void) const
std::string	tempPath (void) const
unsigned int	timeout (void) const
std::unique_ptr< Storage >	wrapNonLocalFile (std::unique_ptr< Storage > s, const std::string &proto, const std::string &path, int mode) const
	~StorageFactory (void)

Static Public Member Functions
static const StorageFactory *	get (void)
static StorageFactory *	getToModify (void)

Private Types
typedef tbb::concurrent_unordered_map< std::string, std::shared_ptr< StorageMaker > >	MakerTable

Private Member Functions
StorageMaker *	getMaker (const std::string &proto) const
StorageMaker *	getMaker (const std::string &url, std::string &protocol, std::string &rest) const
	StorageFactory (void)

Private Attributes
bool	m_accounting
CacheHint	m_cacheHint
unsigned int	m_debugLevel
LocalFileSystem	m_lfs
MakerTable	m_makers
ReadHint	m_readHint
std::string	m_tempdir
double	m_tempfree
std::string	m_temppath
unsigned int	m_timeout
std::string	m_unusableDirWarnings

Static Private Attributes
static StorageFactory	s_instance

Detailed Description

Definition at line 13 of file StorageFactory.h.

Member Typedef Documentation

typedef tbb::concurrent_unordered_map<std::string, std::shared_ptr<StorageMaker> > StorageFactory::MakerTable

private

Definition at line 71 of file StorageFactory.h.

Member Enumeration Documentation

enum StorageFactory::CacheHint

Enumerator
CACHE_HINT_APPLICATION
CACHE_HINT_STORAGE
CACHE_HINT_LAZY_DOWNLOAD
CACHE_HINT_AUTO_DETECT

Definition at line 16 of file StorageFactory.h.

  {
    CACHE_HINT_APPLICATION,
    CACHE_HINT_STORAGE,
    CACHE_HINT_LAZY_DOWNLOAD,
    CACHE_HINT_AUTO_DETECT
  };

enum StorageFactory::ReadHint

Enumerator
READ_HINT_UNBUFFERED
READ_HINT_READAHEAD
READ_HINT_AUTO

Definition at line 24 of file StorageFactory.h.

  {
    READ_HINT_UNBUFFERED,
    READ_HINT_READAHEAD,
    READ_HINT_AUTO
  };

Constructor & Destructor Documentation

StorageFactory::~StorageFactory

(

void

)

Definition at line 25 of file StorageFactory.cc.

{
}

StorageFactory::StorageFactory ( void  )

private

Definition at line 13 of file StorageFactory.cc.

References m_tempfree, m_temppath, and setTempDir().

  : m_cacheHint(CACHE_HINT_AUTO_DETECT),
    m_readHint(READ_HINT_AUTO),
    m_accounting (false),
    m_tempfree (4.), // GB
    m_temppath (".:$TMPDIR"),
    m_timeout(0U),
    m_debugLevel(0U)
{
  setTempDir(m_temppath, m_tempfree);
}

Member Function Documentation

bool StorageFactory::accounting

(

void

)

const

Definition at line 46 of file StorageFactory.cc.

References m_accounting.

Referenced by wrapNonLocalFile().

{ return m_accounting; }

StorageFactory::CacheHint StorageFactory::cacheHint

(

void

)

const

Definition at line 54 of file StorageFactory.cc.

References m_cacheHint.

Referenced by LocalStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), XrdStorageMaker::open(), StormLcgGtStorageMaker::open(), TStorageFactoryFile::ReadBufferAsync(), and wrapNonLocalFile().

{ return m_cacheHint; }

bool StorageFactory::check	(	const std::string &	url,
		IOOffset *	size = 0
	)		const

Definition at line 239 of file StorageFactory.cc.

References StorageAccount::check, StorageAccount::counter(), cms::Exception::explainSelf(), getMaker(), m_accounting, m_debugLevel, m_timeout, StorageMaker::AuxSettings::setDebugLevel(), StorageMaker::AuxSettings::setTimeout(), findQualityFiles::size, AlCaHLTBitMon_QueryRunRegistry::string, and StorageAccount::tokenForStorageClassName().

Referenced by TStorageFactorySystem::AccessPathName(), and MatacqProducer::loadOrbitOffset().

{ 
  std::string protocol;
  std::string rest;

  bool ret = false;
  std::unique_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker (url, protocol, rest))
  {
    if (m_accounting) {
      auto token = StorageAccount::tokenForStorageClassName(protocol);
      stats.reset(new StorageAccount::Stamp(StorageAccount::counter (token, StorageAccount::Operation::check)));
    }
    try
    {
      ret = maker->check (protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size);
      if (stats) stats->tick();
    }
    catch (cms::Exception &err)
    {
      edm::LogWarning("StorageFactory::check()")
    << "Existence or size check for the file '" << url << "' failed because:\n"
    << err.explainSelf();
    }
  }
 
  return ret;
}

unsigned int StorageFactory::debugLevel

(

void

)

const

Definition at line 78 of file StorageFactory.cc.

References m_debugLevel.

{ return m_debugLevel; }

bool StorageFactory::enableAccounting

(

bool

enabled

)

Definition at line 38 of file StorageFactory.cc.

References patPFMETCorrections_cff::enabled, and m_accounting.

Referenced by PixelSLinkDataInputSource::PixelSLinkDataInputSource(), and TFileAdaptor::TFileAdaptor().

{
  bool old = m_accounting;
  m_accounting = enabled;
  return old;
}

const StorageFactory * StorageFactory::get ( void  )

static

Definition at line 30 of file StorageFactory.cc.

References s_instance.

Referenced by Options.Options::__getitem__(), betterConfigParser.BetterConfigParser::__updateDict(), TStorageFactorySystem::AccessPathName(), lhef::LHEReader::FileSource::FileSource(), betterConfigParser.BetterConfigParser::getCompares(), betterConfigParser.BetterConfigParser::getGeneral(), TStorageFactorySystem::GetPathInfo(), betterConfigParser.BetterConfigParser::getResultingSection(), TStorageFactoryFile::Initialize(), MatacqProducer::loadOrbitOffset(), GsiFTPStorageMaker::open(), HttpStorageMaker::open(), DavixStorageMaker::open(), LocalStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), XrdStorageMaker::open(), StormLcgGtStorageMaker::open(), edm::StreamerInputFile::openStreamerFile(), PixelSLinkDataInputSource::PixelSLinkDataInputSource(), TStorageFactoryFile::ReadBufferAsync(), edm::RootPrimaryFileSequence::RootPrimaryFileSequence(), edm::RootSecondaryFileSequence::RootSecondaryFileSequence(), and TStorageFactoryFile::SysOpen().

{ return &s_instance; }

StorageMaker * StorageFactory::getMaker ( const std::string &  proto ) const

private

Definition at line 134 of file StorageFactory.cc.

References edmplugin::standard::config(), edmplugin::PluginManager::configure(), reco::get(), instance, edmplugin::PluginManager::isAvailable(), and m_makers.

Referenced by check(), getMaker(), open(), and stagein().

{
  auto itFound = m_makers.find(proto);
  if(itFound != m_makers.end()) {
     return itFound->second.get();
  }
  if (! edmplugin::PluginManager::isAvailable()) {
    edmplugin::PluginManager::configure(edmplugin::standard::config());
  }
  std::shared_ptr<StorageMaker> instance{ StorageMakerFactory::get()->tryToCreate(proto)};
  auto insertResult = m_makers.insert(MakerTable::value_type(proto,instance));
  //Can't use instance since it is possible that another thread beat
  // us to the insertion so the map contains a different instance.
  return insertResult.first->second.get();
}

StorageMaker * StorageFactory::getMaker ( const std::string &  url, std::string &  protocol, std::string &  rest  ) const

private

Definition at line 151 of file StorageFactory.cc.

References getMaker(), AlCaHLTBitMon_ParallelJobs::p, and relmon_authenticated_wget::url.

{
  size_t p = url.find(':');
  if (p != std::string::npos)
  {
    protocol = url.substr(0,p);
    rest = url.substr(p+1);
  }
  else
  {
    protocol = "file"; 
    rest = url;
  }

  return getMaker (protocol);
}

StorageFactory * StorageFactory::getToModify ( void  )

static

Definition at line 34 of file StorageFactory.cc.

References s_instance.

Referenced by PixelSLinkDataInputSource::PixelSLinkDataInputSource(), and TFileAdaptor::TFileAdaptor().

{ return &s_instance; }

std::unique_ptr< Storage > StorageFactory::open	(	const std::string &	url,
		int	mode = IOFlags::OpenRead
	)		const

Definition at line 171 of file StorageFactory.cc.

References cms::Exception::addAdditionalInfo(), cms::Exception::addContext(), cms::Exception::clearMessage(), StorageAccount::counter(), getMaker(), m_accounting, m_debugLevel, m_timeout, cms::Exception::message(), eostools::move(), StorageAccount::open, StorageMaker::AuxSettings::setDebugLevel(), StorageMaker::AuxSettings::setTimeout(), AlCaHLTBitMon_QueryRunRegistry::string, and StorageAccount::tokenForStorageClassName().

Referenced by lhef::LHEReader::FileSource::FileSource(), MatacqProducer::loadOrbitOffset(), edm::StreamerInputFile::openStreamerFile(), and PixelSLinkDataInputSource::PixelSLinkDataInputSource().

{ 
  std::string protocol;
  std::string rest;
  std::unique_ptr<Storage> ret;
  std::unique_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker (url, protocol, rest))
  {
    if (m_accounting) {
        auto token = StorageAccount::tokenForStorageClassName(protocol);
        stats.reset(new StorageAccount::Stamp(StorageAccount::counter (token, StorageAccount::Operation::open)));
    }
    try
    {
      if (auto storage = maker->open (protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout)))
      {
    if (dynamic_cast<LocalCacheFile *>(storage.get()))
      protocol = "local-cache";

    if (m_accounting)
    ret = std::make_unique<StorageAccountProxy>(protocol, std::move(storage));
    else
    ret = std::move(storage);

        if (stats)
      stats->tick();
      }
    }
    catch (cms::Exception &err)
    {
      err.addContext("Calling StorageFactory::open()");
      err.addAdditionalInfo(err.message());
      err.clearMessage();
      err << "Failed to open the file '" << url << "'";
      throw;
    }
  } 
  return ret;
}

StorageFactory::ReadHint StorageFactory::readHint

(

void

)

const

Definition at line 62 of file StorageFactory.cc.

References m_readHint.

Referenced by LocalStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), XrdStorageMaker::open(), and StormLcgGtStorageMaker::open().

{ return m_readHint; }

void StorageFactory::setCacheHint

(

CacheHint

value

)

Definition at line 50 of file StorageFactory.cc.

References m_cacheHint, and relativeConstraints::value.

Referenced by TFileAdaptor::TFileAdaptor().

{ m_cacheHint = value; }

void StorageFactory::setDebugLevel

(

unsigned int

level

)

Definition at line 74 of file StorageFactory.cc.

References hcalDigis_cfi::level, and m_debugLevel.

Referenced by TFileAdaptor::TFileAdaptor().

{ m_debugLevel = level; }

void StorageFactory::setReadHint

(

ReadHint

value

)

Definition at line 58 of file StorageFactory.cc.

References m_readHint, and relativeConstraints::value.

Referenced by TFileAdaptor::TFileAdaptor().

{ m_readHint = value; }

void StorageFactory::setTempDir	(	const std::string &	s,
		double	minFreeSpace
	)

Definition at line 82 of file StorageFactory.cc.

References begin, MessageLogger_cfi::cerr, KineDebug3::count(), heppy_check::dirs, end, LocalFileSystem::findCachePath(), m_lfs, m_tempdir, m_tempfree, m_temppath, m_unusableDirWarnings, alignCSCRings::s, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by StorageFactory(), and TFileAdaptor::TFileAdaptor().

{
#if 0
  std::cerr /* edm::LogInfo("StorageFactory") */
    << "Considering path '" << s
    << "', min free space " << minFreeSpace
    << "GB for temp dir" << std::endl;
#endif

  size_t begin = 0;
  std::vector<std::string> dirs;
  dirs.reserve(std::count(s.begin(), s.end(), ':') + 1);

  while (true)
  {
    size_t end = s.find(':', begin);
    if (end == std::string::npos)
    {
      dirs.push_back(s.substr(begin, end));
      break;
    }
    else
    {
      dirs.push_back(s.substr(begin, end - begin));
      begin = end+1;
    }
  }

  m_temppath = s;
  m_tempfree = minFreeSpace;
  std::tie(m_tempdir, m_unusableDirWarnings) = m_lfs.findCachePath(dirs, minFreeSpace);

#if 0
  std::cerr /* edm::LogInfo("StorageFactory") */
    << "Using '" << m_tempdir << "' for temp dir"
    << std::endl;
#endif
}

void StorageFactory::setTimeout

(

unsigned int

timeout

)

Definition at line 66 of file StorageFactory.cc.

References m_timeout, and timeout().

Referenced by TFileAdaptor::TFileAdaptor().

{ m_timeout = timeout; }

void StorageFactory::stagein

(

const std::string &

url

)

const

Definition at line 212 of file StorageFactory.cc.

References StorageAccount::counter(), cms::Exception::explainSelf(), getMaker(), m_accounting, m_debugLevel, m_timeout, StorageMaker::AuxSettings::setDebugLevel(), StorageMaker::AuxSettings::setTimeout(), StorageAccount::stagein, AlCaHLTBitMon_QueryRunRegistry::string, and StorageAccount::tokenForStorageClassName().

Referenced by edm::RootPrimaryFileSequence::RootPrimaryFileSequence(), and edm::RootSecondaryFileSequence::RootSecondaryFileSequence().

{ 
  std::string protocol;
  std::string rest;

  std::unique_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker (url, protocol, rest))
  {
    if (m_accounting) {
      auto token = StorageAccount::tokenForStorageClassName(protocol);
      stats.reset(new StorageAccount::Stamp(StorageAccount::counter (token, StorageAccount::Operation::stagein)));
    }
    try
    {
      maker->stagein (protocol, rest,StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
      if (stats) stats->tick();
    }
    catch (cms::Exception &err)
    {
      edm::LogWarning("StorageFactory::stagein()")
    << "Failed to stage in file '" << url << "' because:\n"
    << err.explainSelf();
    }
  }
}

std::string StorageFactory::tempDir

(

void

)

const

Definition at line 122 of file StorageFactory.cc.

References m_tempdir, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by GsiFTPStorageMaker::open(), and HttpStorageMaker::open().

{ return m_tempdir; }

double StorageFactory::tempMinFree

(

void

)

const

Definition at line 130 of file StorageFactory.cc.

References m_tempfree.

Referenced by TFileAdaptor::TFileAdaptor().

{ return m_tempfree; }

std::string StorageFactory::tempPath

(

void

)

const

Definition at line 126 of file StorageFactory.cc.

References m_temppath.

Referenced by TFileAdaptor::TFileAdaptor().

{ return m_temppath; }

unsigned int StorageFactory::timeout

(

void

)

const

Definition at line 70 of file StorageFactory.cc.

References m_timeout.

Referenced by setTimeout().

{ return m_timeout; }

std::unique_ptr< Storage > StorageFactory::wrapNonLocalFile	(	std::unique_ptr< Storage >	s,
		const std::string &	proto,
		const std::string &	path,
		int	mode
	)		const

Definition at line 269 of file StorageFactory.cc.

References accounting(), CACHE_HINT_LAZY_DOWNLOAD, cacheHint(), LocalFileSystem::isLocalPath(), m_lfs, m_tempdir, m_unusableDirWarnings, eostools::move(), IOFlags::OpenWrap, IOFlags::OpenWrite, and alignCSCRings::s.

Referenced by DavixStorageMaker::open(), LocalStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), XrdStorageMaker::open(), and StormLcgGtStorageMaker::open().

{
  StorageFactory::CacheHint hint = cacheHint();
  if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap))
  {
      if (mode & IOFlags::OpenWrite)
      {
        // For now, issue no warning - otherwise, we'd always warn on output files.
      }
      else if (m_tempdir.empty())
      {
        edm::LogWarning("StorageFactory") << m_unusableDirWarnings;
      }
      else if ( (not path.empty()) and m_lfs.isLocalPath(path))
      {
        // For now, issue no warning - otherwise, we'd always warn on local input files.
      }
      else
      {
        if (accounting()) {s = std::make_unique<StorageAccountProxy>(proto, std::move(s));}
        s = std::make_unique<LocalCacheFile>(std::move(s), m_tempdir);
      }
  }

  return s;
}

Member Data Documentation

bool StorageFactory::m_accounting

private

Definition at line 82 of file StorageFactory.h.

Referenced by accounting(), check(), enableAccounting(), open(), and stagein().

CacheHint StorageFactory::m_cacheHint

private

Definition at line 80 of file StorageFactory.h.

Referenced by cacheHint(), and setCacheHint().

unsigned int StorageFactory::m_debugLevel

private

Definition at line 88 of file StorageFactory.h.

Referenced by check(), debugLevel(), open(), setDebugLevel(), and stagein().

LocalFileSystem StorageFactory::m_lfs

private

Definition at line 89 of file StorageFactory.h.

Referenced by setTempDir(), and wrapNonLocalFile().

MakerTable StorageFactory::m_makers

mutableprivate

Definition at line 79 of file StorageFactory.h.

Referenced by getMaker().

ReadHint StorageFactory::m_readHint

private

Definition at line 81 of file StorageFactory.h.

Referenced by readHint(), and setReadHint().

std::string StorageFactory::m_tempdir

private

Definition at line 85 of file StorageFactory.h.

Referenced by setTempDir(), tempDir(), and wrapNonLocalFile().

double StorageFactory::m_tempfree

private

Definition at line 83 of file StorageFactory.h.

Referenced by setTempDir(), StorageFactory(), and tempMinFree().

std::string StorageFactory::m_temppath

private

Definition at line 84 of file StorageFactory.h.

Referenced by setTempDir(), StorageFactory(), and tempPath().

unsigned int StorageFactory::m_timeout

private

Definition at line 87 of file StorageFactory.h.

Referenced by check(), open(), setTimeout(), stagein(), and timeout().

std::string StorageFactory::m_unusableDirWarnings

private

Definition at line 86 of file StorageFactory.h.

Referenced by setTempDir(), and wrapNonLocalFile().

StorageFactory StorageFactory::s_instance

staticprivate

Definition at line 90 of file StorageFactory.h.

Referenced by get(), and getToModify().