StorageFactory Class Reference

#include <StorageFactory.h>

Public Types
enum	CacheHint { CACHE_HINT_APPLICATION, CACHE_HINT_STORAGE, CACHE_HINT_LAZY_DOWNLOAD, CACHE_HINT_AUTO_DETECT }
enum	ReadHint { READ_HINT_UNBUFFERED, READ_HINT_READAHEAD, READ_HINT_AUTO }

Public Member Functions
bool	accounting (void) const
void	activateTimeout (const std::string &url)
CacheHint	cacheHint (void) const
bool	check (const std::string &url, IOOffset *size=0)
unsigned int	debugLevel (void) const
bool	enableAccounting (bool enabled)
Storage *	open (const std::string &url, int mode=IOFlags::OpenRead)
ReadHint	readHint (void) const
void	setCacheHint (CacheHint value)
void	setDebugLevel (unsigned int level)
void	setReadHint (ReadHint value)
void	setTempDir (const std::string &s, double minFreeSpace)
void	setTimeout (unsigned int timeout)
void	stagein (const std::string &url)
std::string	tempDir (void) const
double	tempMinFree (void) const
std::string	tempPath (void) const
unsigned int	timeout (void) const
Storage *	wrapNonLocalFile (Storage *s, const std::string &proto, const std::string &path, int mode)
	~StorageFactory (void)

Static Public Member Functions
static StorageFactory *	get (void)

Private Types
typedef tbb::concurrent_unordered_map < std::string, std::shared_ptr < StorageMaker > >	MakerTable

Private Member Functions
StorageMaker *	getMaker (const std::string &proto)
StorageMaker *	getMaker (const std::string &url, std::string &protocol, std::string &rest)
	StorageFactory (void)

Private Attributes
bool	m_accounting
CacheHint	m_cacheHint
unsigned int	m_debugLevel
LocalFileSystem	m_lfs
MakerTable	m_makers
ReadHint	m_readHint
std::string	m_tempdir
double	m_tempfree
std::string	m_temppath
unsigned int	m_timeout

Static Private Attributes
static StorageFactory	s_instance

Detailed Description

Definition at line 13 of file StorageFactory.h.

Member Typedef Documentation

typedef tbb::concurrent_unordered_map<std::string, std::shared_ptr<StorageMaker> > StorageFactory::MakerTable

private

Definition at line 70 of file StorageFactory.h.

Member Enumeration Documentation

enum StorageFactory::CacheHint

Enumerator
CACHE_HINT_APPLICATION
CACHE_HINT_STORAGE
CACHE_HINT_LAZY_DOWNLOAD
CACHE_HINT_AUTO_DETECT

Definition at line 16 of file StorageFactory.h.

  {
    CACHE_HINT_APPLICATION,
    CACHE_HINT_STORAGE,
    CACHE_HINT_LAZY_DOWNLOAD,
    CACHE_HINT_AUTO_DETECT
  };

enum StorageFactory::ReadHint

Enumerator
READ_HINT_UNBUFFERED
READ_HINT_READAHEAD
READ_HINT_AUTO

Definition at line 24 of file StorageFactory.h.

  {
    READ_HINT_UNBUFFERED,
    READ_HINT_READAHEAD,
    READ_HINT_AUTO
  };

Constructor & Destructor Documentation

StorageFactory::~StorageFactory

(

void

)

Definition at line 26 of file StorageFactory.cc.

{
}

StorageFactory::StorageFactory ( void  )

private

Definition at line 14 of file StorageFactory.cc.

References m_tempfree, m_temppath, and setTempDir().

  : m_cacheHint(CACHE_HINT_AUTO_DETECT),
    m_readHint(READ_HINT_AUTO),
    m_accounting (false),
    m_tempfree (4.), // GB
    m_temppath (".:$TMPDIR"),
    m_timeout(0U),
    m_debugLevel(0U)
{
  setTempDir(m_temppath, m_tempfree);
}

Member Function Documentation

bool StorageFactory::accounting

(

void

)

const

Definition at line 43 of file StorageFactory.cc.

References m_accounting.

Referenced by wrapNonLocalFile().

{ return m_accounting; }

void StorageFactory::activateTimeout

(

const std::string &

url

)

Definition at line 292 of file StorageFactory.cc.

References getMaker(), m_timeout, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by edm::RootInputFileSequence::initFile(), and edm::RootInputFileSequence::RootInputFileSequence().

{
  std::string protocol;
  std::string rest;

  if (StorageMaker *maker = getMaker (url, protocol, rest))
  {
    maker->setTimeout (m_timeout);
  }
}

StorageFactory::CacheHint StorageFactory::cacheHint

(

void

)

const

Definition at line 51 of file StorageFactory.cc.

References m_cacheHint.

Referenced by LocalStorageMaker::open(), XrdStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), StormLcgGtStorageMaker::open(), RFIOStorageMaker::open(), TStorageFactoryFile::ReadBufferAsync(), and wrapNonLocalFile().

{ return m_cacheHint; }

bool StorageFactory::check	(	const std::string &	url,
		IOOffset *	size = 0
	)

Definition at line 233 of file StorageFactory.cc.

References StorageAccount::counter(), cms::Exception::explainSelf(), getMaker(), m_accounting, run_regression::ret, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by TStorageFactorySystem::AccessPathName().

{ 
  std::string protocol;
  std::string rest;

  bool ret = false;
  boost::shared_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker (url, protocol, rest))
  {
    if (m_accounting) 
      stats.reset(new StorageAccount::Stamp(StorageAccount::counter (protocol, "check")));
    try
    {
      ret = maker->check (protocol, rest, size);
      if (stats) stats->tick();
    }
    catch (cms::Exception &err)
    {
      edm::LogWarning("StorageFactory::check()")
    << "Existence or size check for the file '" << url << "' failed because:\n"
    << err.explainSelf();
    }
  }
 
  return ret;
}

unsigned int StorageFactory::debugLevel

(

void

)

const

Definition at line 75 of file StorageFactory.cc.

References m_debugLevel.

{ return m_debugLevel; }

bool StorageFactory::enableAccounting

(

bool

enabled

)

Definition at line 35 of file StorageFactory.cc.

References m_accounting.

Referenced by PixelSLinkDataInputSource::PixelSLinkDataInputSource(), and TFileAdaptor::TFileAdaptor().

{
  bool old = m_accounting;
  m_accounting = enabled;
  return old;
}

StorageFactory * StorageFactory::get ( void  )

static

Definition at line 31 of file StorageFactory.cc.

References s_instance.

Referenced by Options.Options::__getitem__(), betterConfigParser.BetterConfigParser::__updateDict(), TStorageFactorySystem::AccessPathName(), lhef::LHEReader::FileSource::FileSource(), betterConfigParser.BetterConfigParser::getCompares(), betterConfigParser.BetterConfigParser::getGeneral(), TStorageFactorySystem::GetPathInfo(), betterConfigParser.BetterConfigParser::getResultingSection(), edm::RootInputFileSequence::initFile(), TStorageFactoryFile::Initialize(), GsiFTPStorageMaker::open(), HttpStorageMaker::open(), LocalStorageMaker::open(), XrdStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), StormLcgGtStorageMaker::open(), RFIOStorageMaker::open(), edm::StreamerInputFile::openStreamerFile(), PixelSLinkDataInputSource::PixelSLinkDataInputSource(), TStorageFactoryFile::ReadBufferAsync(), edm::RootInputFileSequence::RootInputFileSequence(), TStorageFactoryFile::SysOpen(), and TFileAdaptor::TFileAdaptor().

{ return &s_instance; }

StorageMaker * StorageFactory::getMaker ( const std::string &  proto )

private

Definition at line 131 of file StorageFactory.cc.

References edmplugin::standard::config(), edmplugin::PluginManager::configure(), reco::get(), instance, edmplugin::PluginManager::isAvailable(), and m_makers.

Referenced by activateTimeout(), check(), getMaker(), open(), and stagein().

{
  auto itFound = m_makers.find(proto);
  if(itFound != m_makers.end()) {
     return itFound->second.get();
  }
  if (! edmplugin::PluginManager::isAvailable()) {
    edmplugin::PluginManager::configure(edmplugin::standard::config());
  }
  std::shared_ptr<StorageMaker> instance{ StorageMakerFactory::get()->tryToCreate(proto)};
  auto insertResult = m_makers.insert(MakerTable::value_type(proto,instance));
  //Can't use instance since it is possible that another thread beat
  // us to the insertion so the map contains a different instance.
  return insertResult.first->second.get();
}

StorageMaker * StorageFactory::getMaker ( const std::string &  url, std::string &  protocol, std::string &  rest  )

private

Definition at line 148 of file StorageFactory.cc.

References getMaker(), AlCaHLTBitMon_ParallelJobs::p, and relmon_authenticated_wget::url.

{
  size_t p = url.find(':');
  if (p != std::string::npos)
  {
    protocol = url.substr(0,p);
    rest = url.substr(p+1);
  }
  else
  {
    protocol = "file"; 
    rest = url;
  }

  return getMaker (protocol);
}

Storage * StorageFactory::open	(	const std::string &	url,
		int	mode = IOFlags::OpenRead
	)

Definition at line 168 of file StorageFactory.cc.

References cms::Exception::addAdditionalInfo(), cms::Exception::addContext(), cms::Exception::clearMessage(), StorageAccount::counter(), getMaker(), m_accounting, m_debugLevel, cms::Exception::message(), run_regression::ret, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by lhef::LHEReader::FileSource::FileSource().

{ 
  std::string protocol;
  std::string rest;
  Storage *ret = 0;
  boost::shared_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker (url, protocol, rest))
  {
    maker->setDebugLevel(m_debugLevel);
    if (m_accounting) 
      stats.reset(new StorageAccount::Stamp(StorageAccount::counter (protocol, "open")));
    try
    {
      if (Storage *storage = maker->open (protocol, rest, mode))
      {
    if (dynamic_cast<LocalCacheFile *>(storage))
      protocol = "local-cache";

    if (m_accounting)
      ret = new StorageAccountProxy(protocol, storage);
    else
      ret = storage;

        if (stats)
      stats->tick();
      }
    }
    catch (cms::Exception &err)
    {
      err.addContext("Calling StorageFactory::open()");
      err.addAdditionalInfo(err.message());
      err.clearMessage();
      err << "Failed to open the file '" << url << "'";
      throw;
    }
  } 
  return ret;
}

StorageFactory::ReadHint StorageFactory::readHint

(

void

)

const

Definition at line 59 of file StorageFactory.cc.

References m_readHint.

Referenced by LocalStorageMaker::open(), XrdStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), StormLcgGtStorageMaker::open(), and RFIOStorageMaker::open().

{ return m_readHint; }

void StorageFactory::setCacheHint

(

CacheHint

value

)

Definition at line 47 of file StorageFactory.cc.

References m_cacheHint, and relativeConstraints::value.

Referenced by TFileAdaptor::TFileAdaptor().

{ m_cacheHint = value; }

void StorageFactory::setDebugLevel

(

unsigned int

level

)

Definition at line 71 of file StorageFactory.cc.

References testEve_cfg::level, and m_debugLevel.

Referenced by TFileAdaptor::TFileAdaptor().

{ m_debugLevel = level; }

void StorageFactory::setReadHint

(

ReadHint

value

)

Definition at line 55 of file StorageFactory.cc.

References m_readHint, and relativeConstraints::value.

Referenced by TFileAdaptor::TFileAdaptor().

{ m_readHint = value; }

void StorageFactory::setTempDir	(	const std::string &	s,
		double	minFreeSpace
	)

Definition at line 79 of file StorageFactory.cc.

References begin, dtNoiseDBValidation_cfg::cerr, prof2calltree::count, heppy_check::dirs, end, LocalFileSystem::findCachePath(), m_lfs, m_tempdir, m_tempfree, m_temppath, and alignCSCRings::s.

Referenced by StorageFactory(), and TFileAdaptor::TFileAdaptor().

{
#if 0
  std::cerr /* edm::LogInfo("StorageFactory") */
    << "Considering path '" << s
    << "', min free space " << minFreeSpace
    << "GB for temp dir" << std::endl;
#endif

  size_t begin = 0;
  std::vector<std::string> dirs;
  dirs.reserve(std::count(s.begin(), s.end(), ':') + 1);

  while (true)
  {
    size_t end = s.find(':', begin);
    if (end == std::string::npos)
    {
      dirs.push_back(s.substr(begin, end));
      break;
    }
    else
    {
      dirs.push_back(s.substr(begin, end - begin));
      begin = end+1;
    }
  }

  m_temppath = s;
  m_tempfree = minFreeSpace;
  m_tempdir = m_lfs.findCachePath(dirs, minFreeSpace);

#if 0
  std::cerr /* edm::LogInfo("StorageFactory") */
    << "Using '" << m_tempdir << "' for temp dir"
    << std::endl;
#endif
}

void StorageFactory::setTimeout

(

unsigned int

timeout

)

Definition at line 63 of file StorageFactory.cc.

References m_timeout, and timeout().

Referenced by TFileAdaptor::TFileAdaptor().

{ m_timeout = timeout; }

void StorageFactory::stagein

(

const std::string &

url

)

Definition at line 208 of file StorageFactory.cc.

References StorageAccount::counter(), cms::Exception::explainSelf(), getMaker(), m_accounting, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by edm::RootInputFileSequence::RootInputFileSequence().

{ 
  std::string protocol;
  std::string rest;

  boost::shared_ptr<StorageAccount::Stamp> stats;
  if (StorageMaker *maker = getMaker (url, protocol, rest))
  {
    if (m_accounting) 
      stats.reset(new StorageAccount::Stamp(StorageAccount::counter (protocol, "stagein")));
    try
    {
      maker->stagein (protocol, rest);
      if (stats) stats->tick();
    }
    catch (cms::Exception &err)
    {
      edm::LogWarning("StorageFactory::stagein()")
    << "Failed to stage in file '" << url << "' because:\n"
    << err.explainSelf();
    }
  }
}

std::string StorageFactory::tempDir

(

void

)

const

Definition at line 119 of file StorageFactory.cc.

References m_tempdir.

Referenced by GsiFTPStorageMaker::open(), and HttpStorageMaker::open().

{ return m_tempdir; }

double StorageFactory::tempMinFree

(

void

)

const

Definition at line 127 of file StorageFactory.cc.

References m_tempfree.

Referenced by TFileAdaptor::TFileAdaptor().

{ return m_tempfree; }

std::string StorageFactory::tempPath

(

void

)

const

Definition at line 123 of file StorageFactory.cc.

References m_temppath.

Referenced by TFileAdaptor::TFileAdaptor().

{ return m_temppath; }

unsigned int StorageFactory::timeout

(

void

)

const

Definition at line 67 of file StorageFactory.cc.

References m_timeout.

Referenced by setTimeout().

{ return m_timeout; }

Storage * StorageFactory::wrapNonLocalFile	(	Storage *	s,
		const std::string &	proto,
		const std::string &	path,
		int	mode
	)

Definition at line 261 of file StorageFactory.cc.

References accounting(), CACHE_HINT_LAZY_DOWNLOAD, cacheHint(), LocalFileSystem::isLocalPath(), LocalFileSystem::issueWarning(), m_lfs, m_tempdir, IOFlags::OpenWrap, IOFlags::OpenWrite, and alignCSCRings::s.

Referenced by LocalStorageMaker::open(), XrdStorageMaker::open(), DCacheStorageMaker::open(), StormStorageMaker::open(), StormLcgGtStorageMaker::open(), and RFIOStorageMaker::open().

{
  StorageFactory::CacheHint hint = cacheHint();
  if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap))
  {
      if (mode & IOFlags::OpenWrite)
      {
        // For now, issue no warning - otherwise, we'd always warn on output files.
      }
      else if (m_tempdir.empty())
      {
        m_lfs.issueWarning();
      }
      else if (path.empty() || m_lfs.isLocalPath(path))
      {
        // For now, issue no warning - otherwise, we'd always warn on local input files.
      }
      else
      {
        if (accounting()) {s = new StorageAccountProxy(proto, s);}
        s = new LocalCacheFile(s, m_tempdir);
      }
  }

  return s;
}

Member Data Documentation

bool StorageFactory::m_accounting

private

Definition at line 81 of file StorageFactory.h.

Referenced by accounting(), check(), enableAccounting(), open(), and stagein().

CacheHint StorageFactory::m_cacheHint

private

Definition at line 79 of file StorageFactory.h.

Referenced by cacheHint(), and setCacheHint().

unsigned int StorageFactory::m_debugLevel

private

Definition at line 86 of file StorageFactory.h.

Referenced by debugLevel(), open(), and setDebugLevel().

LocalFileSystem StorageFactory::m_lfs

private

Definition at line 87 of file StorageFactory.h.

Referenced by setTempDir(), and wrapNonLocalFile().

MakerTable StorageFactory::m_makers

private

Definition at line 78 of file StorageFactory.h.

Referenced by getMaker().

ReadHint StorageFactory::m_readHint

private

Definition at line 80 of file StorageFactory.h.

Referenced by readHint(), and setReadHint().

std::string StorageFactory::m_tempdir

private

Definition at line 84 of file StorageFactory.h.

Referenced by setTempDir(), tempDir(), and wrapNonLocalFile().

double StorageFactory::m_tempfree

private

Definition at line 82 of file StorageFactory.h.

Referenced by setTempDir(), StorageFactory(), and tempMinFree().

std::string StorageFactory::m_temppath

private

Definition at line 83 of file StorageFactory.h.

Referenced by setTempDir(), StorageFactory(), and tempPath().

unsigned int StorageFactory::m_timeout

private

Definition at line 85 of file StorageFactory.h.

Referenced by activateTimeout(), setTimeout(), and timeout().

StorageFactory StorageFactory::s_instance

staticprivate

Definition at line 88 of file StorageFactory.h.

Referenced by get().