#include <TStorageFactoryFile.h>

Inheritance diagram for TStorageFactoryFile:

Public Member Functions
	ClassDef (TStorageFactoryFile, 0)

virtual Bool_t	ReadBuffer (char *buf, Int_t len)

virtual Bool_t	ReadBuffer (char *buf, Long64_t pos, Int_t len)

virtual Bool_t	ReadBufferAsync (Long64_t off, Int_t len)

virtual Bool_t	ReadBuffers (char buf, Long64_t pos, Int_t *len, Int_t nbuf)

void	ResetErrno (void) const

	TStorageFactoryFile (const char name, Option_t option, const char *ftitle, Int_t compress, Int_t netopt, Bool_t parallelopen=kFALSE)

	TStorageFactoryFile (const char name, Option_t option="", const char *ftitle="", Int_t compress=1)

virtual Bool_t	WriteBuffer (const char *buf, Int_t len)

	~TStorageFactoryFile (void)

Protected Member Functions
virtual Int_t	SysClose (Int_t fd)

virtual Int_t	SysOpen (const char *pathname, Int_t flags, UInt_t mode)

virtual Long64_t	SysSeek (Int_t fd, Long64_t offset, Int_t whence)

virtual Int_t	SysStat (Int_t fd, Long_t id, Long64_t size, Long_t flags, Long_t modtime)

virtual Int_t	SysSync (Int_t fd)

Private Member Functions
void	Initialize (const char name, Option_t option="")

Bool_t	ReadBuffersSync (char buf, Long64_t pos, Int_t *len, Int_t nbuf)

	TStorageFactoryFile (void)

Private Attributes
std::unique_ptr< Storage >	storage_

Detailed Description

TFile wrapper around StorageFactory and Storage.

Definition at line 15 of file TStorageFactoryFile.h.

Constructor & Destructor Documentation

TStorageFactoryFile::TStorageFactoryFile	(	const char *	name,
		Option_t *	option,
		const char *	ftitle,
		Int_t	compress,
		Int_t	netopt,
		Bool_t	parallelopen = `kFALSE`
	)

Definition at line 108 of file TStorageFactoryFile.cc.

References Initialize(), and edm::threadLocalException::setException().

   : TFile(path, "NET", ftitle, compress), // Pass "NET" to prevent local access in base class
     storage_()
 {
   try {
     Initialize(path, option);
   } catch (...) {
     edm::threadLocalException::setException(std::current_exception()); // capture
   }
 }

TStorageFactoryFile::TStorageFactoryFile	(	const char *	name,
		Option_t *	option = `""`,
		const char *	ftitle = `""`,
		Int_t	compress = `1`
	)

Definition at line 124 of file TStorageFactoryFile.cc.

References Initialize(), and edm::threadLocalException::setException().

   : TFile(path, "NET", ftitle, compress), // Pass "NET" to prevent local access in base class
     storage_()
 {
   try {
     Initialize(path, option);
   } catch (...) {
     edm::threadLocalException::setException(std::current_exception()); // capture
   }
 }

TStorageFactoryFile::~TStorageFactoryFile ( void )

Definition at line 225 of file TStorageFactoryFile.cc.

 {
   Close();
 }

TStorageFactoryFile::TStorageFactoryFile ( void )

private

Definition at line 97 of file TStorageFactoryFile.cc.

References StorageAccount::construct, s_statsCtor, storageCounter(), and StorageAccount::Stamp::tick().

   : storage_()
 {
   StorageAccount::Stamp stats(storageCounter(s_statsCtor, StorageAccount::Operation::construct));
   stats.tick(0);
 }

Member Function Documentation

TStorageFactoryFile::ClassDef	(	TStorageFactoryFile	,
		0
	)

void TStorageFactoryFile::Initialize	(	const char *	name,
		Option_t *	option = `""`
	)

private

Definition at line 139 of file TStorageFactoryFile.cc.

References assert(), edm::Exception::categoryCode(), StorageAccount::construct, SurfaceDeformationFactory::create(), alignCSCRings::e, Exception, StorageFactory::get(), edm::Service< T >::isAvailable(), edm::errors::NotFound, IOFlags::OpenCreate, IOFlags::OpenRead, IOFlags::OpenWrap, IOFlags::OpenWrite, fed_dqm_sourceclient-live_cfg::path, SiPixelLorentzAngle_cfi::read, s_statsCtor, edm::storage::StatisticsSenderService::setSize(), statsService, storage_, storageCounter(), StorageAccount::Stamp::tick(), and update.

Referenced by TStorageFactoryFile().

 {
   StorageAccount::Stamp stats(storageCounter(s_statsCtor, StorageAccount::Operation::construct));
 
   // Enable AsyncReading.
   // This was the default for 5.27, but turned off by default for 5.32.
   // In our testing, AsyncReading is the fastest mechanism available.
   // In 5.32, the AsyncPrefetching mechanism is preferred, but has been a
   // performance hit in our "average case" tests.
   gEnv->SetValue("TFile.AsyncReading", 1);
 
   // Parse options; at the moment we only accept read!
   fOption = option;
   fOption.ToUpper();
 
   if (fOption == "NEW")
     fOption = "CREATE";
 
   Bool_t create   = (fOption == "CREATE");
   Bool_t recreate = (fOption == "RECREATE");
   Bool_t update   = (fOption == "UPDATE");
   Bool_t read     = (fOption == "READ") || (fOption == "READWRAP");
   Bool_t readwrap = (fOption == "READWRAP");
 
   if (!create && !recreate && !update && !read)
   {
     read = true;
     fOption = "READ";
   }
 
   if (recreate)
   {
     if (!gSystem->AccessPathName(path, kFileExists))
       gSystem->Unlink(path);
 
     recreate = false;
     create   = true;
     fOption  = "CREATE";
   }
   assert(!recreate);
 
   if (update && gSystem->AccessPathName(path, kFileExists))
   {
     update = kFALSE;
     create = kTRUE;
   }
 
   assert(read || update || create);
 
   int           openFlags = IOFlags::OpenRead;
   if (!read)    openFlags |= IOFlags::OpenWrite;
   if (create)   openFlags |= IOFlags::OpenCreate;
   //if (recreate) openFlags |= IOFlags::OpenCreate | IOFlags::OpenTruncate;
   if (readwrap) openFlags |= IOFlags::OpenWrap;
 
   // Open storage
   if (! (storage_ = StorageFactory::get()->open(path, openFlags)))
   {
      MakeZombie();
      gDirectory = gROOT;
      throw cms::Exception("TStorageFactoryFile::TStorageFactoryFile()")
        << "Cannot open file '" << path << "'";
   }
 
   // Record the statistics.
   try {
     edm::Service<edm::storage::StatisticsSenderService> statsService;
     if (statsService.isAvailable()) {
       statsService->setSize(storage_->size());
     }
   } catch (edm::Exception e) {
     if (e.categoryCode() != edm::errors::NotFound) {
       throw;
     }
   }
 
   fRealName = path;
   fD = 0; // sorry, meaningless
   fWritable = read ? kFALSE : kTRUE;
 
   Init(create);
 
   stats.tick(0);
 }

Bool_t TStorageFactoryFile::ReadBuffer	(	char *	buf,
		Int_t	len
	)

virtual

Definition at line 244 of file TStorageFactoryFile.cc.

References EnergyCorrector::c, gen::n, StorageAccount::read, StorageAccount::readActual, StorageAccount::readPrefetchToCache, StorageAccount::readViaCache, s_statsCPrefetch, s_statsCRead, s_statsRead, s_statsXRead, storage_, storageCounter(), and StorageAccount::Stamp::tick().

Referenced by ReadBuffer().

 {
   // Check that it's valid to access this file.
   if (IsZombie())
   {
     Error("ReadBuffer", "Cannot read from a zombie file");
     return kTRUE;
   }
 
   if (! IsOpen())
   {
     Error("ReadBuffer", "Cannot read from a file that is not open");
     return kTRUE;
   }
 
   // Read specified byte range from the storage.  Returns kTRUE in
   // case of error.  Note that ROOT uses this function recursively
   // to fill the cache; we use a flag to make sure our accounting
   // is reflected in a comprehensible manner.  The "read" counter
   // will include both, "readc" indicates how much read from the
   // cache, "readu" indicates how much we failed to read from the
   // cache (excluding those recursive reads), and "readx" counts
   // the amount actually passed to read from the storage object.
   StorageAccount::Stamp stats(storageCounter(s_statsRead, StorageAccount::Operation::read));
 
   // If we have a cache, read from there first.  This returns 0
   // if the block hasn't been prefetched, 1 if it was in cache,
   // and 2 if there was an error.
   if (TFileCacheRead *c = GetCacheRead())
   {
     Long64_t here = GetRelOffset();
     Bool_t   async = c->IsAsyncReading();
 
     StorageAccount::Stamp cstats(async
                                  ? storageCounter(s_statsCPrefetch, StorageAccount::Operation::readPrefetchToCache)
                                  : storageCounter(s_statsCRead, StorageAccount::Operation::readViaCache));
 
     Int_t st = ReadBufferViaCache(async ? 0 : buf, len);
 
     if (st == 2) {
       return kTRUE;
     }
 
     if (st == 1) {
       if (async) {
         cstats.tick(len);
         Seek(here);
       } else {
         cstats.tick(len);
         stats.tick(len);
         return kFALSE;
       }
     }
   }
 
   // FIXME: Re-enable read-ahead if the data wasn't in cache.
   // if (! st) storage_->caching(true, -1, s_readahead);
 
   // A real read
   StorageAccount::Stamp xstats(storageCounter(s_statsXRead, StorageAccount::Operation::readActual));
   IOSize n = storage_->xread(buf, len);
   xstats.tick(n);
   stats.tick(n);
   return n ? kFALSE : kTRUE;
 }

Bool_t TStorageFactoryFile::ReadBuffer	(	char *	buf,
		Long64_t	pos,
		Int_t	len
	)

virtual

Definition at line 235 of file TStorageFactoryFile.cc.

References ReadBuffer().

 {
   // This function needs to be optimized to minimize seeks.
   // See TFile::ReadBuffer(char *buf, Long64_t pos, Int_t len) in ROOT 5.27.06.
   Seek(pos);
   return ReadBuffer(buf, len);
 }

Bool_t TStorageFactoryFile::ReadBufferAsync	(	Long64_t	off,
		Int_t	len
	)

virtual

Definition at line 311 of file TStorageFactoryFile.cc.

References StorageFactory::CACHE_HINT_APPLICATION, StorageFactory::CACHE_HINT_STORAGE, StorageFactory::cacheHint(), f, StorageFactory::get(), o2o::iov, PREFETCH_PROBE_LENGTH, StorageAccount::readAsync, s_statsARead, storage_, storageCounter(), and StorageAccount::Stamp::tick().

 {
   // Check that it's valid to access this file.
   if (IsZombie())
   {
     Error("ReadBufferAsync", "Cannot read from a zombie file");
     return kTRUE;
   }
 
   if (! IsOpen())
   {
     Error("ReadBufferAsync", "Cannot read from a file that is not open");
     return kTRUE;
   }
 
   StorageAccount::Stamp stats(storageCounter(s_statsARead, StorageAccount::Operation::readAsync));
 
   // If asynchronous reading is disabled, bail out now, regardless
   // whether the underlying storage supports prefetching.  If it is
   // forced on, pretend it's on, even if the storage doesn't support
   // it, as this turns off the caching in ROOT's side.
   const StorageFactory *f = StorageFactory::get();
 
   // Verify that we never using async reads in app-only mode
   if (f->cacheHint() == StorageFactory::CACHE_HINT_APPLICATION)
     return kTRUE;
 
   // Let the I/O method indicate if it can do client-side prefetch.
   // If it does, then for example TTreeCache will drop its own cache
   // and will use the client-side cache of the actual I/O layer.
   // If len is zero ROOT is probing for prefetch support.
   if (len) {
     // FIXME: Synchronise caching.
     // storage_->caching(true, -1, 0);
     ;
   }
 
   IOPosBuffer iov(off, (void *) 0, len ? len : PREFETCH_PROBE_LENGTH);
   if (storage_->prefetch(&iov, 1))
   {
     stats.tick(len);
     return kFALSE;
   }
 
   // Always ask ROOT to use async reads in storage-only mode,
   // regardless of whether the storage system supports it.
   if (f->cacheHint() == StorageFactory::CACHE_HINT_STORAGE)
     return kFALSE;
 
   // Prefetching not available right now.
   return kTRUE;
 }

Bool_t TStorageFactoryFile::ReadBuffers	(	char *	buf,
		Long64_t *	pos,
		Int_t *	len,
		Int_t	nbuf
	)

virtual

Definition at line 432 of file TStorageFactoryFile.cc.

References i, o2o::iov, StorageAccount::readAsync, ReadBuffersSync(), s_statsARead, storage_, storageCounter(), summarizeEdmComparisonLogfiles::success, StorageAccount::Stamp::tick(), and pileupDistInMC::total.

 {
   // Check that it's valid to access this file.
   if (IsZombie())
   {
     Error("ReadBuffers", "Cannot read from a zombie file");
     return kTRUE;
   }
 
   if (! IsOpen())
   {
     Error("ReadBuffers", "Cannot read from a file that is not open");
     return kTRUE;
   }
 
   // For synchronous reads, we have special logic to optimize the I/O requests
   // from ROOT before handing it to the storage.
   if (buf)
   {
     return ReadBuffersSync(buf, pos, len, nbuf);
   }
   // For an async read, we assume the storage system is smart enough to do the
   // optimization itself.
 
   // Read from underlying storage.
   void* const nobuf = 0;
   Int_t total = 0;
   std::vector<IOPosBuffer> iov;
   iov.reserve(nbuf);
   for (Int_t i = 0; i < nbuf; ++i)
   {
     iov.push_back(IOPosBuffer(pos[i], nobuf, len[i]));
     total += len[i];
   }
 
   // Null buffer means asynchronous reads into I/O system's cache.
   bool success;
   StorageAccount::Stamp astats(storageCounter(s_statsARead, StorageAccount::Operation::readAsync));
   // Synchronise low-level cache with the supposed cache in TFile.
   // storage_->caching(true, -1, 0);
   success = storage_->prefetch(&iov[0], nbuf);
   astats.tick(total);
 
   // If it didn't suceeed, pass down to the base class.
   return success ? kFALSE : TFile::ReadBuffers(buf, pos, len, nbuf);
 }

Bool_t TStorageFactoryFile::ReadBuffersSync	(	char *	buf,
		Long64_t *	pos,
		Int_t *	len,
		Int_t	nbuf
	)

private

Most storage systems are not prepared for the onslaught of small reads that ROOT will perform, even if they implement a vectored read interface.

Typically, on the server side, the loop is unrolled and the reads are issued sequentially - giving the OS no hint that you're about to read a very close-by byte in the near future. Normally, OS read-ahead takes care of such situations; because the storage server has so many clients, and ROOT reads look random to the OS, the read-ahead becomes disabled.

Hence, this function will repack the application-layer request into an optimized storage-layer request. The resulting request to the storage layer typically has a slightly larger number of bytes, but far less individual reads.

On average, the server's disks see a smaller number of overall reads, the number of bytes transferred over the network increases modestly (around 10%), and the single application request becomes one-to-two I/O transactions. A clear win for all cases except high-latency WAN.

Definition at line 365 of file TStorageFactoryFile.cc.

References assert(), ReadRepacker::bufferUsed(), i, ReadRepacker::iov(), o2o::iov, ReadRepacker::pack(), StorageAccount::readActual, ReadRepacker::realBytesProcessed(), query::result, s_statsXRead, storage_, storageCounter(), StorageAccount::Stamp::tick(), and ReadRepacker::unpack().

Referenced by ReadBuffers().

 {
   Int_t remaining = nbuf; // Number of read requests left to process.
   Int_t pack_count; // Number of read requests processed by this iteration.
 
   IOSize remaining_buffer_size=0;
   // Calculate the remaining buffer size for the ROOT-owned buffer by adding
   // the size of the various requests.
   for (Int_t i=0; i<nbuf; i++) remaining_buffer_size+=len[i];
 
   char     *current_buffer = buf;
   Long64_t *current_pos    = pos;
   Int_t    *current_len    = len;
 
   ReadRepacker repacker;
 
   while (remaining > 0) {
 
     pack_count = repacker.pack(static_cast<long long int *>(current_pos), current_len, remaining, current_buffer, remaining_buffer_size);
 
     int real_bytes_processed = repacker.realBytesProcessed();
     IOSize io_buffer_used = repacker.bufferUsed();
 
     // Issue readv, then unpack buffers.
     StorageAccount::Stamp xstats(storageCounter(s_statsXRead, StorageAccount::Operation::readActual));
     std::vector<IOPosBuffer> &iov = repacker.iov();
     IOSize result = storage_->readv(&iov[0], iov.size());
     if (result != io_buffer_used) {
       return kTRUE;
     }
     xstats.tick(io_buffer_used);
     repacker.unpack(current_buffer);
 
     // Update the location of the unused part of the input buffer.
     remaining_buffer_size -= real_bytes_processed;
     current_buffer += real_bytes_processed;
 
     current_pos += pack_count;
     current_len += pack_count;
     remaining   -= pack_count;
 
   }
   assert(remaining_buffer_size == 0);
   return kFALSE;
 }

void TStorageFactoryFile::ResetErrno ( void ) const

Definition at line 625 of file TStorageFactoryFile.cc.

 {
   TSystem::ResetErrno();
 }

Int_t TStorageFactoryFile::SysClose ( Int_t fd )

protectedvirtual

Definition at line 574 of file TStorageFactoryFile.cc.

References StorageAccount::close, s_statsClose, storage_, storageCounter(), and StorageAccount::Stamp::tick().

 {
   StorageAccount::Stamp stats(storageCounter(s_statsClose, StorageAccount::Operation::close));
 
   if (storage_)
   {
     storage_->close();
     storage_.release();
   }
 
   stats.tick();
   return 0;
 }

Int_t TStorageFactoryFile::SysOpen	(	const char *	pathname,
		Int_t	flags,
		UInt_t	mode
	)

protectedvirtual

Definition at line 543 of file TStorageFactoryFile.cc.

References Exception, StorageFactory::get(), O_NONBLOCK, StorageAccount::open, IOFlags::OpenAppend, IOFlags::OpenCreate, IOFlags::OpenExclusive, IOFlags::OpenNonBlock, IOFlags::OpenRead, IOFlags::OpenTruncate, IOFlags::OpenWrite, s_statsOpen, storage_, storageCounter(), and StorageAccount::Stamp::tick().

 {
   StorageAccount::Stamp stats(storageCounter(s_statsOpen, StorageAccount::Operation::open));
 
   if (storage_)
   {
     storage_->close();
   }
 
   int                      openFlags = IOFlags::OpenRead;
   if (flags & O_WRONLY)    openFlags = IOFlags::OpenWrite;
   else if (flags & O_RDWR) openFlags |= IOFlags::OpenWrite;
   if (flags & O_CREAT)     openFlags |= IOFlags::OpenCreate;
   if (flags & O_APPEND)    openFlags |= IOFlags::OpenAppend;
   if (flags & O_EXCL)      openFlags |= IOFlags::OpenExclusive;
   if (flags & O_TRUNC)     openFlags |= IOFlags::OpenTruncate;
   if (flags & O_NONBLOCK)  openFlags |= IOFlags::OpenNonBlock;
 
   if (! (storage_ = StorageFactory::get()->open(pathname, openFlags)))
   {
      MakeZombie();
      gDirectory = gROOT;
      throw cms::Exception("TStorageFactoryFile::SysOpen()")
        << "Cannot open file '" << pathname << "'";
   }
 
   stats.tick();
   return 0;
 }

Long64_t TStorageFactoryFile::SysSeek	(	Int_t	fd,
		Long64_t	offset,
		Int_t	whence
	)

protectedvirtual

Definition at line 589 of file TStorageFactoryFile.cc.

References Storage::CURRENT, Storage::END, hltrates_dqm_sourceclient-live_cfg::offset, s_statsSeek, StorageAccount::seek, Storage::SET, storage_, storageCounter(), and StorageAccount::Stamp::tick().

 {
   StorageAccount::Stamp stats(storageCounter(s_statsSeek, StorageAccount::Operation::seek));
   Storage::Relative rel = (whence == SEEK_SET ? Storage::SET
                                : whence == SEEK_CUR ? Storage::CURRENT
                                : Storage::END);
 
   offset = storage_->position(offset, rel);
   stats.tick();
   return offset;
 }

Int_t TStorageFactoryFile::SysStat	(	Int_t	fd,
		Long_t *	id,
		Long64_t *	size,
		Long_t *	flags,
		Long_t *	modtime
	)

protectedvirtual

Definition at line 611 of file TStorageFactoryFile.cc.

References s_statsStat, StorageAccount::stat, storage_, storageCounter(), and StorageAccount::Stamp::tick().

 {
   StorageAccount::Stamp stats(storageCounter(s_statsStat, StorageAccount::Operation::stat));
   // FIXME: Most of this is unsupported or makes no sense with Storage
   *id = ::Hash(fRealName);
   *size = storage_->size();
   *flags = 0;
   *modtime = 0;
   stats.tick();
   return 0;
 }

Int_t TStorageFactoryFile::SysSync ( Int_t fd )

protectedvirtual

Definition at line 602 of file TStorageFactoryFile.cc.

References StorageAccount::flush, s_statsFlush, storage_, storageCounter(), and StorageAccount::Stamp::tick().

 {
   StorageAccount::Stamp stats(storageCounter(s_statsFlush, StorageAccount::Operation::flush));
   storage_->flush();
   stats.tick();
   return 0;
 }

Bool_t TStorageFactoryFile::WriteBuffer	(	const char *	buf,
		Int_t	len
	)

virtual

Definition at line 480 of file TStorageFactoryFile.cc.

References gen::n, s_statsCWrite, s_statsWrite, s_statsXWrite, storage_, storageCounter(), StorageAccount::Stamp::tick(), StorageAccount::write, StorageAccount::writeActual, and StorageAccount::writeViaCache.

 {
   // Check that it's valid to access this file.
   if (IsZombie())
   {
     Error("WriteBuffer", "Cannot write to a zombie file");
     return kTRUE;
   }
 
   if (! IsOpen())
   {
     Error("WriteBuffer", "Cannot write to a file that is not open");
     return kTRUE;
   }
 
   if (! fWritable)
   {
     Error("WriteBuffer", "File is not writable");
     return kTRUE;
   }
 
   StorageAccount::Stamp stats(storageCounter(s_statsWrite, StorageAccount::Operation::write));
   StorageAccount::Stamp cstats(storageCounter(s_statsCWrite, StorageAccount::Operation::writeViaCache));
 
   // Try first writing via a cache, and if that's not possible, directly.
   Int_t st;
   switch ((st = WriteBufferViaCache(buf, len)))
   {
   case 0:
     // Actual write.
     {
       StorageAccount::Stamp xstats(storageCounter(s_statsXWrite, StorageAccount::Operation::writeActual));
       IOSize n = storage_->xwrite(buf, len);
       xstats.tick(n);
       stats.tick(n);
 
       // FIXME: What if it's a short write?
       return n > 0 ? kFALSE : kTRUE;
     }
 
   case 1:
     cstats.tick(len);
     stats.tick(len);
     return kFALSE;
 
   case 2:
   default:
     Error("WriteBuffer", "Error writing to cache");
     return kTRUE;
   }
 }

Member Data Documentation

std::unique_ptr<Storage> TStorageFactoryFile::storage_

private

Definition at line 57 of file TStorageFactoryFile.h.

Referenced by Initialize(), ReadBuffer(), ReadBufferAsync(), ReadBuffers(), ReadBuffersSync(), SysClose(), SysOpen(), SysSeek(), SysStat(), SysSync(), and WriteBuffer().

Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation