#include <LocalFileSystem.h>

Classes
struct	FSInfo
	Information about file systems on this node. More...

Public Member Functions
std::pair< std::string, std::string >	findCachePath (const std::vector< std::string > &paths, double minFreeSpace) const

bool	isLocalPath (const std::string &path) const

	LocalFileSystem (void)

	~LocalFileSystem (void)

Private Member Functions
FSInfo *	findMount (const char path, struct statfs sfs, struct stat *s, std::vector< std::string > &) const

FSInfo *	initFSInfo (void *p)

int	initFSList (void)

	LocalFileSystem (LocalFileSystem &)=delete

void	operator= (LocalFileSystem &)=delete

int	readFSTypes (void)

int	statFSInfo (FSInfo *i) const

Private Attributes
std::vector< FSInfo * >	fs_

std::vector< std::string >	fstypes_

Detailed Description

Definition at line 11 of file LocalFileSystem.h.

Constructor & Destructor Documentation

LocalFileSystem::LocalFileSystem ( void )

Initialise local file system status.

Definition at line 592 of file LocalFileSystem.cc.

References initFSList(), and readFSTypes().

 {
   if (readFSTypes() < 0)
     return;
 
   if (initFSList() < 0)
     return;
 }

LocalFileSystem::~LocalFileSystem ( void )

Free local file system status resources.

Definition at line 602 of file LocalFileSystem.cc.

References MillePedeFileConverter_cfg::e, fs_, and mps_fire::i.

 {
   for (size_t i = 0, e = fs_.size(); i < e; ++i)
     free(fs_[i]);
 }

LocalFileSystem::LocalFileSystem ( LocalFileSystem & )

privatedelete

Member Function Documentation

std::pair< std::string, std::string > LocalFileSystem::findCachePath	(	const std::vector< std::string > &	paths,
		double	minFreeSpace
	)		const

Find a writeable directory among paths which is known to be local and has at least minFreeSpace amount of free space in gigabytes.

The paths should contain list of relative or absolute candidate directories. If an entry starts with letter "$" then the value of that environment variable is used instead; if the value is $TMPDIR and the environment variable is empty, "/tmp" is used instead.

Returns the first path in paths which satisfies the criteria, expanded to environment variable value if appropriate, resolved to full absolute path. If no suitable path can be found, returns an empty string.

Does not throw exceptions. If any serious errors occur, the errors are reported as message logger warnings but the actual error is swallowed and the directory concerned is skipped. Non-existent and inaccessible directories are silently ignored without warning.

Definition at line 486 of file LocalFileSystem.cc.

References MessageLogger_cfi::cerr, MillePedeFileConverter_cfg::e, findMount(), LocalFileSystem::FSInfo::freespc, mps_fire::i, LocalFileSystem::FSInfo::local, funct::m, eostools::move(), AlCaHLTBitMon_ParallelJobs::p, callgraph::path, mps_fire::result, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by StorageFactory::setTempDir().

 {
   struct stat s;
   struct statfs sfs;
   std::ostringstream warningst;
   warningst << "Cannot use lazy-download because:\n";
 
   for (size_t i = 0, e = paths.size(); i < e; ++i)
   {
     char *fullpath;
     const char *inpath = paths[i].c_str();
     const char *path = inpath;
 
     if (*path == '$')
     {
       char *p = getenv(path+1);
       if (p && *p)
     path = p;
       else if (! strcmp(path, "$TMPDIR"))
     path = "/tmp";
     }
 
     if (! (fullpath = realpath(path, nullptr)))
       fullpath = strdup(path);
 
 #if 0
     std::cerr /* edm::LogInfo("LocalFileSystem") */
       << "Checking if '" << fullpath << "', from '"
       << inpath << "' is valid cache path with "
       << minFreeSpace << " free space" << std::endl;
 #endif
 
     if (lstat(fullpath, &s) < 0)
     {
       int nerr = errno;
       if (nerr != ENOENT && nerr != EACCES)
         edm::LogWarning("LocalFileSystem::findCachePath()")
       << "Cannot lstat('" << fullpath << "', from '"
       << inpath << "'): " << strerror(nerr) << " (error "
       << nerr << ")";
       free(fullpath);
       continue;
     }
     
     if (statfs(fullpath, &sfs) < 0)
     {
       int nerr = errno;
       edm::LogWarning("LocalFileSystem::findCachePath()")
     << "Cannot statfs('" << fullpath << "', from '"
     << inpath << "'): " << strerror(nerr) << " (error "
     << nerr << ")";
       free(fullpath);
       continue;
     }
 
     std::vector<std::string> prev_paths;
     FSInfo *m = findMount(fullpath, &sfs, &s, prev_paths);
 #if 0
     std::cerr /* edm::LogInfo("LocalFileSystem") */
       << "Candidate '" << fullpath << "': "
       << "found=" << (m ? 1 : 0)
       << " local=" << (m && m->local)
       << " free=" << (m ? m->freespc : 0)
       << " access=" << access(fullpath, W_OK)
       << std::endl;
 #endif
 
     if (m
     && m->local
     && m->freespc >= minFreeSpace
     && access(fullpath, W_OK) == 0)
     {
       std::string result(fullpath);
       free(fullpath);
       return std::make_pair(result,std::string());
     }
     else if (m)
     {
       if (!m->local)
       {
         warningst << "- The mount " << fullpath << " is not local.\n";
       }
       else if (m->freespc < minFreeSpace)
       {
         warningst << " - The mount at " << fullpath << " has only " << m->freespc << " GB free; a minumum of " << minFreeSpace << " GB is required.\n";
       }
       else if (access(fullpath, W_OK))
       {
         warningst << " - The process has no permission to write into " << fullpath << "\n";
       }
     }
 
     free(fullpath);
   }
 
   std::string warning_str = warningst.str();
   if (!warning_str.empty())
   {
     warning_str = warning_str.substr(0, warning_str.size()-2);
   }
   
   return std::make_pair(std::string(), std::move(warning_str));
 }

LocalFileSystem::FSInfo * LocalFileSystem::findMount	(	const char *	path,
		struct statfs *	sfs,
		struct stat *	s,
		std::vector< std::string > &	prev_paths
	)		const

private

Find the file system path was mounted from. The statfs() and stat() information for path should be in sfs and s, respectively.

Finds currently mounted file system that path is owned by, and returns the FSInfo object for it, or null if no matching live file system can be found. If the return value is non-null, then it is guaranteed path was on that file system.

A null return value is possible for certain paths which are not on any mounted file system (e.g. /dev or /selinux), or if the file system is unavailable or some other way dysfunctional, such as dead nfs mount or filesystem does not implement statfs().

Definition at line 328 of file LocalFileSystem.cc.

References LocalFileSystem::FSInfo::bind, LocalFileSystem::FSInfo::dev, LocalFileSystem::FSInfo::dir, fs_, mps_fire::i, LocalFileSystem::FSInfo::local, LocalFileSystem::FSInfo::origin, callgraph::path, and statFSInfo().

Referenced by findCachePath(), and isLocalPath().

 {
   for (const auto & old_path : prev_paths)
   {
     if (!strcmp(old_path.c_str(), path))
     {
       edm::LogWarning("LocalFileSystem::findMount()")
         << "Found a loop in bind mounts; stopping evaluation.";
       return nullptr;
     }
   }
 
   FSInfo *best = nullptr;
   size_t bestlen = 0;
   size_t len = strlen(path);
   for (size_t i = 0; i < fs_.size(); ++i)
   {
     // First match simply against the file system path.  We don't
     // touch the file system until the path prefix matches.
     // When we have a path prefix match, check the file system if
     //   we don't have a best match candidate yet, OR
     //   this match is longer (more specific) than the previous best OR
     //   this match is the same length and the previous best isn't local
     // The final condition handles cases such as '/' that can appear twice
     // in the file system list, once as 'rootfs' and once as local fs.
     size_t fslen = strlen(fs_[i]->dir);
     if (! strncmp(fs_[i]->dir, path, fslen)
     && ((fslen == 1 && fs_[i]->dir[0] == '/')
         || len == fslen || path[fslen] == '/')
     && (! best || fslen > bestlen || (fslen == bestlen && !best->local)))
     {
       // Get the file system device and file system ids.
       if (statFSInfo(fs_[i]) < 0)
     return nullptr;
 
       // Check the path is on the same device / file system.  If this
       // fails, we found a better prefix match on path, but it's the
       // wrong device, so reset our idea of the best match: it can't
       // be the outer mount any more.  Not sure this is the right
       // thing to do with e.g. loop-back or union mounts.
       if (fs_[i]->dev != s->st_dev || fs_[i]->fstype != sfs->f_type)
       {
     best = nullptr;
     continue;
       }
 
       // OK this is better than anything else we found so far.
       best = fs_[i];
       bestlen = fslen;
     }
   }
   // In the case of a bind mount, try looking again at the source directory.
   if (best && best->bind && best->origin)
   {
     struct stat s2;
     struct statfs sfs2;
     char *fullpath = realpath(best->origin, nullptr);
 
     if (! fullpath)
       fullpath = strdup(best->origin);
 
     if (lstat(fullpath, &s2) < 0)
     {
       int nerr = errno;
       edm::LogWarning("LocalFileSystem::findMount()")
         << "Cannot lstat('" << fullpath << "' alias '"
         << path << "'): " << strerror(nerr) << " (error "
         << nerr << ")";
       free(fullpath);
       return best;
     }
 
     if (statfs(fullpath, &sfs2) < 0)
     {
       int nerr = errno;
       edm::LogWarning("LocalFileSystem::findMount()")
         << "Cannot statfs('" << fullpath << "' alias '"
         << path << "'): " << strerror(nerr) << " (error "
         << nerr << ")";
       free(fullpath);
       return best;
     }
 
     prev_paths.push_back(path);
     LocalFileSystem::FSInfo *new_best = findMount(fullpath, &sfs2, &s2, prev_paths);
     return new_best ? new_best : best;
   }
 
   return best;
 }

LocalFileSystem::FSInfo * LocalFileSystem::initFSInfo ( void * arg )

private

Initialise file system description from /etc/mtab info.

This function saves the information from getmntent(), matching the file system type to the known local ones. It only remembers the information from /etc/mtab, so the dev and fstype attributes are not yet valid; call statFSInfo() to fill those in. This avoids touching irrelevant filesystems unnecessarily; the file system may not be fully functional, or partially offline, or just very slow.

Definition at line 143 of file LocalFileSystem.cc.

References LocalFileSystem::FSInfo::bind, LocalFileSystem::FSInfo::checked, LocalFileSystem::FSInfo::dev, LocalFileSystem::FSInfo::dir, LocalFileSystem::FSInfo::freespc, LocalFileSystem::FSInfo::fsname, LocalFileSystem::FSInfo::fstype, fstypes_, mps_fire::i, LocalFileSystem::FSInfo::local, funct::m, LocalFileSystem::FSInfo::origin, AlCaHLTBitMon_ParallelJobs::p, and LocalFileSystem::FSInfo::type.

Referenced by initFSList().

 {
 #if BSD
   struct statfs *m = static_cast<struct statfs *>(arg);
   size_t infolen = sizeof(struct FSInfo);
   size_t fslen = strlen(m->f_mntfromname) + 1;
   size_t dirlen = strlen(m->f_mntonname) + 1;
   size_t typelen = strlen(m->f_fstypename) + 1;
   size_t totlen = infolen + fslen + dirlen + typelen;
   FSInfo *i = (FSInfo *) malloc(totlen);
   char *p = (char *) i;
   i->fsname = strncpy(p += infolen, m->f_mntfromname, fslen);
   i->type = strncpy(p += fslen, m->f_fstypename, typelen);
   i->dir = strncpy(p += typelen, m->f_mntonname, dirlen);
   i->dev = m->f_fsid.val[0];
   i->fstype = m->f_type;
   i->freespc = 0;
   i->bind = 0;
   i->origin = nullptr;
   if (m->f_bsize > 0)
   {
     i->freespc = m->f_bavail;
     i->freespc *= m->f_bsize;
     i->freespc /= 1024. * 1024. * 1024.;
   } 
   /* FIXME: This incorrectly says that mounted disk images are local,
      even if it was mounted from a network server. The alternative is
      to walk up the device tree using either a) process IORegistry to
      get the device tree, which lists devices for disk images, and from
      there translate volume uuid to a mount point; b) parse output from
      'hdiutil info -plist' to determine image-path / dev-entry map. */
   i->local = ((m->f_flags & MNT_LOCAL) ? 1 : 0);
   i->checked = 1;
   return i;
 
 #else // ! BSD
   mntent *m = static_cast<mntent *>(arg);
   size_t infolen = sizeof(struct FSInfo);
   size_t fslen = strlen(m->mnt_fsname) + 1;
   size_t dirlen = strlen(m->mnt_dir) + 1;
   size_t typelen = strlen(m->mnt_type) + 1;
   size_t originlen = strlen(m->mnt_fsname) + 1;
   size_t totlen = infolen + fslen + dirlen + typelen + originlen;
   FSInfo *i = (FSInfo *) malloc(totlen);
   char *p = (char *) i;
   i->fsname = static_cast<char*>(memcpy(p += infolen, m->mnt_fsname, fslen));
   i->type = static_cast<char*>(memcpy(p += fslen, m->mnt_type, typelen));
   i->dir = static_cast<char*>(memcpy(p += typelen, m->mnt_dir, dirlen));
   i->origin = static_cast<char*>(memcpy(p += dirlen, m->mnt_fsname, originlen));
   i->dev = -1;
   i->fstype = -1;
   i->freespc = 0;
   i->local = 0;
   i->checked = false;
   i->bind = strstr(m->mnt_opts, "bind") != nullptr;
 
   for (size_t j = 0; j < fstypes_.size() && ! i->local; ++j)
     if (fstypes_[j] == i->type)
       i->local = 1;
 #endif // BSD
 
   return i;
 }

int LocalFileSystem::initFSList ( void )

private

Initialise the list of currently mounted file systems.

Reads /etc/mtab (or equivalent) to determine all currently mounted file systems, and initialises FSInfo structure for them. It does not yet call statFSInfo() on them, so the device and file type ids are not yet complete.

Definition at line 214 of file LocalFileSystem.cc.

References fs_, initFSInfo(), and funct::m.

Referenced by LocalFileSystem().

 {
 #if BSD
   int rc;
   struct statfs *mtab = 0;
   if ((rc = getmntinfo(&mtab, MNT_NOWAIT)) < 0)
   {
     int nerr = errno;
     edm::LogWarning("LocalFileSystem::initFSList()")
       << "getmntinfo() failed: " << strerror(nerr)
       << " (error " << nerr << ")";
     return -1;
   }
 
   fs_.reserve(rc);
   for (int ix = 0; ix < rc; ++ix)
     fs_.push_back(initFSInfo(&mtab[ix]));
 
   free(mtab);
 #else
   const char * const _PATH_MOUNTED_LINUX = "/proc/self/mounts";
   struct mntent *m;
   FILE *mtab = setmntent(_PATH_MOUNTED_LINUX, "r");
   if (! mtab)
   {
     int nerr = errno;
     edm::LogWarning("LocalFileSystem::initFSList()")
       << "Cannot read '" << _PATH_MOUNTED_LINUX << "': "
       << strerror(nerr) << " (error " << nerr << ")";
     return -1;
   }
 
   fs_.reserve(20);
   while ((m = getmntent(mtab)))
     fs_.push_back(initFSInfo(m));
 
   endmntent(mtab);
 #endif
 
   return 0;
 }

bool LocalFileSystem::isLocalPath ( const std::string & path ) const

Determine if path is on a file system known to be local.

Returns true if the path is definitely known to be local. Returns false otherwise, including when it's not possible to determine anything about the path at all.

Does not throw exceptions. If any errors occur, the errors are reported as message logger warnings but the actual error is swallowed and the function simply returns false.

Definition at line 429 of file LocalFileSystem.cc.

References findMount(), LocalFileSystem::FSInfo::local, and funct::m.

Referenced by StorageFactory::wrapNonLocalFile().

 {
   struct stat s;
   struct statfs sfs;
   char *fullpath = realpath(path.c_str(), nullptr);
 
   if (! fullpath)
     fullpath = strdup(path.c_str());
 
   if (lstat(fullpath, &s) < 0)
   {
     int nerr = errno;
     edm::LogWarning("LocalFileSystem::isLocalPath()")
       << "Cannot lstat('" << fullpath << "' alias '"
       << path << "'): " << strerror(nerr) << " (error "
       << nerr << ")";
     free(fullpath);
     return false;
   }
     
   if (statfs(fullpath, &sfs) < 0)
   {
     int nerr = errno;
     edm::LogWarning("LocalFileSystem::isLocalPath()")
       << "Cannot statfs('" << fullpath << "' alias '"
       << path << "'): " << strerror(nerr) << " (error "
       << nerr << ")";
     free(fullpath);
     return false;
   }
 
   std::vector<std::string> prev_paths;
   FSInfo *m = findMount(fullpath, &sfs, &s, prev_paths);
   free(fullpath);
 
   return m ? m->local : false;
 }

void LocalFileSystem::operator= ( LocalFileSystem & )

privatedelete

int LocalFileSystem::readFSTypes ( void )

private

Read /proc/filesystems to determine which filesystems are local, meaning access latency is tolerably small, and operating system buffer cache will likely do a good job at caching file contents and accelerate many small file operations reasonably well.

The /proc list enumerates all filesystems known by the kernel, except a few special ones like /dev and /selinux. The ones marked as "nodev" have unstable device definition, meaning they are some way or another "virtual" file systems. This labeling is used by kernel nfsd to determine which file systems are safe for exporting without help (fixing fsid), and turns out to be close enough to list of file systems that we can consider to be high-speed local, minus a few exceptions. Everything else we consider "remote" or "slow" file systems where application should prefer massive bulk streaming I/O for better performance.

The exceptions to /proc/filesystems list: lustre and fuse file systems are forced to remote status. Everything else like NFS, AFS, GPFS and various cluster-based systems are already remote.

Definition at line 61 of file LocalFileSystem.cc.

References LocalFileSystem::FSInfo::fstype, fstypes_, and mps_splice::line.

Referenced by LocalFileSystem().

 {
   int ret = 0;
 
 #if __linux__
   static const char procfs[] = "/proc/filesystems";
   FILE *fs = fopen(procfs, "r");
   if (! fs)
   {
     int nerr = errno;
     edm::LogWarning("LocalFileSystem::readFSTypes()")
       << "Cannot read '" << procfs << "': "
       << strerror(nerr) << " (error " << nerr << ")";
     return -1;
   }
 
   ssize_t nread;
   int line = 0;
   while (! feof(fs))
   {
     char *type = nullptr;
     char *fstype = nullptr;
     size_t len = 0;
     ++line;
 
     if ((nread = getdelim(&type, &len, '\t', fs)) == -1 && ! feof(fs))
     {
       fprintf(stderr, "%s:%d: %s (%zd; 1)\n",
           procfs, line, strerror(errno), nread);
       free(type);
       ret = -1;
       break;
     }
 
     if ((nread = getdelim(&fstype, &len, '\n', fs)) == -1 && ! feof(fs))
     {
       fprintf(stderr, "%s:%d: %s (%zd; 2)\n",
           procfs, line, strerror(errno), nread);
       free(type);
       free(fstype);
       ret = -1;
       break;
     }
 
     if (feof (fs))
     {
       free(type);
       free(fstype);
       break;
     }
     
     if (! strcmp(type, "nodev\t")
     || ! strcmp(fstype, "lustre\n")
     || ! strncmp(fstype, "fuse", 4))
     {
       free(type);
       free(fstype);
       continue;
     }
 
     assert(nread >= 1);
     fstype[nread-1] = 0;
     fstypes_.push_back(fstype);
     free(fstype);
     free(type);
   }
 
   fclose(fs);
 #endif // __linux__
 
   return ret;
 }

int LocalFileSystem::statFSInfo ( FSInfo * i ) const

private

Figure out file system device and type ids.

Calls stat() and statfs() on the file system to determine device and file system type ids. These are required to determine if two paths are actually on the same file system.

This function can be called any number of times. It only does the file system check the first time the function is called.

Definition at line 265 of file LocalFileSystem.cc.

References LocalFileSystem::FSInfo::checked, LocalFileSystem::FSInfo::dev, LocalFileSystem::FSInfo::dir, LocalFileSystem::FSInfo::freespc, and LocalFileSystem::FSInfo::fstype.

Referenced by findMount().

 {
   int ret = 0;
   struct stat s;
   struct statfs sfs;
 
   if (! i->checked)
   {
     if (lstat(i->dir, &s) < 0)
     {
       i->checked = true;
 
       int nerr = errno;
       if (nerr != ENOENT && nerr != EACCES)
         edm::LogWarning("LocalFileSystem::statFSInfo()")
       << "Cannot lstat('" << i->dir << "'): "
       << strerror(nerr) << " (error " << nerr << ")";
       return -1;
     }
 
     if (statfs(i->dir, &sfs) < 0)
     {
       i->checked = true;
       int nerr = errno;
       edm::LogWarning("LocalFileSystem::statFSInfo()")
     << "Cannot statfs('" << i->dir << "'): "
     << strerror(nerr) << " (error " << nerr << ")";
       return -1;
     }
 
     i->dev = s.st_dev;
     i->fstype = sfs.f_type;
     if (sfs.f_bsize > 0)
     {
       i->freespc = sfs.f_bavail;
       i->freespc *= sfs.f_bsize;
       i->freespc /= 1024. * 1024. * 1024.;
     }
     i->checked = true;
   }
   else if (i->fstype == -1)
   {
     errno = ENOENT;
     ret = -1;
   }
 
   return ret;
 }

Member Data Documentation

std::vector<FSInfo *> LocalFileSystem::fs_

private

Definition at line 28 of file LocalFileSystem.h.

Referenced by findMount(), initFSList(), and ~LocalFileSystem().

std::vector<std::string> LocalFileSystem::fstypes_

private

Definition at line 29 of file LocalFileSystem.h.

Referenced by initFSInfo(), and readFSTypes().

Classes

Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation