LocalFileSystem Class Reference

#include <LocalFileSystem.h>

Classes
struct	FSInfo
	Information about file systems on this node. More...

Public Member Functions
std::pair< std::string, std::string >	findCachePath (const std::vector< std::string > &paths, double minFreeSpace) const
bool	isLocalPath (const std::string &path) const
	LocalFileSystem (void)
	~LocalFileSystem (void)

Private Member Functions
FSInfo *	findMount (const char *path, struct statfs *sfs, struct stat *s, std::vector< std::string > &) const
FSInfo *	initFSInfo (void *p)
int	initFSList (void)
	LocalFileSystem (LocalFileSystem &)=delete
void	operator= (LocalFileSystem &)=delete
int	readFSTypes (void)
int	statFSInfo (FSInfo *i) const

Private Attributes
std::vector< FSInfo * >	fs_
std::vector< std::string >	fstypes_

Detailed Description

Definition at line 11 of file LocalFileSystem.h.

Constructor & Destructor Documentation

LocalFileSystem() [1/2]

LocalFileSystem::LocalFileSystem

(

void

)

Initialise local file system status.

Definition at line 512 of file LocalFileSystem.cc.

                                     {
  if (readFSTypes() < 0)
    return;
 
  if (initFSList() < 0)
    return;
}

References initFSList(), and readFSTypes().

~LocalFileSystem()

LocalFileSystem::~LocalFileSystem

(

void

)

Free local file system status resources.

Definition at line 521 of file LocalFileSystem.cc.

                                      {
  for (size_t i = 0, e = fs_.size(); i < e; ++i)
    free(fs_[i]);
}

References MillePedeFileConverter_cfg::e, fs_, and mps_fire::i.

LocalFileSystem() [2/2]

LocalFileSystem::LocalFileSystem ( LocalFileSystem &  )

privatedelete

Member Function Documentation

findCachePath()

std::pair< std::string, std::string > LocalFileSystem::findCachePath	(	const std::vector< std::string > &	paths,
		double	minFreeSpace
	)		const

Find a writeable directory among paths which is known to be local and has at least minFreeSpace amount of free space in gigabytes.

The paths should contain list of relative or absolute candidate directories. If an entry starts with letter "$" then the value of that environment variable is used instead; if the value is $TMPDIR and the environment variable is empty, "/tmp" is used instead.

Returns the first path in paths which satisfies the criteria, expanded to environment variable value if appropriate, resolved to full absolute path. If no suitable path can be found, returns an empty string.

Does not throw exceptions. If any serious errors occur, the errors are reported as message logger warnings but the actual error is swallowed and the directory concerned is skipped. Non-existent and inaccessible directories are silently ignored without warning.

Definition at line 426 of file LocalFileSystem.cc.

                                                                                              {
  struct stat s;
  struct statfs sfs;
  std::ostringstream warningst;
  warningst << "Cannot use lazy-download because:\n";
 
  for (size_t i = 0, e = paths.size(); i < e; ++i) {
    char *fullpath;
    const char *inpath = paths[i].c_str();
    const char *path = inpath;
 
    if (*path == '$') {
      char *p = std::getenv(path + 1);
      if (p && *p)
        path = p;
      else if (!strcmp(path, "$TMPDIR"))
        path = "/tmp";
    }
 
    if (!(fullpath = realpath(path, nullptr)))
      fullpath = strdup(path);
 
#if 0
    std::cerr /* edm::LogInfo("LocalFileSystem") */
      << "Checking if '" << fullpath << "', from '"
      << inpath << "' is valid cache path with "
      << minFreeSpace << " free space" << std::endl;
#endif
 
    if (lstat(fullpath, &s) < 0) {
      int nerr = errno;
      if (nerr != ENOENT && nerr != EACCES)
        edm::LogWarning("LocalFileSystem::findCachePath()") << "Cannot lstat('" << fullpath << "', from '" << inpath
                                                            << "'): " << strerror(nerr) << " (error " << nerr << ")";
      free(fullpath);
      continue;
    }
 
    if (statfs(fullpath, &sfs) < 0) {
      int nerr = errno;
      edm::LogWarning("LocalFileSystem::findCachePath()") << "Cannot statfs('" << fullpath << "', from '" << inpath
                                                          << "'): " << strerror(nerr) << " (error " << nerr << ")";
      free(fullpath);
      continue;
    }
 
    std::vector<std::string> prev_paths;
    FSInfo *m = findMount(fullpath, &sfs, &s, prev_paths);
#if 0
    std::cerr /* edm::LogInfo("LocalFileSystem") */
      << "Candidate '" << fullpath << "': "
      << "found=" << (m ? 1 : 0)
      << " local=" << (m && m->local)
      << " free=" << (m ? m->freespc : 0)
      << " access=" << access(fullpath, W_OK)
      << std::endl;
#endif
 
    if (m && m->local && m->freespc >= minFreeSpace && access(fullpath, W_OK) == 0) {
      std::string result(fullpath);
      free(fullpath);
      return std::make_pair(result, std::string());
    } else if (m) {
      if (!m->local) {
        warningst << "- The mount " << fullpath << " is not local.\n";
      } else if (m->freespc < minFreeSpace) {
        warningst << " - The mount at " << fullpath << " has only " << m->freespc << " GB free; a minumum of "
                  << minFreeSpace << " GB is required.\n";
      } else if (access(fullpath, W_OK)) {
        warningst << " - The process has no permission to write into " << fullpath << "\n";
      }
    }
 
    free(fullpath);
  }
 
  std::string warning_str = warningst.str();
  if (!warning_str.empty()) {
    warning_str = warning_str.substr(0, warning_str.size() - 2);
  }
 
  return std::make_pair(std::string(), std::move(warning_str));
}

References EcnaPython_AdcPeg12_S1_10_R170298_1_0_150_Dee0::cerr, MillePedeFileConverter_cfg::e, findMount(), reco_skim_cfg_mod::fullpath, mps_fire::i, visualization-live-secondInstance_cfg::m, eostools::move(), AlCaHLTBitMon_ParallelJobs::p, castor_dqm_sourceclient_file_cfg::path, Skims_PA_cff::paths, mps_fire::result, alignCSCRings::s, edm_modernize_messagelogger::stat, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by StorageFactory::setTempDir().

findMount()

LocalFileSystem::FSInfo * LocalFileSystem::findMount ( const char *  path, struct statfs *  sfs, struct stat *  s, std::vector< std::string > &  prev_paths  ) const

private

Find the file system path was mounted from. The statfs() and stat() information for path should be in sfs and s, respectively.

Finds currently mounted file system that path is owned by, and returns the FSInfo object for it, or null if no matching live file system can be found. If the return value is non-null, then it is guaranteed path was on that file system.

A null return value is possible for certain paths which are not on any mounted file system (e.g. /dev or /selinux), or if the file system is unavailable or some other way dysfunctional, such as dead nfs mount or filesystem does not implement statfs().

Definition at line 289 of file LocalFileSystem.cc.

                                                                                            {
  for (const auto &old_path : prev_paths) {
    if (!strcmp(old_path.c_str(), path)) {
      edm::LogWarning("LocalFileSystem::findMount()") << "Found a loop in bind mounts; stopping evaluation.";
      return nullptr;
    }
  }
 
  FSInfo *best = nullptr;
  size_t bestlen = 0;
  size_t len = strlen(path);
  for (size_t i = 0; i < fs_.size(); ++i) {
    // First match simply against the file system path.  We don't
    // touch the file system until the path prefix matches.
    // When we have a path prefix match, check the file system if
    //   we don't have a best match candidate yet, OR
    //   this match is longer (more specific) than the previous best OR
    //   this match is the same length and the previous best isn't local
    // The final condition handles cases such as '/' that can appear twice
    // in the file system list, once as 'rootfs' and once as local fs.
    size_t fslen = strlen(fs_[i]->dir);
    if (!strncmp(fs_[i]->dir, path, fslen) &&
        ((fslen == 1 && fs_[i]->dir[0] == '/') || len == fslen || path[fslen] == '/') &&
        (!best || fslen > bestlen || (fslen == bestlen && !best->local))) {
      // Get the file system device and file system ids.
      if (statFSInfo(fs_[i]) < 0)
        return nullptr;
 
      // Check the path is on the same device / file system.  If this
      // fails, we found a better prefix match on path, but it's the
      // wrong device, so reset our idea of the best match: it can't
      // be the outer mount any more.  Not sure this is the right
      // thing to do with e.g. loop-back or union mounts.
      if (fs_[i]->dev != s->st_dev || fs_[i]->fstype != sfs->f_type) {
        best = nullptr;
        continue;
      }
 
      // OK this is better than anything else we found so far.
      best = fs_[i];
      bestlen = fslen;
    }
  }
  // In the case of a bind mount, try looking again at the source directory.
  if (best && best->bind && best->origin) {
    struct stat s2;
    struct statfs sfs2;
    char *fullpath = realpath(best->origin, nullptr);
 
    if (!fullpath)
      fullpath = strdup(best->origin);
 
    if (lstat(fullpath, &s2) < 0) {
      int nerr = errno;
      edm::LogWarning("LocalFileSystem::findMount()") << "Cannot lstat('" << fullpath << "' alias '" << path
                                                      << "'): " << strerror(nerr) << " (error " << nerr << ")";
      free(fullpath);
      return best;
    }
 
    if (statfs(fullpath, &sfs2) < 0) {
      int nerr = errno;
      edm::LogWarning("LocalFileSystem::findMount()") << "Cannot statfs('" << fullpath << "' alias '" << path
                                                      << "'): " << strerror(nerr) << " (error " << nerr << ")";
      free(fullpath);
      return best;
    }
 
    prev_paths.push_back(path);
    LocalFileSystem::FSInfo *new_best = findMount(fullpath, &sfs2, &s2, prev_paths);
    return new_best ? new_best : best;
  }
 
  return best;
}

References LocalFileSystem::FSInfo::bind, DeadROC_duringRun::dir, fs_, reco_skim_cfg_mod::fullpath, mps_fire::i, LocalFileSystem::FSInfo::origin, castor_dqm_sourceclient_file_cfg::path, alignCSCRings::s, indexGen::s2, edm_modernize_messagelogger::stat, and statFSInfo().

Referenced by findCachePath(), and isLocalPath().

initFSInfo()

LocalFileSystem::FSInfo * LocalFileSystem::initFSInfo ( void *  arg )

private

Initialise file system description from /etc/mtab info.

This function saves the information from getmntent(), matching the file system type to the known local ones. It only remembers the information from /etc/mtab, so the dev and fstype attributes are not yet valid; call statFSInfo() to fill those in. This avoids touching irrelevant filesystems unnecessarily; the file system may not be fully functional, or partially offline, or just very slow.

Definition at line 123 of file LocalFileSystem.cc.

                                                            {
#if BSD
  struct statfs *m = static_cast<struct statfs *>(arg);
  size_t infolen = sizeof(struct FSInfo);
  size_t fslen = strlen(m->f_mntfromname) + 1;
  size_t dirlen = strlen(m->f_mntonname) + 1;
  size_t typelen = strlen(m->f_fstypename) + 1;
  size_t totlen = infolen + fslen + dirlen + typelen;
  FSInfo *i = (FSInfo *)malloc(totlen);
  char *p = (char *)i;
  i->fsname = strncpy(p += infolen, m->f_mntfromname, fslen);
  i->type = strncpy(p += fslen, m->f_fstypename, typelen);
  i->dir = strncpy(p += typelen, m->f_mntonname, dirlen);
  i->dev = m->f_fsid.val[0];
  i->fstype = m->f_type;
  i->freespc = 0;
  i->bind = 0;
  i->origin = nullptr;
  if (m->f_bsize > 0) {
    i->freespc = m->f_bavail;
    i->freespc *= m->f_bsize;
    i->freespc /= 1024. * 1024. * 1024.;
  }
  /* FIXME: This incorrectly says that mounted disk images are local,
     even if it was mounted from a network server. The alternative is
     to walk up the device tree using either a) process IORegistry to
     get the device tree, which lists devices for disk images, and from
     there translate volume uuid to a mount point; b) parse output from
     'hdiutil info -plist' to determine image-path / dev-entry map. */
  i->local = ((m->f_flags & MNT_LOCAL) ? 1 : 0);
  i->checked = 1;
  return i;
 
#else   // ! BSD
  mntent *m = static_cast<mntent *>(arg);
  size_t infolen = sizeof(struct FSInfo);
  size_t fslen = strlen(m->mnt_fsname) + 1;
  size_t dirlen = strlen(m->mnt_dir) + 1;
  size_t typelen = strlen(m->mnt_type) + 1;
  size_t originlen = strlen(m->mnt_fsname) + 1;
  size_t totlen = infolen + fslen + dirlen + typelen + originlen;
  FSInfo *i = (FSInfo *)malloc(totlen);
  char *p = (char *)i;
  i->fsname = static_cast<char *>(memcpy(p += infolen, m->mnt_fsname, fslen));
  i->type = static_cast<char *>(memcpy(p += fslen, m->mnt_type, typelen));
  i->dir = static_cast<char *>(memcpy(p += typelen, m->mnt_dir, dirlen));
  i->origin = static_cast<char *>(memcpy(p += dirlen, m->mnt_fsname, originlen));
  i->dev = -1;
  i->fstype = -1;
  i->freespc = 0;
  i->local = 0;
  i->checked = false;
  i->bind = strstr(m->mnt_opts, "bind") != nullptr;
 
  for (size_t j = 0; j < fstypes_.size() && !i->local; ++j)
    if (fstypes_[j] == i->type)
      i->local = 1;
#endif  // BSD
 
  return i;
}

References fstypes_, mps_fire::i, dqmiolumiharvest::j, visualization-live-secondInstance_cfg::m, and AlCaHLTBitMon_ParallelJobs::p.

Referenced by initFSList().

initFSList()

int LocalFileSystem::initFSList ( void  )

private

Initialise the list of currently mounted file systems.

Reads /etc/mtab (or equivalent) to determine all currently mounted file systems, and initialises FSInfo structure for them. It does not yet call statFSInfo() on them, so the device and file type ids are not yet complete.

Definition at line 191 of file LocalFileSystem.cc.

                                    {
#if BSD
  int rc;
  struct statfs *mtab = 0;
  if ((rc = getmntinfo(&mtab, MNT_NOWAIT)) < 0) {
    int nerr = errno;
    edm::LogWarning("LocalFileSystem::initFSList()")
        << "getmntinfo() failed: " << strerror(nerr) << " (error " << nerr << ")";
    return -1;
  }
 
  fs_.reserve(rc);
  for (int ix = 0; ix < rc; ++ix)
    fs_.push_back(initFSInfo(&mtab[ix]));
 
  free(mtab);
#else
  const char *const _PATH_MOUNTED_LINUX = "/proc/self/mounts";
  struct mntent *m;
  FILE *mtab = setmntent(_PATH_MOUNTED_LINUX, "r");
  if (!mtab) {
    int nerr = errno;
    edm::LogWarning("LocalFileSystem::initFSList()")
        << "Cannot read '" << _PATH_MOUNTED_LINUX << "': " << strerror(nerr) << " (error " << nerr << ")";
    return -1;
  }
 
  fs_.reserve(20);
  while ((m = getmntent(mtab)))
    fs_.push_back(initFSInfo(m));
 
  endmntent(mtab);
#endif
 
  return 0;
}

References fs_, initFSInfo(), and visualization-live-secondInstance_cfg::m.

Referenced by LocalFileSystem().

isLocalPath()

bool LocalFileSystem::isLocalPath

(

const std::string &

path

)

const

Determine if path is on a file system known to be local.

Returns true if the path is definitely known to be local. Returns false otherwise, including when it's not possible to determine anything about the path at all.

Does not throw exceptions. If any errors occur, the errors are reported as message logger warnings but the actual error is swallowed and the function simply returns false.

Definition at line 377 of file LocalFileSystem.cc.

                                                             {
  struct stat s;
  struct statfs sfs;
  char *fullpath = realpath(path.c_str(), nullptr);
 
  if (!fullpath)
    fullpath = strdup(path.c_str());
 
  if (lstat(fullpath, &s) < 0) {
    int nerr = errno;
    edm::LogWarning("LocalFileSystem::isLocalPath()")
        << "Cannot lstat('" << fullpath << "' alias '" << path << "'): " << strerror(nerr) << " (error " << nerr << ")";
    free(fullpath);
    return false;
  }
 
  if (statfs(fullpath, &sfs) < 0) {
    int nerr = errno;
    edm::LogWarning("LocalFileSystem::isLocalPath()") << "Cannot statfs('" << fullpath << "' alias '" << path
                                                      << "'): " << strerror(nerr) << " (error " << nerr << ")";
    free(fullpath);
    return false;
  }
 
  std::vector<std::string> prev_paths;
  FSInfo *m = findMount(fullpath, &sfs, &s, prev_paths);
  free(fullpath);
 
  return m ? m->local : false;
}

References findMount(), reco_skim_cfg_mod::fullpath, visualization-live-secondInstance_cfg::m, castor_dqm_sourceclient_file_cfg::path, alignCSCRings::s, and edm_modernize_messagelogger::stat.

Referenced by StorageFactory::wrapNonLocalFile().

operator=()

void LocalFileSystem::operator= ( LocalFileSystem &  )

privatedelete

readFSTypes()

int LocalFileSystem::readFSTypes ( void  )

private

Read /proc/filesystems to determine which filesystems are local, meaning access latency is tolerably small, and operating system buffer cache will likely do a good job at caching file contents and accelerate many small file operations reasonably well.

The /proc list enumerates all filesystems known by the kernel, except a few special ones like /dev and /selinux. The ones marked as "nodev" have unstable device definition, meaning they are some way or another "virtual" file systems. This labeling is used by kernel nfsd to determine which file systems are safe for exporting without help (fixing fsid), and turns out to be close enough to list of file systems that we can consider to be high-speed local, minus a few exceptions. Everything else we consider "remote" or "slow" file systems where application should prefer massive bulk streaming I/O for better performance.

The exceptions to /proc/filesystems list: lustre and fuse file systems are forced to remote status. Everything else like NFS, AFS, GPFS and various cluster-based systems are already remote.

Definition at line 58 of file LocalFileSystem.cc.

                                     {
  int ret = 0;
 
#if __linux__
  constexpr char procfs[] = "/proc/filesystems";
  auto close_ = [](FILE *iFile) { fclose(iFile); };
  std::unique_ptr<FILE, decltype(close_)> fs(fopen(procfs, "r"), close_);
  if (!fs) {
    int nerr = errno;
    edm::LogWarning("LocalFileSystem::readFSTypes()")
        << "Cannot read '" << procfs << "': " << strerror(nerr) << " (error " << nerr << ")";
    return -1;
  }
 
  ssize_t nread;
  int line = 0;
  auto free_ = [](char **iPtr) { free(*iPtr); };
  while (!feof(fs.get())) {
    char *type = nullptr;
    std::unique_ptr<char *, decltype(free_)> freeType(&type, free_);
 
    size_t len = 0;
    ++line;
 
    if ((nread = getdelim(&type, &len, '\t', fs.get())) == -1 && !feof(fs.get())) {
      edm::LogError("LocalFileSystem::readFSTypes()")
          .format("{}:{}: {} ({}; 1)\n", procfs, line, strerror(errno), nread);
      ret = -1;
      break;
    }
 
    char *fstype = nullptr;
    std::unique_ptr<char *, decltype(free_)> freeFSType(&fstype, free_);
    if ((nread = getdelim(&fstype, &len, '\n', fs.get())) == -1 && !feof(fs.get())) {
      edm::LogError("LocalFileSystem::readFSTypes()")
          .format("{}:{}: {} ({}; 2)\n", procfs, line, strerror(errno), nread);
      ret = -1;
      break;
    }
 
    if (feof(fs.get())) {
      break;
    }
 
    if (!strcmp(type, "nodev\t") || !strcmp(fstype, "lustre\n") || !strncmp(fstype, "fuse", 4)) {
      continue;
    }
 
    assert(nread >= 1);
    fstype[nread - 1] = 0;
    fstypes_.push_back(fstype);
  }
#endif  // __linux__
 
  return ret;
}

References cms::cuda::assert(), fstypes_, mps_splice::line, and runTheMatrix::ret.

Referenced by LocalFileSystem().

statFSInfo()

int LocalFileSystem::statFSInfo ( FSInfo *  i ) const

private

Figure out file system device and type ids.

Calls stat() and statfs() on the file system to determine device and file system type ids. These are required to determine if two paths are actually on the same file system.

This function can be called any number of times. It only does the file system check the first time the function is called.

Definition at line 236 of file LocalFileSystem.cc.

                                               {
  int ret = 0;
  struct stat s;
  struct statfs sfs;
 
  if (!i->checked) {
    if (lstat(i->dir, &s) < 0) {
      i->checked = true;
 
      int nerr = errno;
      if (nerr != ENOENT && nerr != EACCES)
        edm::LogWarning("LocalFileSystem::statFSInfo()")
            << "Cannot lstat('" << i->dir << "'): " << strerror(nerr) << " (error " << nerr << ")";
      return -1;
    }
 
    if (statfs(i->dir, &sfs) < 0) {
      i->checked = true;
      int nerr = errno;
      edm::LogWarning("LocalFileSystem::statFSInfo()")
          << "Cannot statfs('" << i->dir << "'): " << strerror(nerr) << " (error " << nerr << ")";
      return -1;
    }
 
    i->dev = s.st_dev;
    i->fstype = sfs.f_type;
    if (sfs.f_bsize > 0) {
      i->freespc = sfs.f_bavail;
      i->freespc *= sfs.f_bsize;
      i->freespc /= 1024. * 1024. * 1024.;
    }
    i->checked = true;
  } else if (i->fstype == -1) {
    errno = ENOENT;
    ret = -1;
  }
 
  return ret;
}

References mps_fire::i, runTheMatrix::ret, alignCSCRings::s, and edm_modernize_messagelogger::stat.

Referenced by findMount().

Member Data Documentation

fs_

std::vector<FSInfo *> LocalFileSystem::fs_

private

Definition at line 28 of file LocalFileSystem.h.

Referenced by findMount(), initFSList(), and ~LocalFileSystem().

fstypes_

std::vector<std::string> LocalFileSystem::fstypes_

private

Definition at line 29 of file LocalFileSystem.h.

Referenced by initFSInfo(), and readFSTypes().