CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
LocalFileSystem.cc
Go to the documentation of this file.
1 #define _GNU_SOURCE 1
2 #define _FILE_OFFSET_BITS 64
5 #include <cerrno>
6 #include <cstdio>
7 #include <cstdlib>
8 #include <cstring>
9 #include <cassert>
10 #include <sys/param.h>
11 #if BSD
12 #include <sys/statvfs.h>
13 #include <sys/ucred.h>
14 #include <sys/mount.h>
15 #else
16 #include <mntent.h>
17 #include <sys/vfs.h>
18 #endif
19 #include <sys/stat.h>
20 #include <unistd.h>
21 #include <iostream>
22 #include <atomic>
23 #include <memory>
24 
25 using namespace edm::storage;
26 
29  char *fsname; //< file system name
30  char *type; //< file system type
31  char *dir; //< mount point directory
32  char *origin = nullptr; //< mount origin
33  dev_t dev; //< device id
34  long fstype; //< file system id
35  double freespc; //< free space in megabytes
36  unsigned local : 1; //< flag for local device
37  unsigned bind : 1; //< flag for bind mounts
38  std::atomic<bool> checked; //< flag for valid dev, fstype
39 };
40 
61  int ret = 0;
62 
63 #if __linux__
64  constexpr char procfs[] = "/proc/filesystems";
65  auto close_ = [](FILE *iFile) { fclose(iFile); };
66  std::unique_ptr<FILE, decltype(close_)> fs(fopen(procfs, "r"), close_);
67  if (!fs) {
68  int nerr = errno;
69  edm::LogWarning("LocalFileSystem::readFSTypes()")
70  << "Cannot read '" << procfs << "': " << strerror(nerr) << " (error " << nerr << ")";
71  return -1;
72  }
73 
74  ssize_t nread;
75  int line = 0;
76  auto free_ = [](char **iPtr) { free(*iPtr); };
77  while (!feof(fs.get())) {
78  char *type = nullptr;
79  std::unique_ptr<char *, decltype(free_)> freeType(&type, free_);
80 
81  size_t len = 0;
82  ++line;
83 
84  if ((nread = getdelim(&type, &len, '\t', fs.get())) == -1 && !feof(fs.get())) {
85  edm::LogError("LocalFileSystem::readFSTypes()")
86  .format("{}:{}: {} ({}; 1)\n", procfs, line, strerror(errno), nread);
87  ret = -1;
88  break;
89  }
90 
91  char *fstype = nullptr;
92  std::unique_ptr<char *, decltype(free_)> freeFSType(&fstype, free_);
93  if ((nread = getdelim(&fstype, &len, '\n', fs.get())) == -1 && !feof(fs.get())) {
94  edm::LogError("LocalFileSystem::readFSTypes()")
95  .format("{}:{}: {} ({}; 2)\n", procfs, line, strerror(errno), nread);
96  ret = -1;
97  break;
98  }
99 
100  if (feof(fs.get())) {
101  break;
102  }
103 
104  if (!strcmp(type, "nodev\t") || !strcmp(fstype, "lustre\n") || !strncmp(fstype, "fuse", 4)) {
105  continue;
106  }
107 
108  assert(nread >= 1);
109  fstype[nread - 1] = 0;
110  fstypes_.push_back(fstype);
111  }
112 #endif // __linux__
113 
114  return ret;
115 }
116 
126 #if BSD
127  struct statfs *m = static_cast<struct statfs *>(arg);
128  size_t infolen = sizeof(struct FSInfo);
129  size_t fslen = strlen(m->f_mntfromname) + 1;
130  size_t dirlen = strlen(m->f_mntonname) + 1;
131  size_t typelen = strlen(m->f_fstypename) + 1;
132  size_t totlen = infolen + fslen + dirlen + typelen;
133  FSInfo *i = (FSInfo *)malloc(totlen);
134  char *p = (char *)i;
135  i->fsname = strncpy(p += infolen, m->f_mntfromname, fslen);
136  i->type = strncpy(p += fslen, m->f_fstypename, typelen);
137  i->dir = strncpy(p += typelen, m->f_mntonname, dirlen);
138  i->dev = m->f_fsid.val[0];
139  i->fstype = m->f_type;
140  i->freespc = 0;
141  i->bind = 0;
142  i->origin = nullptr;
143  if (m->f_bsize > 0) {
144  i->freespc = m->f_bavail;
145  i->freespc *= m->f_bsize;
146  i->freespc /= 1024. * 1024. * 1024.;
147  }
148  /* FIXME: This incorrectly says that mounted disk images are local,
149  even if it was mounted from a network server. The alternative is
150  to walk up the device tree using either a) process IORegistry to
151  get the device tree, which lists devices for disk images, and from
152  there translate volume uuid to a mount point; b) parse output from
153  'hdiutil info -plist' to determine image-path / dev-entry map. */
154  i->local = ((m->f_flags & MNT_LOCAL) ? 1 : 0);
155  i->checked = 1;
156  return i;
157 
158 #else // ! BSD
159  mntent *m = static_cast<mntent *>(arg);
160  size_t infolen = sizeof(struct FSInfo);
161  size_t fslen = strlen(m->mnt_fsname) + 1;
162  size_t dirlen = strlen(m->mnt_dir) + 1;
163  size_t typelen = strlen(m->mnt_type) + 1;
164  size_t originlen = strlen(m->mnt_fsname) + 1;
165  size_t totlen = infolen + fslen + dirlen + typelen + originlen;
166  FSInfo *i = (FSInfo *)malloc(totlen);
167  char *p = (char *)i;
168  i->fsname = static_cast<char *>(memcpy(p += infolen, m->mnt_fsname, fslen));
169  i->type = static_cast<char *>(memcpy(p += fslen, m->mnt_type, typelen));
170  i->dir = static_cast<char *>(memcpy(p += typelen, m->mnt_dir, dirlen));
171  i->origin = static_cast<char *>(memcpy(p + dirlen, m->mnt_fsname, originlen));
172  i->dev = -1;
173  i->fstype = -1;
174  i->freespc = 0;
175  i->local = 0;
176  i->checked = false;
177  i->bind = strstr(m->mnt_opts, "bind") != nullptr;
178 
179  for (size_t j = 0; j < fstypes_.size() && !i->local; ++j)
180  if (fstypes_[j] == i->type)
181  i->local = 1;
182 #endif // BSD
183 
184  return i;
185 }
186 
194 #if BSD
195  int rc;
196  struct statfs *mtab = 0;
197  if ((rc = getmntinfo(&mtab, MNT_NOWAIT)) < 0) {
198  int nerr = errno;
199  edm::LogWarning("LocalFileSystem::initFSList()")
200  << "getmntinfo() failed: " << strerror(nerr) << " (error " << nerr << ")";
201  return -1;
202  }
203 
204  fs_.reserve(rc);
205  for (int ix = 0; ix < rc; ++ix)
206  fs_.push_back(initFSInfo(&mtab[ix]));
207 
208  free(mtab);
209 #else
210  const char *const _PATH_MOUNTED_LINUX = "/proc/self/mounts";
211  struct mntent *m;
212  FILE *mtab = setmntent(_PATH_MOUNTED_LINUX, "r");
213  if (!mtab) {
214  int nerr = errno;
215  edm::LogWarning("LocalFileSystem::initFSList()")
216  << "Cannot read '" << _PATH_MOUNTED_LINUX << "': " << strerror(nerr) << " (error " << nerr << ")";
217  return -1;
218  }
219 
220  fs_.reserve(20);
221  while ((m = getmntent(mtab)))
222  fs_.push_back(initFSInfo(m));
223 
224  endmntent(mtab);
225 #endif
226 
227  return 0;
228 }
229 
239  int ret = 0;
240  struct stat s;
241  struct statfs sfs;
242 
243  if (!i->checked) {
244  if (lstat(i->dir, &s) < 0) {
245  i->checked = true;
246 
247  int nerr = errno;
248  if (nerr != ENOENT && nerr != EACCES)
249  edm::LogWarning("LocalFileSystem::statFSInfo()")
250  << "Cannot lstat('" << i->dir << "'): " << strerror(nerr) << " (error " << nerr << ")";
251  return -1;
252  }
253 
254  if (statfs(i->dir, &sfs) < 0) {
255  i->checked = true;
256  int nerr = errno;
257  edm::LogWarning("LocalFileSystem::statFSInfo()")
258  << "Cannot statfs('" << i->dir << "'): " << strerror(nerr) << " (error " << nerr << ")";
259  return -1;
260  }
261 
262  i->dev = s.st_dev;
263  i->fstype = sfs.f_type;
264  if (sfs.f_bsize > 0) {
265  i->freespc = sfs.f_bavail;
266  i->freespc *= sfs.f_bsize;
267  i->freespc /= 1024. * 1024. * 1024.;
268  }
269  i->checked = true;
270  } else if (i->fstype == -1) {
271  errno = ENOENT;
272  ret = -1;
273  }
274 
275  return ret;
276 }
277 
292  struct statfs *sfs,
293  struct stat *s,
294  std::vector<std::string> &prev_paths) const {
295  for (const auto &old_path : prev_paths) {
296  if (!strcmp(old_path.c_str(), path)) {
297  edm::LogWarning("LocalFileSystem::findMount()") << "Found a loop in bind mounts; stopping evaluation.";
298  return nullptr;
299  }
300  }
301 
302  FSInfo *best = nullptr;
303  size_t bestlen = 0;
304  size_t len = strlen(path);
305  for (size_t i = 0; i < fs_.size(); ++i) {
306  // First match simply against the file system path. We don't
307  // touch the file system until the path prefix matches.
308  // When we have a path prefix match, check the file system if
309  // we don't have a best match candidate yet, OR
310  // this match is longer (more specific) than the previous best OR
311  // this match is the same length and the previous best isn't local
312  // The final condition handles cases such as '/' that can appear twice
313  // in the file system list, once as 'rootfs' and once as local fs.
314  size_t fslen = strlen(fs_[i]->dir);
315  if (!strncmp(fs_[i]->dir, path, fslen) &&
316  ((fslen == 1 && fs_[i]->dir[0] == '/') || len == fslen || path[fslen] == '/') &&
317  (!best || fslen > bestlen || (fslen == bestlen && !best->local))) {
318  // Get the file system device and file system ids.
319  if (statFSInfo(fs_[i]) < 0)
320  return nullptr;
321 
322  // Check the path is on the same device / file system. If this
323  // fails, we found a better prefix match on path, but it's the
324  // wrong device, so reset our idea of the best match: it can't
325  // be the outer mount any more. Not sure this is the right
326  // thing to do with e.g. loop-back or union mounts.
327  if (fs_[i]->dev != s->st_dev || fs_[i]->fstype != sfs->f_type) {
328  best = nullptr;
329  continue;
330  }
331 
332  // OK this is better than anything else we found so far.
333  best = fs_[i];
334  bestlen = fslen;
335  }
336  }
337  // In the case of a bind mount, try looking again at the source directory.
338  if (best && best->bind && best->origin) {
339  struct stat s2;
340  struct statfs sfs2;
341  std::unique_ptr<char, decltype(std::free) *> fullpath{realpath(best->origin, nullptr), std::free};
342 
343  if (!fullpath)
344  fullpath.reset(strdup(best->origin));
345 
346  if (lstat(fullpath.get(), &s2) < 0) {
347  int nerr = errno;
348  edm::LogWarning("LocalFileSystem::findMount()") << "Cannot lstat('" << fullpath.get() << "' alias '" << path
349  << "'): " << strerror(nerr) << " (error " << nerr << ")";
350  return best;
351  }
352 
353  if (statfs(fullpath.get(), &sfs2) < 0) {
354  int nerr = errno;
355  edm::LogWarning("LocalFileSystem::findMount()") << "Cannot statfs('" << fullpath.get() << "' alias '" << path
356  << "'): " << strerror(nerr) << " (error " << nerr << ")";
357  return best;
358  }
359 
360  prev_paths.push_back(path);
361  LocalFileSystem::FSInfo *new_best = findMount(fullpath.get(), &sfs2, &s2, prev_paths);
362  return new_best ? new_best : best;
363  }
364 
365  return best;
366 }
367 
378  struct stat s;
379  struct statfs sfs;
380  std::unique_ptr<char, decltype(std::free) *> fullpath{realpath(path.c_str(), nullptr), std::free};
381 
382  if (!fullpath)
383  fullpath.reset(strdup(path.c_str()));
384 
385  if (lstat(fullpath.get(), &s) < 0) {
386  int nerr = errno;
387  edm::LogWarning("LocalFileSystem::isLocalPath()") << "Cannot lstat('" << fullpath.get() << "' alias '" << path
388  << "'): " << strerror(nerr) << " (error " << nerr << ")";
389  return false;
390  }
391 
392  if (statfs(fullpath.get(), &sfs) < 0) {
393  int nerr = errno;
394  edm::LogWarning("LocalFileSystem::isLocalPath()") << "Cannot statfs('" << fullpath.get() << "' alias '" << path
395  << "'): " << strerror(nerr) << " (error " << nerr << ")";
396  return false;
397  }
398 
399  std::vector<std::string> prev_paths;
400  FSInfo *m = findMount(fullpath.get(), &sfs, &s, prev_paths);
401  return m ? m->local : false;
402 }
403 
422 std::pair<std::string, std::string> LocalFileSystem::findCachePath(const std::vector<std::string> &paths,
423  double minFreeSpace) const {
424  struct stat s;
425  struct statfs sfs;
426  std::ostringstream warningst;
427  warningst << "Cannot use lazy-download because:\n";
428 
429  for (size_t i = 0, e = paths.size(); i < e; ++i) {
430  const char *inpath = paths[i].c_str();
431  const char *path = inpath;
432 
433  if (*path == '$') {
434  char *p = std::getenv(path + 1);
435  if (p && *p)
436  path = p;
437  else if (!strcmp(path, "$TMPDIR"))
438  path = "/tmp";
439  }
440 
441  std::unique_ptr<char, decltype(std::free) *> fullpath{realpath(path, nullptr), std::free};
442  if (!fullpath)
443  fullpath.reset(strdup(path));
444 
445 #if 0
446  std::cerr /* edm::LogInfo("LocalFileSystem") */
447  << "Checking if '" << fullpath.get() << "', from '"
448  << inpath << "' is valid cache path with "
449  << minFreeSpace << " free space" << std::endl;
450 #endif
451 
452  if (lstat(fullpath.get(), &s) < 0) {
453  int nerr = errno;
454  if (nerr != ENOENT && nerr != EACCES)
455  edm::LogWarning("LocalFileSystem::findCachePath()")
456  << "Cannot lstat('" << fullpath.get() << "', from '" << inpath << "'): " << strerror(nerr) << " (error "
457  << nerr << ")";
458  continue;
459  }
460 
461  if (statfs(fullpath.get(), &sfs) < 0) {
462  int nerr = errno;
463  edm::LogWarning("LocalFileSystem::findCachePath()")
464  << "Cannot statfs('" << fullpath.get() << "', from '" << inpath << "'): " << strerror(nerr) << " (error "
465  << nerr << ")";
466  continue;
467  }
468 
469  std::vector<std::string> prev_paths;
470  FSInfo *m = findMount(fullpath.get(), &sfs, &s, prev_paths);
471 #if 0
472  std::cerr /* edm::LogInfo("LocalFileSystem") */
473  << "Candidate '" << fullpath.get() << "': "
474  << "found=" << (m ? 1 : 0)
475  << " local=" << (m && m->local)
476  << " free=" << (m ? m->freespc : 0)
477  << " access=" << access(fullpath.get(), W_OK)
478  << std::endl;
479 #endif
480 
481  if (m && m->local && m->freespc >= minFreeSpace && access(fullpath.get(), W_OK) == 0) {
482  return std::make_pair(std::string(fullpath.get()), std::string());
483  } else if (m) {
484  if (!m->local) {
485  warningst << "- The mount " << fullpath.get() << " is not local.\n";
486  } else if (m->freespc < minFreeSpace) {
487  warningst << " - The mount at " << fullpath.get() << " has only " << m->freespc << " GB free; a minumum of "
488  << minFreeSpace << " GB is required.\n";
489  } else if (access(fullpath.get(), W_OK)) {
490  warningst << " - The process has no permission to write into " << fullpath.get() << "\n";
491  }
492  }
493  }
494 
495  std::string warning_str = warningst.str();
496  if (!warning_str.empty()) {
497  warning_str = warning_str.substr(0, warning_str.size() - 2);
498  }
499 
500  return std::make_pair(std::string(), std::move(warning_str));
501 }
502 
505  if (readFSTypes() < 0)
506  return;
507 
508  if (initFSList() < 0)
509  return;
510 }
511 
514  for (size_t i = 0, e = fs_.size(); i < e; ++i)
515  free(fs_[i]);
516 }
bool isLocalPath(const std::string &path) const
std::vector< FSInfo * > fs_
FSInfo * findMount(const char *path, struct statfs *sfs, struct stat *s, std::vector< std::string > &) const
ret
prodAgent to be discontinued
std::vector< std::string > fstypes_
Information about file systems on this node.
Log< level::Error, false > LogError
assert(be >=bs)
A arg
Definition: Factorize.h:31
void free(void *ptr) noexcept
ALPAKA_FN_ACC ALPAKA_FN_INLINE uint32_t ix(uint32_t id)
void * malloc(size_t size) noexcept
int statFSInfo(FSInfo *i) const
Log< level::Warning, false > LogWarning
std::pair< std::string, std::string > findCachePath(const std::vector< std::string > &paths, double minFreeSpace) const
def move(src, dest)
Definition: eostools.py:511