CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
LocalCacheFile.cc
Go to the documentation of this file.
3 #include <utility>
4 #include <iostream>
5 #include <cstdlib>
6 #include <cstring>
7 #include <unistd.h>
8 #include <sys/mman.h>
9 #include <cerrno>
10 #include <sstream>
11 
12 using namespace edm::storage;
13 
14 static constexpr IOOffset CHUNK_SIZE = 128 * 1024 * 1024;
15 
16 static void nowrite(const std::string &why) {
17  cms::Exception ex("LocalCacheFile");
18  ex << "Cannot change file but operation '" << why << "' was called";
19  ex.addContext("LocalCacheFile::" + why + "()");
20  throw ex;
21 }
22 
23 LocalCacheFile::LocalCacheFile(std::unique_ptr<Storage> base, const std::string &tmpdir /* = "" */)
24  : image_(base->size()),
25  file_(),
26  storage_(std::move(base)),
27  closedFile_(false),
28  cacheCount_(0),
29  cacheTotal_((image_ + CHUNK_SIZE - 1) / CHUNK_SIZE) {
30  present_.resize(cacheTotal_, 0);
31 
32  std::string pattern(tmpdir);
33  if (pattern.empty())
34  if (char *p = std::getenv("TMPDIR"))
35  pattern = p;
36  if (pattern.empty())
37  pattern = "/tmp";
38  pattern += "/cmssw-shadow-XXXXXX";
39 
40  std::vector<char> temp(pattern.c_str(), pattern.c_str() + pattern.size() + 1);
41  int fd = mkstemp(&temp[0]);
42  if (fd == -1) {
44  ex << "Cannot create temporary file '" << pattern << "': " << strerror(errno) << " (error " << errno << ")";
45  ex.addContext("LocalCacheFile::LocalCacheFile");
46  }
47 
48  unlink(&temp[0]);
49  file_ = std::make_unique<File>(fd);
50  file_->resize(image_);
51 }
52 
54 
57  end = std::min(end, image_);
58 
59  IOSize nread = 0;
61 
62  while (start < end) {
64  if (!present_[index]) {
65  void *window = mmap(nullptr, len, PROT_READ | PROT_WRITE, MAP_SHARED, file_->fd(), start);
66  if (window == MAP_FAILED) {
68  ex << "Unable to map a window of local cache file: " << strerror(errno) << " (error " << errno << ")";
69  ex.addContext("LocalCacheFile::cache()");
70  throw ex;
71  }
72 
73  try {
74  nread = storage_->read(window, len, start);
75  } catch (cms::Exception &e) {
76  munmap(window, len);
77  std::ostringstream ost;
78  ost << "Unable to cache " << len << " byte file segment at " << start << ": ";
80  ex.addContext("LocalCacheFile::cache()");
81  throw ex;
82  }
83 
84  munmap(window, len);
85 
86  if (nread != len) {
88  ex << "Unable to cache " << len << " byte file segment at " << start << ": got only " << nread << " bytes back";
89  ex.addContext("LocalCacheFile::cache()");
90  throw ex;
91  }
92 
93  present_[index] = 1;
94  ++cacheCount_;
95  if (cacheCount_ == cacheTotal_) {
96  storage_->close();
97  closedFile_ = true;
98  }
99  }
100 
101  start += len;
102  ++index;
103  }
104 }
105 
107  IOOffset here = file_->position();
108  cache(here, here + n);
109 
110  return file_->read(into, n);
111 }
112 
114  cache(pos, pos + n);
115  return file_->read(into, n, pos);
116 }
117 
119  IOOffset start = file_->position();
120  IOOffset end = start;
121  for (IOSize i = 0; i < n; ++i)
122  end += into[i].size();
123  cache(start, end);
124 
125  return file_->readv(into, n);
126 }
127 
129  for (IOSize i = 0; i < n; ++i) {
130  IOOffset start = into[i].offset();
131  IOOffset end = start + into[i].size();
132  cache(start, end);
133  }
134 
135  return storage_->readv(into, n);
136 }
137 
138 IOSize LocalCacheFile::write(const void * /*from*/, IOSize) {
139  nowrite("write");
140  return 0;
141 }
142 
143 IOSize LocalCacheFile::write(const void * /*from*/, IOSize, IOOffset /*pos*/) {
144  nowrite("write");
145  return 0;
146 }
147 
149  nowrite("writev");
150  return 0;
151 }
152 
154  nowrite("writev");
155  return 0;
156 }
157 
158 IOOffset LocalCacheFile::position(IOOffset offset, Relative whence) { return file_->position(offset, whence); }
159 
160 void LocalCacheFile::resize(IOOffset /*size*/) { nowrite("resize"); }
161 
162 void LocalCacheFile::flush(void) { nowrite("flush"); }
163 
165  if (!closedFile_) {
166  storage_->close();
167  }
168  file_->close();
169 }
170 
172  for (IOSize i = 0; i < n; ++i) {
173  IOOffset start = what[i].offset();
174  IOOffset end = start + what[i].size();
175  cache(start, end);
176  }
177 
178  return file_->prefetch(what, n);
179 }
size
Write out results.
Definition: start.py:1
int64_t IOOffset
Definition: IOTypes.h:20
IOSize write(const void *from, IOSize n) override
base
Main Program
Definition: newFWLiteAna.py:92
edm::propagate_const< std::unique_ptr< Storage > > storage_
IOSize writev(const IOBuffer *from, IOSize n) override
void cache(IOOffset start, IOOffset end)
IOOffset offset() const
Definition: IOPosBuffer.h:41
virtual IOOffset size() const
Definition: Storage.cc:509
edm::propagate_const< std::unique_ptr< File > > file_
def window(xmin, xmax, ymin, ymax, x=0, y=0, width=100, height=100, xlogbase=None, ylogbase=None, minusInfinity=-1000, flipx=False, flipy=True)
Definition: svgfig.py:643
bool prefetch(const IOPosBuffer *what, IOSize n) override
size_t IOSize
Definition: IOTypes.h:15
IOSize readv(IOBuffer *into, IOSize n) override
void addContext(std::string const &context)
Definition: Exception.cc:169
virtual IOOffset position() const
Definition: Storage.cc:504
LocalCacheFile(std::unique_ptr< Storage > base, const std::string &tmpdir="")
static constexpr IOOffset CHUNK_SIZE
void resize(IOOffset size) override
std::vector< char > present_
fd
Definition: ztee.py:136
def move(src, dest)
Definition: eostools.py:511
static void nowrite(const std::string &why)