BinaryArchiveBase.cc

Go to the documentation of this file.

#include <cassert>
#include <cctype>
#include <cerrno>
#include <climits>
#include <cstdlib>
#include <cstring>

#include "Alignment/Geners/interface/BinaryArchiveBase.hh"
#include "Alignment/Geners/interface/CatalogIO.hh"
#include "Alignment/Geners/interface/binaryIO.hh"

#ifdef GENERS_BINARY_ARCHIVE_FORMAT_ID
#undef GENERS_BINARY_ARCHIVE_FORMAT_ID
#endif
#define GENERS_BINARY_ARCHIVE_FORMAT_ID (0x1f2e3d4c)

static bool parse_unsigned(std::ostringstream &err, const char *c, unsigned *result) {
  char *endptr;
  errno = 0;
  const unsigned long value = strtoul(c, &endptr, 0);
  if (errno || *endptr != '\0') {
    err << "expected an unsigned integer, got \"" << c << '"';
    if (errno)
      err << ", " << strerror(errno);
    return false;
  }
  if (value > UINT_MAX) {
    err << "unsigned value \"" << c << "\" is out of range";
    return false;
  }
  *result = value;
  return true;
}

static bool parse_int(std::ostringstream &err, const char *c, int *result) {
  char *endptr;
  errno = 0;
  const long value = strtol(c, &endptr, 0);
  if (errno || *endptr != '\0') {
    err << "expected an integer, got \"" << c << '"';
    if (errno)
      err << ", " << strerror(errno);
    return false;
  }
  if (value < INT_MIN || value > INT_MAX) {
    err << "integer value \"" << c << "\" is out of range";
    return false;
  }
  *result = value;
  return true;
}

namespace gs {
  BinaryArchiveBase::BinaryArchiveBase(const char *name, const char *mode)
      : AbsArchive(name),
        mode_(parseMode(mode)),
        errorStream_(nullptr),
        cStream_(nullptr),
        catalog_(nullptr),
        storedEntryId_(nullptr),
        storedLocationId_(nullptr),
        catalogIsSet_(false),
        addCatalogToData_(false) {
    CStringStream::CompressionMode m = CStringStream::NOT_COMPRESSED;
    int compressionLevel = -1;
    unsigned minSizeToCompress = 1024U;
    unsigned bufSize = 1048576U;  // 1024*1024

    std::ostringstream err;
    modeIsValid_ =
        parseArchiveOptions(err, mode, &m, &compressionLevel, &minSizeToCompress, &bufSize, &addCatalogToData_);
    if (modeIsValid_)
      cStream_ = new CStringStream(m, compressionLevel, minSizeToCompress, bufSize);
    else {
      errorStream() << "In BinaryArchiveBase constructor: "
                    << "invalid archive opening mode \"" << mode << '"';
      const std::string &errInfo = err.str();
      if (!errInfo.empty())
        errorStream() << ": " << errInfo;
    }
  }

  void BinaryArchiveBase::releaseClassIds() {
    delete storedEntryId_;
    storedEntryId_ = nullptr;
    delete storedLocationId_;
    storedLocationId_ = nullptr;
  }

  BinaryArchiveBase::~BinaryArchiveBase() {
    releaseClassIds();
    delete errorStream_;
    delete catalog_;
    delete cStream_;
  }

  void BinaryArchiveBase::writeHeader(std::ostream &os) {
    const unsigned format = GENERS_BINARY_ARCHIVE_FORMAT_ID;
    write_pod(os, format);

    // Write some other info
    const unsigned multiplex = addCatalogToData_ ? 1 : 0;
    const unsigned sizeoflong = sizeof(long);
    const unsigned infoword = (sizeoflong << 1) | multiplex;
    write_pod(os, infoword);

    if (multiplex) {
      // Write class ids for CatalogEntry and ItemLocation
      releaseClassIds();
      storedEntryId_ = new ClassId(ClassId::makeId<CatalogEntry>());
      storedEntryId_->write(os);
      storedLocationId_ = new ClassId(ClassId::makeId<ItemLocation>());
      storedLocationId_->write(os);
    }
  }

  bool BinaryArchiveBase::readHeader(std::istream &is) {
    const unsigned expectedFormat = GENERS_BINARY_ARCHIVE_FORMAT_ID;
    is.seekg(0, std::ios_base::beg);
    unsigned format = 0;
    read_pod(is, &format);
    if (format != expectedFormat)
      return false;

    unsigned infoword = 0xffffffff;
    read_pod(is, &infoword);
    const unsigned multiplex = infoword & 0x1U;
    const unsigned sizeoflong = infoword >> 1;

    // The following check will make sure that we are not reading
    // an archive created on a 32-bit machine with a 64-bit system
    // (and otherwise)
    if (sizeoflong != sizeof(long))
      return false;

    addCatalogToData_ = multiplex;
    if (addCatalogToData_) {
      releaseClassIds();
      storedEntryId_ = new ClassId(is, 1);
      storedLocationId_ = new ClassId(is, 1);

      // Can't open this archive for update if the above class ids
      // are obsolete -- otherwise we will loose the capability to
      // restore the catalog
      if (mode_ & std::ios_base::out) {
        const ClassId &entryId = ClassId::makeId<CatalogEntry>();
        const ClassId &locId = ClassId::makeId<ItemLocation>();
        if (entryId != *storedEntryId_ || locId != *storedLocationId_)
          throw IOInvalidData(
              "In gs::BinaryArchiveBase::readHeader: this "
              "archive can no longer be open for update as it was "
              "created using an older version of I/O software");
      }
    }
    return !is.fail();
  }

  void BinaryArchiveBase::openDataFile(std::fstream &stream, const char *filename) {
    assert(filename);
    if (stream.is_open())
      stream.close();
    stream.clear();
    stream.open(filename, mode_);
    if (!stream.is_open())
      throw IOOpeningFailure("gs::BinaryArchiveBase::openDataFile", filename);

    // Do we need to write the header out or to read it in?
    bool writeHead = false;
    if (mode_ & std::ios_base::out) {
      if (mode_ & std::ios_base::trunc)
        writeHead = true;
      else if (isEmptyFile(stream))
        writeHead = true;
    }

    if (writeHead) {
      writeHeader(stream);
      if (stream.fail()) {
        stream.close();
        std::string e =
            "In gs::BinaryArchiveBase::openDataFile: "
            "failed to write archive header to file \"";
        e += filename;
        e += "\"";
        throw IOWriteFailure(e);
      }
    } else {
      if (!readHeader(stream)) {
        const bool failed = stream.fail();
        stream.close();
        std::string e = "In gs::BinaryArchiveBase::openDataFile: ";
        if (failed) {
          e += "could not read archive header from file \"";
          e += filename;
          e += "\"";
          throw IOReadFailure(e);
        } else {
          e += "no valid archive header in file \"";
          e += filename;
          e += "\"";
          throw IOInvalidData(e);
        }
      }
    }
  }

  void BinaryArchiveBase::setCatalog(AbsCatalog *c) {
    if (c) {
      assert(!catalogIsSet_);
      catalogIsSet_ = true;
    }
    delete catalog_;
    catalog_ = c;
  }

  void BinaryArchiveBase::itemSearch(const SearchSpecifier &namePattern,
                                     const SearchSpecifier &categoryPattern,
                                     std::vector<unsigned long long> *idsFound) const {
    if (catalog_)
      catalog_->search(namePattern, categoryPattern, idsFound);
    else {
      assert(idsFound);
      idsFound->clear();
    }
  }

  bool BinaryArchiveBase::parseArchiveOptions(std::ostringstream &err,
                                              const char *modeIn,
                                              CStringStream::CompressionMode *m,
                                              int *compressionLevel,
                                              unsigned *minSizeToCompress,
                                              unsigned *bufSize,
                                              bool *multiplexCatalog) {
    if (!modeIn)
      return true;
    std::string cmode(modeIn ? modeIn : "");
    if (cmode.empty())
      return true;
    char *mode = const_cast<char *>(cmode.c_str());

    unsigned cnt = 0;
    for (char *opt = strtok(mode, ":"); opt; opt = strtok(nullptr, ":"), ++cnt) {
      // Skip the first word -- this is the file opening mode
      if (!cnt)
        continue;
      char *eq = strchr(opt, '=');
      if (eq) {
        // Get rid of spaces around option name
        char *optname = opt;
        while (isspace(*optname) && optname < eq)
          ++optname;
        if (optname == eq) {
          err << "invalid binary archive option \"\"";
          return false;
        }
        char *optend = eq - 1;
        while (isspace(*optend))
          --optend;
        ++optend;
        *optend = '\0';

        // Get rid of spaces around option value
        char *optval = eq + 1;
        while (*optval && isspace(*optval))
          ++optval;
        if (!*optval) {
          err << "invalid binary archive option value \"\"";
          return false;
        }
        char *valend = opt + strlen(opt) - 1;
        while (isspace(*valend))
          --valend;
        ++valend;
        *valend = '\0';
        if (strlen(optval) == 0) {
          err << "invalid binary archive option value \"\"";
          return false;
        }

        // Go over possible options
        if (!strcasecmp(optname, "z")) {
          // Compression type
          if (!CStringStream::getCompressionModeByName(optval, m)) {
            err << "invalid compression type \"" << optval << '"';
            return false;
          }
        } else if (!strcasecmp(optname, "cl")) {
          // Compression level
          if (!parse_int(err, optval, compressionLevel))
            return false;
          if (*compressionLevel < -1 || *compressionLevel > 9) {
            err << "compression level is out of range";
            return false;
          }
        } else if (!strcasecmp(optname, "cb")) {
          // Compression buffer size
          if (!parse_unsigned(err, optval, bufSize))
            return false;
        } else if (!strcasecmp(optname, "cm")) {
          // Compression minimum size
          if (!parse_unsigned(err, optval, minSizeToCompress))
            return false;
        } else if (!strcasecmp(optname, "cat")) {
          // Internal or external catalog
          if (optval[0] == 'i' || optval[0] == 'I')
            *multiplexCatalog = true;
          else if (optval[0] == 's' || optval[0] == 'S')
            *multiplexCatalog = false;
          else {
            err << "invalid catalog mode \"" << optval << '"';
            return false;
          }
        } else {
          // Unknown option
          err << "unrecognized binary archive option \"" << optname << '"';
          return false;
        }
      } else {
        err << "invalid binary archive option \"" << opt << '"';
        return false;
      }
    }
    return true;
  }

  std::ios_base::openmode BinaryArchiveBase::parseMode(const char *mode) {
    std::ios_base::openmode m = std::ios_base::binary;
    if (mode) {
      const unsigned len = strlen(mode);
      for (unsigned i = 0; i < len; ++i) {
        // Note that all characters other than 'r', 'w',
        // 'a', and '+' are basically ignored inside this cycle
        if (mode[i] == 'r')
          m |= std::ios_base::in;
        else if (mode[i] == 'w')
          m |= (std::ios_base::out | std::ios_base::trunc);
        else if (mode[i] == 'a')
          m |= (std::ios_base::out | std::ios_base::app);
        else if (mode[i] == '+')
          m |= (std::ios_base::in | std::ios_base::out);
        else if (mode[i] == ':')
          break;
      }
    }

    // Make sure that we are at least reading
    if (!(m & (std::ios_base::in | std::ios_base::out)))
      m |= std::ios_base::in;
    return m;
  }

  void BinaryArchiveBase::search(AbsReference &reference) {
    if (catalog_) {
      std::vector<unsigned long long> idlist;
      catalog_->search(reference.namePattern(), reference.categoryPattern(), &idlist);
      const unsigned long nfound = idlist.size();
      for (unsigned long i = 0; i < nfound; ++i) {
        std::shared_ptr<const CatalogEntry> pentry = catalog_->retrieveEntry(idlist[i]);
        if (reference.isIOCompatible(*pentry))
          addItemToReference(reference, idlist[i]);
      }
    }
  }

  bool BinaryArchiveBase::isEmptyFile(std::fstream &s) {
    s.seekg(0, std::ios_base::end);
    return s.tellg() == std::streampos(0);
  }

  std::istream &BinaryArchiveBase::inputStream(const unsigned long long id, long long *sz) {
    unsigned long long length = 0;
    unsigned compressionCode = 0;
    std::istream &is = plainInputStream(id, &compressionCode, &length);
    if (cStream_->compressionMode() == CStringStream::NOT_COMPRESSED) {
      if (sz)
        *sz = -1LL;
      return is;
    } else {
      cStream_->readCompressed(is, compressionCode, length);
      if (sz) {
        std::streamoff off = cStream_->tellp();
        *sz = off;
      }
      return *cStream_;
    }
  }

  std::ostream &BinaryArchiveBase::outputStream() { return plainOutputStream(); }

  std::ostream &BinaryArchiveBase::compressedStream(std::ostream &os) {
    if (cStream_->compressionMode() == CStringStream::NOT_COMPRESSED)
      return os;
    else {
      cStream_->reset();
      cStream_->setSink(os);
      return *cStream_;
    }
  }

  unsigned BinaryArchiveBase::flushCompressedRecord(std::ostream &) {
    CStringStream::CompressionMode m = cStream_->compressionMode();
    if (m != CStringStream::NOT_COMPRESSED) {
      cStream_->flush();
      m = cStream_->writeCompressed();
    }
    return static_cast<unsigned>(m);
  }
}  // namespace gs