spu Namespace Reference

Functions
void	create_dir (char *, int)
FILE *	create_file (char *, int)
int	def (FILE *, FILE *, int)
int	inf (FILE *, FILE *)
int	is_end_of_archive (const char *)
void	md5_File (std::string, char *)
int	parseoct (const char *, size_t)
void	Untar (FILE *, const char *)
int	Unzip (std::string, std::string)
int	verify_checksum (const char *)
void	zerr (int)

Function Documentation

create_dir()

void spu::create_dir	(	char *	pathname,
		int	mode
	)

Definition at line 213 of file SherpackUtilities.cc.

References eostools::mkdir(), ALCARECOPromptCalibProdSiPixelAli0T_cff::mode, AlCaHLTBitMon_ParallelJobs::p, BTVHLTOfflineSource_cfi::pathname, and submitPVResolutionJobs::stderr.

Referenced by create_file(), and Untar().

                                            {
    char *p;
    int r;

    /* Strip trailing '/' */
    if (pathname[strlen(pathname) - 1] == '/')
      pathname[strlen(pathname) - 1] = '\0';

    /* Try creating the directory. */
    r = mkdir(pathname, mode);

    if (r != 0) {
      /* On failure, try creating parent directory. */
      p = strrchr(pathname, '/');
      if (p != nullptr) {
        *p = '\0';
        create_dir(pathname, 0755);
        *p = '/';
        r = mkdir(pathname, mode);
      }
    }
    if (r != 0)
      fprintf(stderr, "Could not create directory %s\n", pathname);
  }

create_file()

FILE * spu::create_file	(	char *	pathname,
		int	mode
	)

Definition at line 239 of file SherpackUtilities.cc.

References create_dir(), f, AlCaHLTBitMon_ParallelJobs::p, and BTVHLTOfflineSource_cfi::pathname.

Referenced by Untar().

                                              {
    FILE *f;
    f = fopen(pathname, "w+");
    if (f == nullptr) {
      /* Try creating parent dir and then creating file. */
      char *p = strrchr(pathname, '/');
      if (p != nullptr) {
        *p = '\0';
        create_dir(pathname, 0755);
        *p = '/';
        f = fopen(pathname, "w+");
      }
    }
    return (f);
  }

def()

int spu::def	(	FILE *	source,
		FILE *	dest,
		int	level
	)

Definition at line 15 of file SherpackUtilities.cc.

References cms::cuda::assert(), CHUNK, mps_fire::dest, recoMuon::in, personalPlayback::level, MillePedeFileConverter_cfg::out, runTheMatrix::ret, source, and funct::void.

Referenced by FWProxyBuilderConfiguration::assertParam(), BOOST_PYTHON_MODULE(), CSCOfflineClient::dqmEndJob(), DTReadOutMapping::expandMap(), DTExpandMap::expandSteering(), MonRunOutcomeDef::fetchAllDefs(), edm::service::ThreadSafeLogMessageLoggerScribe::getAparameter(), popcon::EcalSRPHandler::getNewObjects(), evf::EvFDaqDirector::getNFilesFromEoLS(), FWTableWidget::handleResize(), MagCylinder::MagCylinder(), MultipleAlgoIterator::offsetCorrectJets(), ReflectedIterator::offsetCorrectJets(), ParametrizedSubtractor::offsetCorrectJets(), operator<<(), hitfit::operator<<(), PYBIND11_MODULE(), l1t::PYBIND11_MODULE(), readCfgParameter(), JME::JetResolutionObject::Record::Record(), and L1DataEmulDigi::reset().

                                               {
    int ret, flush;
    unsigned have;
    z_stream strm;
    unsigned char in[CHUNK];
    unsigned char out[CHUNK];

    /* allocate deflate state */
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    ret = deflateInit(&strm, level);
    if (ret != Z_OK)
      return ret;

    /* compress until end of file */
    do {
      strm.avail_in = fread(in, 1, CHUNK, source);
      if (ferror(source)) {
        (void)deflateEnd(&strm);
        return Z_ERRNO;
      }
      flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
      strm.next_in = in;

      /* run deflate() on input until output buffer not full, finish
           compression if all of source has been read in */
      do {
        strm.avail_out = CHUNK;
        strm.next_out = out;
        ret = deflate(&strm, flush);   /* no bad return value */
        assert(ret != Z_STREAM_ERROR); /* state not clobbered */
        have = CHUNK - strm.avail_out;
        if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
          (void)deflateEnd(&strm);
          return Z_ERRNO;
        }
      } while (strm.avail_out == 0);
      assert(strm.avail_in == 0); /* all input will be used */

      /* done when last data in file processed */
    } while (flush != Z_FINISH);
    assert(ret == Z_STREAM_END); /* stream will be complete */

    /* clean up and return */
    (void)deflateEnd(&strm);
    return Z_OK;
  }

inf()

int spu::inf	(	FILE *	source,
		FILE *	dest
	)

Definition at line 70 of file SherpackUtilities.cc.

References cms::cuda::assert(), CHUNK, mps_fire::dest, recoMuon::in, MillePedeFileConverter_cfg::out, runTheMatrix::ret, source, and funct::void.

Referenced by Unzip().

                                    {
    int ret;
    unsigned have;
    z_stream strm;
    unsigned char in[CHUNK];
    unsigned char out[CHUNK];

    /* allocate inflate state */
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.avail_in = 0;
    strm.next_in = Z_NULL;
    //~ ret = inflateInit(&strm,15);
    ret = inflateInit2(&strm, (16 + MAX_WBITS));
    if (ret != Z_OK)
      return ret;

    /* decompress until deflate stream ends or end of file */
    do {
      strm.avail_in = fread(in, 1, CHUNK, source);
      if (ferror(source)) {
        (void)inflateEnd(&strm);
        return Z_ERRNO;
      }
      if (strm.avail_in == 0)
        break;
      strm.next_in = in;

      /* run inflate() on input until output buffer not full */
      do {
        strm.avail_out = CHUNK;
        strm.next_out = out;
        ret = inflate(&strm, Z_NO_FLUSH);
        assert(ret != Z_STREAM_ERROR); /* state not clobbered */
        switch (ret) {
          case Z_NEED_DICT:
            ret = Z_DATA_ERROR;
            [[fallthrough]];
          case Z_DATA_ERROR:
          case Z_MEM_ERROR:
            (void)inflateEnd(&strm);
            return ret;
        }
        have = CHUNK - strm.avail_out;
        if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
          (void)inflateEnd(&strm);
          return Z_ERRNO;
        }
      } while (strm.avail_out == 0);

      /* done when inflate() says it's done */
    } while (ret != Z_STREAM_END);

    /* clean up and return */
    (void)inflateEnd(&strm);
    return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
  }

is_end_of_archive()

int spu::is_end_of_archive

(

const char *

p

)

Definition at line 204 of file SherpackUtilities.cc.

References dqmiodumpmetadata::n, and AlCaHLTBitMon_ParallelJobs::p.

Referenced by Untar().

                                       {
    int n;
    for (n = 511; n >= 0; --n)
      if (p[n] != '\0')
        return (0);
    return (1);
  }

md5_File()

void spu::md5_File	(	std::string	filename,
		char *	result
	)

Definition at line 487 of file SherpackUtilities.cc.

References edmScanValgrind::buffer, EVP_MD_CTX_free, EVP_MD_CTX_new, ztee::fd, corrVsCorr::filename, dqmdumpme::k, cms::openssl_init(), fileinputsource_cfi::read, mps_fire::result, and createJobs::tmp.

Referenced by spf::SherpackFetcher::Fetch().

                                                  {
    char buffer[4096];
    cms::openssl_init();
    EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
    const EVP_MD *md = EVP_get_digestbyname("MD5");
    EVP_DigestInit_ex(mdctx, md, nullptr);

    //Open File
    int fd = open(filename.c_str(), O_RDONLY);
    int nb_read;
    while ((nb_read = read(fd, buffer, 4096 - 1))) {
      EVP_DigestUpdate(mdctx, buffer, nb_read);
      memset(buffer, 0, 4096);
    }

    unsigned int md_len = 0;
    unsigned char tmp[EVP_MAX_MD_SIZE];
    EVP_DigestFinal_ex(mdctx, tmp, &md_len);
    EVP_MD_CTX_free(mdctx);

    //Convert the result
    for (unsigned int k = 0; k < md_len; ++k) {
      sprintf(result + k * 2, "%02x", tmp[k]);
    }
  }

parseoct()

int spu::parseoct	(	const char *	p,
		size_t	n
	)

Definition at line 187 of file SherpackUtilities.cc.

References mps_fire::i, dqmiodumpmetadata::n, and AlCaHLTBitMon_ParallelJobs::p.

Referenced by Untar(), and verify_checksum().

                                        {
    int i = 0;

    while (*p < '0' || *p > '7') {
      ++p;
      --n;
    }
    while (*p >= '0' && *p <= '7' && n > 0) {
      i *= 8;
      i += *p - '0';
      ++p;
      --n;
    }
    return (i);
  }

Untar()

void spu::Untar	(	FILE *	a,
		const char *	path
	)

Definition at line 269 of file SherpackUtilities.cc.

References a, create_dir(), create_file(), f, is_end_of_archive(), dqmdumpme::k, parseoct(), castor_dqm_sourceclient_file_cfg::path, submitPVResolutionJobs::stderr, and verify_checksum().

Referenced by spf::SherpackFetcher::Fetch().

                                        {
    bool longpathname = false;
    bool longlinkname = false;
    char newlongpathname[512];
    char newlonglinkname[512];
    char buff[512];
    FILE *f = nullptr;
    size_t bytes_read;
    int filesize;

    printf("Extracting from %s\n", path);
    for (;;) {
      bytes_read = fread(buff, 1, 512, a);
      if (bytes_read < 512) {
        fprintf(stderr, "Short read on %s: expected 512, got %d\n", path, (int)bytes_read);
        return;
      }
      if (is_end_of_archive(buff)) {
        printf("End of %s\n", path);
        return;
      }
      if (!verify_checksum(buff)) {
        fprintf(stderr, "Checksum failure\n");
        return;
      }
      filesize = parseoct(buff + 124, 12);
      //        printf("%c %d\n",buff[156],filesize);
      switch (buff[156]) {
        case '1':
          printf(" Ignoring hardlink %s\n", buff);
          break;
        case '2':
          if (longpathname && longlinkname) {
            longlinkname = false;
            longpathname = false;
            printf(" Extracting symlink %s\n", newlongpathname);
            symlink(newlonglinkname, newlongpathname);
          } else if (longpathname) {
            longpathname = false;
            printf(" Extracting symlink %s\n", newlongpathname);
            symlink(buff + 157, newlongpathname);
          } else if (longlinkname) {
            longlinkname = false;
            printf(" Extracting symlink %s\n", buff);
            symlink(newlonglinkname, buff);
          } else {
            printf(" Extracting symlink %s\n", buff);
            symlink(buff + 157, buff);
          }
          break;
        case '3':
          printf(" Ignoring character device %s\n", buff);
          break;
        case '4':
          printf(" Ignoring block device %s\n", buff);
          break;
        case '5':
          if (!longpathname) {
            int endposition = -1;
            for (int k = 99; k >= 0; k--) {
              if (buff[k] == '\0')
                endposition = k;
            }
            if (endposition == -1) {
              //~ printf("OLDNAME : %s\n",buff);
              longpathname = true;
              for (int k = 0; k < 100; k++) {
                newlongpathname[k] = buff[k];
              }
              newlongpathname[100] = '\0';
              //~ printf("NEWNAME : %s\n",newlongpathname);
            }
          }

          if (longpathname) {
            printf(" Extracting dir %s\n", newlongpathname);
            create_dir(newlongpathname, parseoct(buff + 100, 8));
            longpathname = false;
          } else {
            printf(" Extracting dir %s\n", buff);
            create_dir(buff, parseoct(buff + 100, 8));
          }
          //~ printf(" Extracting dir %s\n", buff);
          //~ create_dir(buff, parseoct(buff + 100, 8));
          filesize = 0;
          break;
        case '6':
          printf(" Ignoring FIFO %s\n", buff);
          break;
        case 'L':
          longpathname = true;
          //~ printf(" Long Filename found 0 %s\n", buff);
          //~ printf(" Long Filename found 100 %s\n", buff+100);
          //~ printf(" Long Filename found 108 %s\n", buff+108);
          //~ printf(" Long Filename found 116 %s\n", buff+116);
          //~ printf(" Long Filename found 124 %s\n", buff+124);
          //~ printf(" Long Filename found 136 %s\n", buff+136);
          //~ printf(" Long Filename found 148 %s\n", buff+148);
          //~ printf(" Long Filename found 156 %s\n", buff+156);
          //~ printf(" Long Filename found 157 %s\n", buff+157);
          //~ printf(" Long Filename found 158 %s\n", buff+158);
          //~ printf(" Long Filename found 159 %s\n", buff+159);
          //~ printf(" Long Filename found 257 %s\n", buff+257);
          //~ printf(" Long Filename found 263 %s\n", buff+263);
          //~ printf(" Long Filename found 265 %s\n", buff+265);
          //~ printf(" Long Filename found 297 %s\n", buff+297);
          //~ printf(" Long Filename found 329 %s\n", buff+329);
          //~ printf(" Long Filename found 337 %s\n", buff+337);
          //~ printf(" Long Filename found 345 %s\n", buff+345);
          //~ printf(" Long Filename found 346 %s\n", buff+346);
          //~ printf(" Long Filename found 347 %s\n", buff+347);
          break;

        case 'K':
          longlinkname = true;
          break;

        default:
          if (!longpathname) {
            int endposition = -1;
            for (int k = 99; k >= 0; k--) {
              if (buff[k] == '\0')
                endposition = k;
            }
            if (endposition == -1) {
              //~ printf("OLDNAME : %s\n",buff);
              longpathname = true;
              for (int k = 0; k < 100; k++) {
                newlongpathname[k] = buff[k];
              }
              newlongpathname[100] = '\0';
              //~ printf("NEWNAME : %s\n",newlongpathname);
            }
          }
          if (longpathname) {
            printf(" Extracting file %s\n", newlongpathname);
            f = create_file(newlongpathname, parseoct(buff + 100, 8));
            longpathname = false;
          } else {
            printf(" Extracting file %s\n", buff);
            f = create_file(buff, parseoct(buff + 100, 8));
          }
          break;
      }

      if (longlinkname || longpathname) {
        if (buff[156] == 'K') {
          for (int ll = 0; ll < 512; ll++) {
            printf("%c", buff[ll]);
          }
          printf("\n");
          bytes_read = fread(buff, 1, 512, a);
          for (int ll = 0; ll < 512; ll++) {
            printf("%c", buff[ll]);
          }
          printf("\n");
          for (int k = 0; k < filesize; k++) {
            newlonglinkname[k] = buff[k];
          }
          newlonglinkname[filesize] = '\0';
          for (int k = filesize + 1; k < 512; k++) {
            newlonglinkname[k] = '0';
          }
          //~ printf("NEW LinkNAME: %s\n",newlonglinkname);
        } else if (buff[156] == 'L') {
          bytes_read = fread(buff, 1, 512, a);
          for (int k = 0; k < filesize; k++) {
            newlongpathname[k] = buff[k];
          }
          newlongpathname[filesize] = '\0';
          for (int k = filesize + 1; k < 512; k++) {
            newlongpathname[k] = '0';
          }
          //~ printf("NEW FILENAME: %s\n",newlongpathname);
        }
      }

      //~
      //~ if (longpathname) {
      //~ bytes_read = fread(buff, 1, 512, a);
      //~ for (int k=0; k<filesize; k++){
      //~ newlongpathname[k]=buff[k];
      //~ }
      //~ newlongpathname[filesize]='\0';
      //~ for (int k=filesize+1; k<512; k++){
      //~ newlongpathname[k]='0';
      //~ }
      //~ printf("NEW FILENAME: %s\n",newlongpathname);
      //~
      //~ }
      //~ else if (!longpathname && !longlinkname) {
      if (!longpathname && !longlinkname) {
        while (filesize > 0) {
          bytes_read = fread(buff, 1, 512, a);
          if (bytes_read < 512) {
            fprintf(stderr, "Short read on %s: Expected 512, got %d\n", path, (int)bytes_read);
            return;
          }
          if (filesize < 512)
            bytes_read = filesize;
          if (f != nullptr) {
            if (fwrite(buff, 1, bytes_read, f) != bytes_read) {
              fprintf(stderr, "Failed write\n");
              fclose(f);
              f = nullptr;
            }
          }
          filesize -= bytes_read;
        }
        if (f != nullptr) {
          fclose(f);
          f = nullptr;
        }
      }
    }
  }

Unzip()

int spu::Unzip	(	std::string	infile,
		std::string	outfile
	)

Definition at line 154 of file SherpackUtilities.cc.

References recoMuon::in, inf(), timingPdfMaker::infile, MillePedeFileConverter_cfg::out, timingPdfMaker::outfile, runTheMatrix::ret, SET_BINARY_MODE, and zerr().

Referenced by spf::SherpackFetcher::Fetch().

                                                 {
    const char *tmpdir = std::getenv("TMPDIR");
    if (tmpdir && (strlen(tmpdir) > 50)) {
      setenv("TMPDIR", "/tmp", true);
    }
    int ret;
    FILE *in = fopen(infile.c_str(), "r");
    if (!in)
      return -1;
    FILE *out = fopen(outfile.c_str(), "w");
    if (!out)
      return -2;
    /* avoid end-of-line conversions */
    SET_BINARY_MODE(in);
    SET_BINARY_MODE(out);

    ret = inf(in, out);
    if (ret != Z_OK)
      zerr(ret);

    fclose(in);
    fclose(out);
    return ret;
  }

verify_checksum()

int spu::verify_checksum

(

const char *

p

)

Definition at line 256 of file SherpackUtilities.cc.

References dqmiodumpmetadata::n, AlCaHLTBitMon_ParallelJobs::p, and parseoct().

Referenced by Untar().

                                     {
    int n, u = 0;
    for (n = 0; n < 512; ++n) {
      if (n < 148 || n > 155)
        /* Standard tar checksum adds unsigned bytes. */
        u += ((unsigned char *)p)[n];
      else
        u += 0x20;
    }
    return (u == parseoct(p + 148, 8));
  }

zerr()

void spu::zerr

(

int

ret

)

Definition at line 130 of file SherpackUtilities.cc.

References runTheMatrix::ret, submitPVResolutionJobs::stderr, cmsLHEtoEOSManager::stdin, and mps_setup::stdout.

Referenced by MultiFileBlob::expand(), MultiFileBlob::finalized(), FileBlob::getUncompressedBlob(), TkMSParameterizationBuilder::produce(), FileBlob::read(), Unzip(), FileBlob::write(), and MillePedeFileExtractor::writeGzipped().

                     {
    fputs("zpipe: ", stderr);
    switch (ret) {
      case Z_ERRNO:
        if (ferror(stdin))
          fputs("error reading stdin\n", stderr);
        if (ferror(stdout))
          fputs("error writing stdout\n", stderr);
        break;
      case Z_STREAM_ERROR:
        fputs("invalid compression level\n", stderr);
        break;
      case Z_DATA_ERROR:
        fputs("invalid or incomplete deflate data\n", stderr);
        break;
      case Z_MEM_ERROR:
        fputs("out of memory\n", stderr);
        break;
      case Z_VERSION_ERROR:
        fputs("zlib version mismatch!\n", stderr);
    }
  }