]> cygwin.com Git - cygwin-apps/setup.git/commitdiff
Add support for ZStandard compression
authorAchim Gratz <Stromeko@Stromeko.DE>
Sun, 2 Sep 2018 16:41:24 +0000 (18:41 +0200)
committerJon Turney <jon.turney@dronecode.org.uk>
Sat, 13 Oct 2018 14:37:43 +0000 (15:37 +0100)
Add support for ZStandard compression for packages and .ini files

Makefile.am
compress.cc
compress_zstd.cc [new file with mode: 0644]
compress_zstd.h [new file with mode: 0644]
configure.ac
ini.h

index a4f9a12b9b274124510a9a221a4219a0026b2fe5..0ad4e5c86f26c4345290fdd12182417d92faadfc 100644 (file)
@@ -29,7 +29,7 @@ AM_YFLAGS = -d
 AM_LFLAGS = -8
 WINDRES = @WINDRES@
 AM_CPPFLAGS = -DLZMA_API_STATIC -I$(srcdir)/libgetopt++/include \
-              $(ZLIB_CFLAGS) $(LZMA_CFLAGS) $(LIBCRYPT_CFLAGS) $(LIBSOLV_CFLAGS)
+              $(ZLIB_CFLAGS) $(LZMA_CFLAGS) $(ZSTD_CFLAGS) $(LIBCRYPT_CFLAGS) $(LIBSOLV_CFLAGS)
 
 inilex_CXXFLAGS:=-Wno-sign-compare
 
@@ -100,6 +100,7 @@ inilint_SOURCES = \
 @SETUP@_LDADD = \
        libgetopt++/libgetopt++.la \
        $(LIBGCRYPT_LIBS) \
+       $(ZSTD_LIBS) \
        $(LZMA_LIBS) \
        $(BZ2_LIBS) \
        $(ZLIB_LIBS) \
@@ -124,6 +125,8 @@ inilint_SOURCES = \
        compress_gz.h \
        compress_xz.cc \
        compress_xz.h \
+       compress_zstd.cc \
+       compress_zstd.h \
        confirm.cc \
        confirm.h \
        ConnectionSetting.cc \
index 7052f96085ea55453b64f0d27b25f47da8fa932d..9ff41d385ac20d7ec7542de62d6daaaf8f247c20 100644 (file)
@@ -17,6 +17,7 @@
 #include "compress_gz.h"
 #include "compress_bz.h"
 #include "compress_xz.h"
+#include "compress_zstd.h"
 #include <string.h>
 
 /* In case you are wondering why the file magic is not in one place:
@@ -28,7 +29,7 @@
  * the class could test itself. 
  */
 
-#define longest_magic 14 /* lzma_alone */
+#define longest_magic 18 /* ZStandard longest frame header (magic is only 4 bytes) */
 
 io_stream *
 compress::decompress (io_stream * original)
@@ -49,6 +50,16 @@ compress::decompress (io_stream * original)
          delete rv;
          return NULL;
        }
+      else if (compress_zstd::is_zstd (magic, 18))
+       {
+         compress_zstd *rv = new compress_zstd (original);
+         if (!rv->error ())
+           return rv;
+         /* else */
+         rv->release_original();
+         delete rv;
+         return NULL;
+       }
       else if (memcmp (magic, "BZh", 3) == 0)
        {
          compress_bz *rv = new compress_bz (original);
diff --git a/compress_zstd.cc b/compress_zstd.cc
new file mode 100644 (file)
index 0000000..3588dbd
--- /dev/null
@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2018, Cygwin
+ *
+ *     This program is free software; you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation; either version 2 of the License, or
+ *     (at your option) any later version.
+ *
+ *     A copy of the GNU General Public License can be found at
+ *     http://www.gnu.org/
+ *
+ */
+
+#include "compress_zstd.h"
+
+#include <stdexcept>
+
+#include <errno.h>
+#include <memory.h>
+#include <malloc.h>
+
+/*
+ * Predicate: the stream is open for read.
+ */
+compress_zstd::compress_zstd (io_stream * parent)
+:
+  original(NULL),
+  owns_original(true),
+  lasterr(0)
+{
+  /* read only */
+  if (!parent || parent->error())
+    {
+      lasterr = EBADF;
+      return;
+    }
+  original = parent;
+
+  state = (struct private_data *)calloc(sizeof(*state), 1);
+  if (state == NULL)
+    {
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+
+  state->stream = ZSTD_createDStream();
+  if (state->stream == NULL)
+    {
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+  ZSTD_initDStream(state->stream);
+  state->out_block.size = state->out_block.pos  = state->out_pos = state->out_bsz = ZSTD_DStreamOutSize();
+  state->out_block.dst  = (unsigned char *)malloc(state->out_bsz);
+  if (state->out_block.dst == NULL)
+    {
+      free(state->out_block.dst);
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+  state->in_block.size = state->in_block.pos  = state->in_bsz = ZSTD_DStreamInSize();
+  state->in_block.src  = (unsigned char *)malloc(state->in_bsz);
+  state->in_pos  = 0;
+  if (state->in_block.src == NULL)
+    {
+      free(state->out_block.dst);
+      free((void*)state->in_block.src);
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+}
+
+ssize_t
+compress_zstd::read (void *buffer, size_t len)
+{
+  /* there is no recovery from a busted stream */
+  if (this->lasterr)
+    {
+      return -1;
+    }
+  if (len == 0)
+    {
+      return 0;
+    }
+
+  size_t lenRemaining = len;
+  size_t lenBuffered  = 0;
+  do
+    {
+      if (state->in_block.size > 0 && state->in_block.pos >= state->in_block.size)
+        {
+         /* no compressed data ready; read some more input */
+         state->in_block.size = state->in_bsz;
+         ssize_t got = this->original->read((void *)state->in_block.src, state->in_bsz);
+         if (got >= 0)
+           {
+             state->in_block.size = got;
+             state->in_block.pos = 0;
+           }
+         else
+           {
+             lasterr = EIO;
+             return -1;
+           }
+         continue;
+        }
+
+      if (state->out_pos < state->out_block.pos)
+       {
+         /* output buffer has unused data */
+         ssize_t tmplen = std::min (state->out_block.pos - state->out_pos, lenRemaining);
+         memcpy (&((char *)buffer)[lenBuffered], &((char *)state->out_block.dst)[state->out_pos], tmplen);
+         state->out_pos += tmplen;
+         lenBuffered    += tmplen;
+         lenRemaining   -= tmplen;
+         if (state->eof)
+           {
+             break;
+           }
+       }
+      else
+       {
+         if (state->eof)
+           {
+             break;
+           }
+         /* output buffer is empty; decompress more data */
+         state->out_block.size = state->out_bsz;
+         state->out_pos = state->out_block.pos = 0;
+         size_t ret = ZSTD_decompressStream (state->stream, &state->out_block, &state->in_block);
+         if (ZSTD_isError(ret))
+           {
+             // TODO return/print error
+             return -1;
+           }
+         state->eof = (ret == 0);
+       }
+    }
+  while (lenRemaining != 0);
+
+  return (len - lenRemaining);
+}
+
+ssize_t
+compress_zstd::write (const void *buffer, size_t len)
+{
+  throw new std::logic_error("compress_zstd::write is not implemented");
+}
+
+ssize_t
+compress_zstd::peek (void *buffer, size_t len)
+{
+  /* can only peek 512 bytes */
+  if (len > 512)
+    return ENOMEM;
+
+  // we only peek at the beginning of a file, so no buffer tearing can happen
+  // do a real read first…
+  ssize_t got = read (buffer, len);
+  if (got >= 0)
+    {
+      // …then rewind read position for the next read()
+      state->out_pos -= got;
+    }
+  /* error */
+  return got;
+}
+
+long
+compress_zstd::tell ()
+{
+  throw new std::logic_error("compress_zstd::tell is not implemented");
+}
+
+int
+compress_zstd::seek (long where, io_stream_seek_t whence)
+{
+  throw new std::logic_error("compress_zstd::seek is not implemented");
+}
+
+int
+compress_zstd::error ()
+{
+  return lasterr;
+}
+
+int
+compress_zstd::set_mtime (time_t mtime)
+{
+  if (original)
+    return original->set_mtime (mtime);
+  return 1;
+}
+
+time_t
+compress_zstd::get_mtime ()
+{
+  if (original)
+    return original->get_mtime ();
+  return 0;
+}
+
+mode_t
+compress_zstd::get_mode ()
+{
+  if (original)
+    return original->get_mode ();
+  return 0;
+}
+
+void
+compress_zstd::release_original ()
+{
+  owns_original = false;
+}
+
+void
+compress_zstd::destroy ()
+{
+  if (state)
+    {
+      ZSTD_freeDStream(state->stream);
+
+      if (state->out_block.dst)
+        {
+          free (state->out_block.dst);
+          state->out_block.dst = NULL;
+        }
+
+      if (state->in_block.src)
+        {
+          free ((void*)state->in_block.src);
+          state->in_block.src = NULL;
+        }
+
+      free(state);
+      state = NULL;
+    }
+
+  if (original && owns_original)
+    delete original;
+}
+
+compress_zstd::~compress_zstd ()
+{
+  destroy ();
+}
+
+bool
+compress_zstd::is_zstd (void * buffer, size_t len)
+{
+  return (ZSTD_getFrameContentSize(buffer, len) != ZSTD_CONTENTSIZE_ERROR);
+}
diff --git a/compress_zstd.h b/compress_zstd.h
new file mode 100644 (file)
index 0000000..be5712c
--- /dev/null
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2018, Cygwin
+ *
+ *     This program is free software; you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation; either version 2 of the License, or
+ *     (at your option) any later version.
+ *
+ *     A copy of the GNU General Public License can be found at
+ *     http://www.gnu.org/
+ *
+ */
+
+#ifndef SETUP_COMPRESS_ZSTD_H
+#define SETUP_COMPRESS_ZSTD_H
+
+#include "compress.h"
+#include <zstd.h>
+
+class compress_zstd:public compress
+{
+public:
+  compress_zstd (io_stream *); /* decompress (read) only */
+  virtual ssize_t read (void *buffer, size_t len);
+  virtual ssize_t write (const void *buffer, size_t len); /* not implemented */
+  virtual ssize_t peek (void *buffer, size_t len);
+  virtual long tell (); /* not implemented */
+  virtual int seek (long where, io_stream_seek_t whence); /* not implemented */
+  virtual int error ();
+  virtual const char *next_file_name () { return NULL; };
+  virtual int set_mtime (time_t);
+  virtual time_t get_mtime ();
+  virtual mode_t get_mode ();
+  virtual size_t get_size () {return 0;};
+  virtual ~compress_zstd ();
+  static bool is_zstd (void *buffer, size_t len);
+  virtual void release_original(); /* give up ownership of original io_stream */
+
+private:
+  compress_zstd () {};
+
+  io_stream *original;
+  bool owns_original;
+  int lasterr;
+  void destroy ();
+
+  struct private_data {
+    ZSTD_DStream    *stream;
+    ZSTD_outBuffer   out_block;
+    size_t           out_bsz;
+    size_t           out_pos;
+    uint64_t         total_out;
+    char             eof; /* True = found end of compressed data. */
+    ZSTD_inBuffer    in_block;
+    size_t           in_bsz;
+    size_t           in_pos;
+    uint64_t         total_in;
+    size_t           in_processed;
+    size_t           out_processed;
+  };
+
+  struct private_data *state;
+};
+
+#endif /* SETUP_COMPRESS_ZSTD_H */
index 3854088763fca32df25b200403dedb9064a7c014..08fe16be945fa2d0b4a6dddc5901abbc4b86564b 100644 (file)
@@ -51,6 +51,7 @@ AC_CHECK_TOOL(OBJCOPY, objcopy, objcopy)
 dnl dependencies we can check for using pkgconfig
 PKG_CHECK_MODULES(ZLIB, [zlib])
 PKG_CHECK_MODULES(LZMA, [liblzma])
+PKG_CHECK_MODULES(ZSTD, [libzstd])
 PKG_CHECK_MODULES(LIBSOLV, [libsolv])
 
 dnl dependencies we need to check for by hand
diff --git a/ini.h b/ini.h
index 4b9ed691b6f9305efa9d70e34adc1e9ec652446e..41ba8ec65a3c7091ace9589f4eb5560fe413d68a 100644 (file)
--- a/ini.h
+++ b/ini.h
@@ -22,7 +22,7 @@ class io_stream;
 
 typedef std::vector <std::string> IniList;
 extern IniList found_ini_list, setup_ext_list;
-const std::string setup_exts[] = { "xz", "bz2", "ini" };
+const std::string setup_exts[] = { "zst", "xz", "bz2", "ini" };
 extern bool is_64bit;
 extern bool is_new_install;
 extern std::string SetupArch;
This page took 0.044873 seconds and 5 git commands to generate.