.gitignore

@@ -23,6 +23,7 @@ zstdmt
 # Test artefacts
 tmp*
 dictionary*
+NUL
 
 # Build artefacts
 projects/

Makefile

@@ -27,7 +27,7 @@ endif
 default: lib-release zstd-release
 
 .PHONY: all
-all: | allmost examples manual contrib
+all: allmost examples manual contrib
 
 .PHONY: allmost
 allmost: allzstd
@@ -35,8 +35,7 @@ allmost: allzstd
 
 #skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
-allzstd:
-	$(MAKE) -C $(ZSTDDIR) all
+allzstd: lib
 	$(MAKE) -C $(PRGDIR) all
 	$(MAKE) -C $(TESTDIR) all
 
@@ -45,24 +44,15 @@ all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32
 
-.PHONY: lib
-lib:
+.PHONY: lib lib-release
+lib lib-release:
 	@$(MAKE) -C $(ZSTDDIR) $@
 
-.PHONY: lib-release
-lib-release:
-	@$(MAKE) -C $(ZSTDDIR)
-
-.PHONY: zstd
-zstd:
+.PHONY: zstd zstd-release
+zstd zstd-release:
 	@$(MAKE) -C $(PRGDIR) $@
 	cp $(PRGDIR)/zstd$(EXT) .
 
-.PHONY: zstd-release
-zstd-release:
-	@$(MAKE) -C $(PRGDIR)
-	cp $(PRGDIR)/zstd$(EXT) .
-
 .PHONY: zstdmt
 zstdmt:
 	@$(MAKE) -C $(PRGDIR) $@
@@ -74,7 +64,7 @@ zlibwrapper:
 
 .PHONY: test
 test:
-	$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DZSTD_DEBUG=1"
+	$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DDEBUGLEVEL=1"
 	$(MAKE) -C $(TESTDIR) $@
 
 .PHONY: shortest
@@ -85,7 +75,7 @@ shortest:
 check: shortest
 
 .PHONY: examples
-examples:
+examples: lib
 	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
 
 .PHONY: manual
@@ -118,9 +108,9 @@ clean:
 	@echo Cleaning completed
 
 #------------------------------------------------------------------------------
-# make install is validated only for Linux, OSX, Hurd and some BSD targets
+# make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD MSYS_NT))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
 
 HOST_OS = POSIX
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
@@ -183,6 +173,7 @@ armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 aarch64fuzz: clean
+	ld -v
 	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 ppcfuzz: clean

NEWS

+v1.3.5
+perf: much faster dictionary compression, by @felixhandte
+perf: small quality improvement for dictionary generation, by @terrelln
+perf: slightly improved high compression levels (notably level 19)
+mem : automatic memory release for long duration contexts
+cli : fix : overlapLog can be manually set
+cli : fix : decoding invalid lz4 frames
+api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
+api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
+build: select custom libzstd scope through control macros, by @GeorgeLu97
+build: OpenBSD patch, by @bket
+build: make and make all are compatible with -j
+doc : clarify zstd_compression_format.md, updated for IETF RFC process
+misc: pzstd compatible with reproducible compilation, by @lamby
+
 v1.3.4
 perf: faster speed (especially decoding speed) on recent cpus (haswell+)
 perf: much better performance associating --long with multi-threading, by @terrelln

README.md

@@ -4,7 +4,7 @@ __Zstandard__, or `zstd` as short version, is a fast lossless compression algori
 targeting real-time compression scenarios at zlib-level and better compression ratios.
 It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
 
-The project is provided as an open-source BSD-licensed **C** library,
+The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
 and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).

TESTING.md

@@ -41,4 +41,4 @@ They consist of the following tests:
 - `pzstd` with asan and tsan, as well as in 32-bits mode
 - Testing `zstd` with legacy mode off
 - Testing `zbuff` (old streaming API)
-- Entire test suite and make install on OS X
+- Entire test suite and make install on macOS

contrib/adaptive-compression/Makefile

@@ -48,7 +48,7 @@ clean:
 	@echo "finished cleaning"
 
 #-----------------------------------------------------------------------------
-# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 

contrib/gen_html/Makefile

@@ -10,7 +10,7 @@
 CXXFLAGS ?= -O3
 CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
 CXXFLAGS += $(MOREFLAGS)
-FLAGS   = $(CPPFLAGS) $(CXXFLAGS) $(CXXFLAGS) $(LDFLAGS)
+FLAGS   = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
 
 ZSTDAPI = ../../lib/zstd.h
 ZSTDMANUAL = ../../doc/zstd_manual.html

contrib/pzstd/Makefile

@@ -42,7 +42,7 @@ PZSTD_LDFLAGS   =
 EXTRA_FLAGS     =
 ALL_CFLAGS      = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS)   $(PZSTD_CFLAGS)
 ALL_CXXFLAGS    = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS)
-ALL_LDFLAGS     = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
+ALL_LDFLAGS     = $(EXTRA_FLAGS) $(CXXFLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
 
 
 # gtest libraries need to go before "-lpthread" because they depend on it.
@@ -50,7 +50,7 @@ GTEST_LIB  = -L googletest/build/googlemock/gtest
 LIBS       =
 
 # Compilation commands
-LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -lpthread -o $@
+LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -pthread -o $@
 CC_COMMAND  = $(CC)  $(DEPFLAGS) $(ALL_CFLAGS)   -c $<  -o $@
 CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $<  -o $@
 

contrib/pzstd/Options.cpp

@@ -85,7 +85,7 @@ void usage() {
   std::fprintf(stderr, "Usage:\n");
   std::fprintf(stderr, "  pzstd [args] [FILE(s)]\n");
   std::fprintf(stderr, "Parallel ZSTD options:\n");
-  std::fprintf(stderr, "  -p, --processes   #    : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads());
+  std::fprintf(stderr, "  -p, --processes   #    : number of threads to use for (de)compression (default:<numcpus>)\n");
 
   std::fprintf(stderr, "ZSTD options:\n");
   std::fprintf(stderr, "  -#                     : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);

contrib/seekable_format/examples/Makefile

@@ -9,13 +9,16 @@
 
 # This Makefile presumes libzstd is built, using `make` in / or /lib/
 
-LDFLAGS += ../../../lib/libzstd.a
+ZSTDLIB_PATH = ../../../lib
+ZSTDLIB_NAME = libzstd.a
+ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
+
 CPPFLAGS += -I../ -I../../../lib -I../../../lib/common
 
 CFLAGS ?= -O3
 CFLAGS += -g
 
-SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c
+SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
 
 .PHONY: default all clean test
 
@@ -23,6 +26,9 @@ default: all
 
 all: seekable_compression seekable_decompression parallel_processing
 
+$(ZSTDLIB):
+	make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
+
 seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 

contrib/seekable_format/zstd_seekable.h

@@ -6,8 +6,10 @@ extern "C" {
 #endif
 
 #include <stdio.h>
+#include "zstd.h"   /* ZSTDLIB_API */
 
-static const unsigned ZSTD_seekTableFooterSize = 9;
+
+#define ZSTD_seekTableFooterSize 9
 
 #define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1
 

contrib/seekable_format/zstdseek_decompress.c

@@ -24,7 +24,7 @@
 #endif
 
 /* ************************************************************
-* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
+* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
 ***************************************************************/
 #if defined(_MSC_VER) && _MSC_VER >= 1400
 #   define LONG_SEEK _fseeki64
@@ -88,7 +88,7 @@ static int ZSTD_seekable_read_FILE(void* opaque, void* buffer, size_t n)
     return 0;
 }
 
-static int ZSTD_seekable_seek_FILE(void* opaque, S64 offset, int origin)
+static int ZSTD_seekable_seek_FILE(void* opaque, long long offset, int origin)
 {
     int const ret = LONG_SEEK((FILE*)opaque, offset, origin);
     if (ret) return ret;
@@ -110,7 +110,7 @@ static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
     return 0;
 }
 
-static int ZSTD_seekable_seek_buff(void* opaque, S64 offset, int origin)
+static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
 {
     buffWrapper_t* buff = (buffWrapper_t*) opaque;
     unsigned long long newOffset;
@@ -197,7 +197,7 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
  *  Performs a binary search to find the last frame with a decompressed offset
  *  <= pos
  *  @return : the frame's index */
-U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, U64 pos)
+U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
 {
     U32 lo = 0;
     U32 hi = zs->seekTable.tableLen;
@@ -222,13 +222,13 @@ U32 ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
     return zs->seekTable.tableLen;
 }
 
-U64 ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
+unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
 {
     if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
     return zs->seekTable.entries[frameIndex].cOffset;
 }
 
-U64 ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
+unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
 {
     if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
     return zs->seekTable.entries[frameIndex].dOffset;
@@ -294,7 +294,6 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
         {   /* Allocate an extra entry at the end so that we can do size
              * computations on the last element without special case */
             seekEntry_t* entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1));
-            const BYTE* tableBase = zs->inBuff + ZSTD_skippableHeaderSize;
 
             U32 idx = 0;
             U32 pos = 8;
@@ -372,7 +371,7 @@ size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile sr
     return 0;
 }
 
-size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, U64 offset)
+size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsigned long long offset)
 {
     U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset);
     do {

debian/changelog

+libzstd (1.3.5+dfsg-1) UNRELEASED; urgency=medium
+
+  * Add attribuition for the patch
+  * Fix varying number of threads in pzstd man page, making build reproducible
+  * New upstream version 1.3.5+dfsg
+  * Refresh and drop patches applied by upstream
+
+ -- Alexandre Mestiashvili <mestia@debian.org>  Wed, 11 Jul 2018 15:47:58 +0200
+
 libzstd (1.3.4+dfsg-3) unstable; urgency=medium
 
   * Add patch provided by Chris Lamb making build reproducible,

debian/patches/0012-typos.patch

-Description: Fix typo in zstd man page s/minumum/minimum/
-From: Alex Mestiashvili <mailatgoogl@gmail.com>
-Forwarded: https://github.com/facebook/zstd/issues/1115
---- libzstd.orig/programs/zstd.1.md
-+++ libzstd/programs/zstd.1.md
-@@ -392,7 +392,7 @@
- 
-     Larger/very small values usually decrease compression ratio.
- 
--    The minumum _ldmslen_ is 4 and the maximum is 4096 (default: 64).
-+    The minimum _ldmslen_ is 4 and the maximum is 4096 (default: 64).
- 
- - `ldmBucketSizeLog`=_ldmblog_, `ldmblog`=_ldmblog_:
-     Specify the size of each bucket for the hash table used for long distance
---- libzstd.orig/programs/zstd.1
-+++ libzstd/programs/zstd.1
-@@ -374,7 +374,7 @@
- Larger/very small values usually decrease compression ratio\.
- .
- .IP
--The minumum \fIldmslen\fR is 4 and the maximum is 4096 (default: 64)\.
-+The minimum \fIldmslen\fR is 4 and the maximum is 4096 (default: 64)\.
- .
- .TP
- \fBldmBucketSizeLog\fR=\fIldmblog\fR, \fBldmblog\fR=\fIldmblog\fR

debian/patches/0013-skip-memory-greedy-tests.patch

@@ -2,7 +2,7 @@ Subject: Skip memory heavy tests causing FTBFS on mips(el) and hurd buildds
 From: Alex Mestiashvili <mestia@debian.org>
 --- libzstd.orig/tests/playTests.sh
 +++ libzstd/tests/playTests.sh
-@@ -743,8 +743,13 @@
+@@ -799,8 +799,13 @@
  roundTripTest -g5000000000 -P99 1
  roundTripTest -g1700000000 -P0 "1 --zstd=strategy=6"   # ensure btlazy2 can survive an overflow rescale
  
@@ -17,7 +17,7 @@ From: Alex Mestiashvili <mestia@debian.org>
  
  $ECHO "\n===>   zstd long, long distance matching round-trip tests "
  roundTripTest -g270000000 "1 --single-thread --long"
-@@ -761,9 +766,14 @@
+@@ -817,9 +822,14 @@
      $ECHO "\n===>   zstdmt long round-trip tests "
      roundTripTest -g80000000 -P99 "19 -T2" " "
      roundTripTest -g5000000000 -P99 "1 -T2" " "

debian/patches/0014-Reproducible-build.patch

@@ -6,7 +6,7 @@ Applied-Upstream: https://github.com/facebook/zstd/commit/ef1abd3c071ce42a457404
 --- libzstd.orig/contrib/pzstd/Options.cpp
 +++ libzstd/contrib/pzstd/Options.cpp
 @@ -88,7 +88,7 @@
-   std::fprintf(stderr, "  -p, --processes   #    : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads());
+   std::fprintf(stderr, "  -p, --processes   #    : number of threads to use for (de)compression (default:<numcpus>)\n");
  
    std::fprintf(stderr, "ZSTD options:\n");
 -  std::fprintf(stderr, "  -#                     : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);

debian/patches/0015-Non-regular-file-test.patch

-Description: Use /dev/random instead of /dev/zero as non-regular file,
- because /dev/zero isn't writable on GNU/Hurd.
- Fix a typo in tests s/INTONULL/INTOVOID/
-Author: Alex Mestiashvili <mestia@debian.org>
-Applied-Upstream: https://github.com/facebook/zstd/pull/1124/commits/2dde9d5abaade164329bf698aa938f1a5b0526d1
---- libzstd.orig/tests/playTests.sh
-+++ libzstd/tests/playTests.sh
-@@ -56,7 +56,7 @@
- 
- isWindows=false
- INTOVOID="/dev/null"
--DEVDEVICE="/dev/zero"
-+DEVDEVICE="/dev/random"
- case "$OS" in
-   Windows*)
-     isWindows=true
-@@ -175,7 +175,7 @@
- $ZSTD tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
- grep -v "Refusing to remove non-regular file" tmplog
- rm -f tmplog
--$ZSTD tmp -f -o "$INTONULL" 2>&1 | grep -v "Refusing to remove non-regular file"
-+$ZSTD tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file"
- $ECHO "test : --rm on stdin"
- $ECHO a | $ZSTD --rm > $INTOVOID   # --rm should remain silent
- rm tmp

debian/patches/pthread.patch

-Author: Adam Borowski <kilobyte@angband.pl>
-Last-Update: Mon, 09 Apr 2018 00:09:44 +0000
-Description: Enable build on riscv64
-Forwarded-Upstream: https://github.com/facebook/zstd/pull/1103
-
---- a/contrib/pzstd/Makefile
-+++ b/contrib/pzstd/Makefile
-@@ -50,7 +50,7 @@ GTEST_LIB  = -L googletest/build/googlem
- LIBS       =
- 
- # Compilation commands
--LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -lpthread -o $@
-+LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -pthread -o $@
- CC_COMMAND  = $(CC)  $(DEPFLAGS) $(ALL_CFLAGS)   -c $<  -o $@
- CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $<  -o $@
- 

debian/patches/series

 0006-Use-bash-for-test-script-portablitity.patch
 0008-Address-embedded-zlib.patch
-0012-typos.patch
-pthread.patch
 0013-skip-memory-greedy-tests.patch
 0014-Reproducible-build.patch
-0015-Non-regular-file-test.patch