New upstream version 1.3.4+dfsg

default: lib-release zstd-release

.PHONY: all

all: | allmost examples manual

all: | allmost examples manual contrib

.PHONY: allmost

allmost: allzstd

zlibwrapper:

	$(MAKE) -C $(ZWRAPDIR) test

.PHONY: check

check: shortest

.PHONY: test

test:

	$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DZSTD_DEBUG=1"

	$(MAKE) -C $(TESTDIR) $@

.PHONY: test shortest

test shortest:

	$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS="-g -DZSTD_DEBUG=1"

.PHONY: shortest

shortest:

	$(MAKE) -C $(TESTDIR) $@

.PHONY: check

check: shortest

.PHONY: examples

examples:

	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all

manual:

	$(MAKE) -C contrib/gen_html $@

.PHONY: contrib

contrib: lib

	$(MAKE) -C contrib/pzstd all

	$(MAKE) -C contrib/seekable_format/examples all

	$(MAKE) -C contrib/adaptive-compression all

.PHONY: cleanTabs

cleanTabs:

	cd contrib; ./cleanTabs

	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)

	@$(MAKE) -C examples/ $@ > $(VOID)

	@$(MAKE) -C contrib/gen_html $@ > $(VOID)

	@$(MAKE) -C contrib/pzstd $@ > $(VOID)

	@$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)

	@$(MAKE) -C contrib/adaptive-compression $@ > $(VOID)

	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*

	@$(RM) -r lz4

	@echo Cleaning completed

# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303

usan: clean

	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined"

	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"

asan: clean

	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"

	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"

asan-%: clean

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $*

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror" $(MAKE) -C $(TESTDIR) $*

msan: clean

	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" HAVE_LZMA=0   # datagen.c fails this test for no obvious reason

	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror" HAVE_LZMA=0   # datagen.c fails this test for no obvious reason

msan-%: clean

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*

asan32: clean

	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"

uasan: clean

	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined"

	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"

uasan-%: clean

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*

tsan-%: clean

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests

	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests

apt-install:

	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)

gcc6install: apt-add-repo

	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install

gcc7install: apt-add-repo

	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install

gpp6install: apt-add-repo

	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install

v1.3.4

perf: faster speed (especially decoding speed) on recent cpus (haswell+)

perf: much better performance associating --long with multi-threading, by @terrelln

perf: better compression at levels 13-15

cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)

cli : smoother status report in multi-threading mode

cli : added command --fast=#, for faster compression modes

cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)

api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode

api : compression levels can be negative, for even more speed

api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime

api : ZSTDMT can accept new compression parameters during compression

api : implemented all advanced dictionary decompression prototypes

build: improved meson recipe, by Shawn Landden (@shawnl)

build: VS2017 scripts, by @HaydnTrigg

misc: all /contrib projects fixed

misc: added /contrib/docker script by @gyscos

v1.3.3

perf: faster zstd_opt strategy (levels 17-19)

perf: faster zstd_opt strategy (levels 16-19)

fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen

cli : fix : content size written in header by default

cli : fix : improved LZ4 format support, by @felixhandte

<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/readme/doc/images/zstd_logo86.png" alt="Zstandard"></p>

<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/dev/doc/images/zstd_logo86.png" alt="Zstandard"></p>

__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,

targeting real-time compression scenarios at zlib-level and better compression ratios.

### Benchmarks

For reference, several fast compression algorithms were tested and compared

on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),

on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),

with a Core i7-6700K CPU @ 4.0GHz,

using [lzbench], an open-source in-memory benchmark by @inikep

compiled with GCC 6.3.0,

compiled with [gcc] 7.3.0,

on the [Silesia compression corpus].

[lzbench]: https://github.com/inikep/lzbench

[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia

[gcc]: https://gcc.gnu.org/

| Compressor name         | Ratio | Compression| Decompress.|

| ---------------         | ------| -----------| ---------- |

| **zstd 1.1.3 -1**       | 2.877 |   430 MB/s |  1110 MB/s |

| zlib 1.2.8 -1           | 2.743 |   110 MB/s |   400 MB/s |

| brotli 0.5.2 -0         | 2.708 |   400 MB/s |   430 MB/s |

| **zstd 1.3.4 -1**       | 2.877 |   470 MB/s |  1380 MB/s |

| zlib 1.2.11 -1          | 2.743 |   110 MB/s |   400 MB/s |

| brotli 1.0.2 -0         | 2.701 |   410 MB/s |   430 MB/s |

| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |

| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |

| lz4 1.7.5               | 2.101 |   720 MB/s |  3600 MB/s |

| snappy 1.1.3            | 2.091 |   500 MB/s |  1650 MB/s |

| lz4 1.8.1               | 2.101 |   750 MB/s |  3700 MB/s |

| snappy 1.1.4            | 2.091 |   530 MB/s |  1800 MB/s |

| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |

[zlib]:http://www.zlib.net/

a property shared by most LZ compression algorithms, such as [zlib] or lzma.

The following tests were run

on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)

on a server running Linux Debian (`Linux version 4.14.0-3-amd64`)

with a Core i7-6700K CPU @ 4.0GHz,

using [lzbench], an open-source in-memory benchmark by @inikep

compiled with GCC 6.3.0,

compiled with [gcc] 7.3.0,

on the [Silesia compression corpus].

Compression Speed vs Ratio | Decompression Speed

---------------------------|--------------------

![Compression Speed vs Ratio](doc/images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/Dspeed4.png "Decompression Speed")

![Compression Speed vs Ratio](doc/images/CSpeed2.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/DSpeed3.png "Decompression Speed")

A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.

For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).

Going into `build` directory, you will find additional possibilities:

- Projects for Visual Studio 2005, 2008 and 2010.

  + VS2010 project is compatible with VS2012, VS2013 and VS2015.

- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`,

  + VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017.

- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,

  which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.

# binaries generated

adapt

datagen

static UTIL_time_t g_startTime;

static size_t g_streamedSize = 0;

static unsigned g_useProgressBar = 1;

static UTIL_freq_t g_ticksPerSecond;

static unsigned g_forceCompressionLevel = 0;

static unsigned g_minCLevel = 1;

static unsigned g_maxCLevel;

            /* begin compression */

            {

                size_t const useDictSize = MIN(getUseableDictSize(cLevel), job->dictSize);

                size_t const dictModeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceRawDict, 1);

                ZSTD_parameters params = ZSTD_getParams(cLevel, 0, useDictSize);

                params.cParams.windowLog = 23;

                {

                    size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0);

                    size_t const windowSizeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceWindow, 1);

                    if (ZSTD_isError(dictModeError) || ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {

                    size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1);

                    if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {

                        DISPLAY("Error: something went wrong while starting compression\n");

                        signalErrorToThreads(ctx);

                        return arg;

static void displayProgress(unsigned cLevel, unsigned last)

{

    UTIL_time_t currTime;

    UTIL_getTime(&currTime);

    UTIL_time_t currTime = UTIL_getTime();

    if (!g_useProgressBar) return;

    {

        double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);

    {   double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_startTime, currTime) / 1000.0);

        double const sizeMB = (double)g_streamedSize / (1 << 20);

        double const avgCompRate = sizeMB * 1000 / timeElapsed;

        fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);

        if (last) {

            fprintf(stderr, "\n");

        }

        else {

        } else {

            fflush(stderr);

        }

    }

    }   }

}

static void* outputThread(void* arg)

{

    int ret = 0;

    fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);

    UTIL_getTime(&g_startTime);

    g_streamedSize = 0;

    ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg);

    ret |= freeFileCompressionResources(&fcr);

    filenameTable[0] = stdinmark;

    g_maxCLevel = ZSTD_maxCLevel();

    UTIL_initTimer(&g_ticksPerSecond);

    if (filenameTable == NULL) {

        DISPLAY("Error: could not allocate sapce for filename table.\n");

        return 1;