.gitattributes

@@ -19,6 +19,3 @@
 # Windows
 *.bat text eol=crlf
 *.cmd text eol=crlf
-
-# .travis.yml merging
-.travis.yml merge=ours

NEWS → CHANGELOG

+v1.3.8
+perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
+perf: slightly better compression ratio at high compression modes
+api : finalized advanced API, last stage before "stable" status
+api : new --rsyncable mode, by @terrelln
+api : support decompression of empty frames into NULL (used to be an error) (#1385)
+build: new set of macros to build a minimal size decoder, by @felixhandte
+build: fix compilation on MIPS32, reported by @clbr (#1441)
+build: fix compilation with multiple -arch flags, by @ryandesign
+build: highly upgraded meson build, by @lzutao
+build: improved buck support, by @obelisk
+build: fix cmake script : can create debug build, by @pitrou
+build: Makefile : grep works on both colored consoles and systems without color support
+build: fixed zstd-pgo, by @bmwiedemann
+cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
+cli : --no-progress flag, preserving final summary (#1371), by @terrelln
+cli : ensure destination file is not source file (#1422)
+cli : clearer error messages, especially when input file not present
+doc : clarified zstd_compression_format.md, by @ulikunitz
+misc: fixed zstdgrep, returns 1 on failure, by @lzutao
+misc: NEWS renamed as CHANGELOG, in accordance with fboss
+
+v1.3.7
+perf: slightly better decompression speed on clang (depending on hardware target)
+fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
+build: no longer build backtrace by default in release mode; restrict further automatic mode
+build: control backtrace support through build macro BACKTRACE
+misc: added man pages for zstdless and zstdgrep, by @samrussell
+
+v1.3.6
+perf: much faster dictionary builder, by @jenniferliu
+perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
+perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
+cli : fix : does no longer overwrite destination when source does not exist (#1082)
+cli : new command --adapt, for automatic compression level adaptation
+api : fix : block api can be streamed with > 4 GB, reported by @catid
+api : reduced ZSTD_DDict size by 2 KB
+api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
+build: support Haiku target, by @korli
+build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
+doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
+misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
+
 v1.3.5
 perf: much faster dictionary compression, by @felixhandte
 perf: small quality improvement for dictionary generation, by @terrelln

CODE_OF_CONDUCT.md

+# Code of Conduct
+
+Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
+Please read the [full text](https://code.fb.com/codeofconduct/)
+so that you can understand what actions will and will not be tolerated.

Makefile

@@ -23,6 +23,7 @@ else
 EXT =
 endif
 
+## default: Build lib-release and zstd-release
 .PHONY: default
 default: lib-release zstd-release
 
@@ -30,8 +31,7 @@ default: lib-release zstd-release
 all: allmost examples manual contrib
 
 .PHONY: allmost
-allmost: allzstd
-	$(MAKE) -C $(ZWRAPDIR) all
+allmost: allzstd zlibwrapper
 
 # skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
@@ -44,7 +44,7 @@ all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32
 
-.PHONY: lib lib-release
+.PHONY: lib lib-release libzstd.a
 lib lib-release :
 	@$(MAKE) -C $(ZSTDDIR) $@
 
@@ -59,34 +59,48 @@ zstdmt:
 	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
 
 .PHONY: zlibwrapper
-zlibwrapper:
-	$(MAKE) -C $(ZWRAPDIR) test
+zlibwrapper: lib
+	$(MAKE) -C $(ZWRAPDIR) all
 
+## test: run long-duration tests
 .PHONY: test
+DEBUGLEVEL ?= 1
+test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
 test:
-	$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DDEBUGLEVEL=1"
+	MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
 	$(MAKE) -C $(TESTDIR) $@
 
+## shortest: same as `make check`
 .PHONY: shortest
 shortest:
 	$(MAKE) -C $(TESTDIR) $@
 
+## check: run basic tests for `zstd` cli
 .PHONY: check
 check: shortest
 
+## examples: build all examples in `/examples` directory
 .PHONY: examples
 examples: lib
 	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
 
+## manual: generate API documentation in html format
 .PHONY: manual
 manual:
 	$(MAKE) -C contrib/gen_html $@
 
+## man: generate man page
+.PHONY: man
+man:
+	$(MAKE) -C programs $@
+
+## contrib: build all supported projects in `/contrib` directory
 .PHONY: contrib
 contrib: lib
 	$(MAKE) -C contrib/pzstd all
 	$(MAKE) -C contrib/seekable_format/examples all
 	$(MAKE) -C contrib/adaptive-compression all
+	$(MAKE) -C contrib/largeNbDicts all
 
 .PHONY: cleanTabs
 cleanTabs:
@@ -103,6 +117,7 @@ clean:
 	@$(MAKE) -C contrib/pzstd $@ > $(VOID)
 	@$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
 	@$(MAKE) -C contrib/adaptive-compression $@ > $(VOID)
+	@$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
 	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	@$(RM) -r lz4
 	@echo Cleaning completed
@@ -110,14 +125,36 @@ clean:
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))
 
 HOST_OS = POSIX
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
 
+HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+EGREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+EGREP_OPTIONS += --color=never
+endif
+EGREP = egrep $(EGREP_OPTIONS)
+
+# Print a two column output of targets and their description. To add a target description, put a
+# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>".  For example:
+#
+## list: Print all targets and their descriptions (if provided)
 .PHONY: list
 list:
-	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+	@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
+		| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
+		| $(EGREP) -v  -e '^[^[:alnum:]]' | sort); \
+	{ \
+	    printf "Target Name\tDescription\n"; \
+	    printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
+	    for target in $$TARGETS; do \
+	        line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
+	        description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
+	        printf "$$target\t$$description\n"; \
+	    done \
+	} | column -t -s $$'\t'
 
 .PHONY: install clangtest armtest usan asan uasan
 install:
@@ -197,7 +234,7 @@ gcc6test: clean
 
 clangtest: clean
 	clang -v
-	$(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+	$(MAKE) all CXX=clang++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
 
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
@@ -286,6 +323,9 @@ gcc6install: apt-add-repo
 gcc7install: apt-add-repo
 	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install
 
+gcc8install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install
+
 gpp6install: apt-add-repo
 	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
 
@@ -317,23 +357,23 @@ cmakebuild:
 
 c90build: clean
 	$(CC) -v
-	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c90 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
 gnu90build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu90" $(MAKE) allmost
+	CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
 
 c99build: clean
 	$(CC) -v
-	CFLAGS="-std=c99" $(MAKE) allmost
+	CFLAGS="-std=c99 -Werror" $(MAKE) allmost
 
 gnu99build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu99" $(MAKE) allmost
+	CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
 
 c11build: clean
 	$(CC) -v
-	CFLAGS="-std=c11" $(MAKE) allmost
+	CFLAGS="-std=c11 -Werror" $(MAKE) allmost
 
 bmix64build: clean
 	$(CC) -v
@@ -347,7 +387,10 @@ bmi32build: clean
 	$(CC) -v
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
 
-staticAnalyze: clean
+# static analyzer test uses clang's scan-build
+# does not analyze zlibWrapper, due to detected issues in zlib source code
+staticAnalyze: SCANBUILD ?= scan-build
+staticAnalyze:
 	$(CC) -v
-	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
+	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
 endif

README.md

@@ -9,7 +9,11 @@ and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
 
-Development branch status : [![Build Status][travisDevBadge]][travisLink]   [![Build status][AppveyorDevBadge]][AppveyorLink]   [![Build status][CircleDevBadge]][CircleLink]
+**Development branch status:**
+
+[![Build Status][travisDevBadge]][travisLink]
+[![Build status][AppveyorDevBadge]][AppveyorLink]
+[![Build status][CircleDevBadge]][CircleLink]
 
 [travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
 [travisLink]: https://travis-ci.org/facebook/zstd
@@ -18,7 +22,7 @@ Development branch status : [![Build Status][travisDevBadge]][travisLink]   [![B
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
 [CircleLink]: https://circleci.com/gh/facebook/zstd
 
-### Benchmarks
+## Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
 on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
@@ -65,7 +69,7 @@ A few other algorithms can produce higher compression ratios at slower speeds, f
 For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
 
 
-### The case for Small Data compression
+## The case for Small Data compression
 
 Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
 
@@ -89,24 +93,24 @@ Training works if there is some correlation in a family of small data samples. T
 Hence, deploying one dictionary per type of data will provide the greatest benefits.
 Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
 
-#### Dictionary compression How To:
+### Dictionary compression How To:
 
-1) Create the dictionary
+1. Create the dictionary
 
    `zstd --train FullPathToTrainingSet/* -o dictionaryName`
 
-2) Compress with dictionary
+2. Compress with dictionary
 
    `zstd -D dictionaryName FILE`
 
-3) Decompress with dictionary
+3. Decompress with dictionary
 
    `zstd -D dictionaryName --decompress FILE.zst`
 
 
-### Build instructions
+## Build instructions
 
-#### Makefile
+### Makefile
 
 If your system is compatible with standard `make` (or `gmake`),
 invoking `make` in root directory will generate `zstd` cli in root directory.
@@ -115,17 +119,25 @@ Other available options include:
 - `make install` : create and install zstd cli, library and man pages
 - `make check` : create and run `zstd`, tests its behavior on local platform
 
-#### cmake
+### cmake
 
 A `cmake` project generator is provided within `build/cmake`.
 It can generate Makefiles or other build scripts
 to create `zstd` binary, and `libzstd` dynamic and static libraries.
 
-#### Meson
+By default, `CMAKE_BUILD_TYPE` is set to `Release`.
+
+### Meson
+
+A Meson project is provided within [`build/meson`](build/meson). Follow
+build instructions in that directory.
 
-A Meson project is provided within `contrib/meson`.
+You can also take a look at [`.travis.yml`](.travis.yml) file for an
+example about how Meson is used to build this project.
 
-#### Visual Studio (Windows)
+Note that default build type is **release**.
+
+### Visual Studio (Windows)
 
 Going into `build` directory, you will find additional possibilities:
 - Projects for Visual Studio 2005, 2008 and 2010.
@@ -133,17 +145,21 @@ Going into `build` directory, you will find additional possibilities:
 - Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
   which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
 
+### Buck
+
+You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
+The output binary will be in `buck-out/gen/programs/`.
 
-### Status
+## Status
 
 Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
 Zstandard is considered safe for production environments.
 
-### License
+## License
 
 Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
 
-### Contributing
+## Contributing
 
 The "dev" branch is the one where all contributions are merged before reaching "master".
 If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.

contrib/adaptive-compression/Makefile

@@ -22,10 +22,10 @@ FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
 
 all: adapt datagen
 
-adapt: $(ZSTD_FILES) adapt.c
+adapt: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
 	$(CC) $(FLAGS) $^ -o $@
 
-adapt-debug: $(ZSTD_FILES) adapt.c
+adapt-debug: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
 	$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
 
 datagen : $(PRGDIR)/datagen.c datagencli.c

contrib/adaptive-compression/adapt.c

@@ -579,7 +579,7 @@ static void* compressionThread(void* arg)
                 params.cParams.windowLog = 23;
                 {
                     size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0);
-                    size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1);
+                    size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_c_forceMaxWindow, 1);
                     if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {
                         DISPLAY("Error: something went wrong while starting compression\n");
                         signalErrorToThreads(ctx);

contrib/adaptive-compression/datagencli.c

@@ -120,7 +120,7 @@ int main(int argc, const char** argv)
     DISPLAYLEVEL(4, "Compressible data Generator \n");
     if (probaU32!=COMPRESSIBILITY_DEFAULT)
         DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
-    DISPLAYLEVEL(3, "Seed = %u \n", seed);
+    DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed);
 
     RDG_genStdout(size, (double)probaU32/100, litProba, seed);
     DISPLAYLEVEL(1, "\n");

contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile

+ARG :=
+
+CC ?= gcc
+CFLAGS ?= -O3
+INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
+
+RANDOM_FILE := ../randomDictBuilder/random.c
+IO_FILE := ../randomDictBuilder/io.c
+
+all: run clean
+
+.PHONY: run
+run: benchmark
+	echo "Benchmarking with $(ARG)"
+	./benchmark $(ARG)
+
+.PHONY: test
+test: benchmarkTest clean
+
+.PHONY: benchmarkTest
+benchmarkTest: benchmark test.sh
+	sh test.sh
+
+benchmark: benchmark.o io.o random.o libzstd.a
+	$(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark
+
+benchmark.o: benchmark.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c
+
+random.o: $(RANDOM_FILE)
+	$(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE)
+
+io.o: $(IO_FILE)
+	$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
+
+libzstd.a:
+	$(MAKE) -C ../../../lib libzstd.a
+	mv ../../../lib/libzstd.a .
+
+.PHONY: clean
+clean:
+	rm -f *.o benchmark libzstd.a
+	$(MAKE) -C ../../../lib clean
+	echo "Cleaning is completed"

contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c

+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h>   /* strcmp, strlen */
+#include <errno.h>    /* errno */
+#include <ctype.h>
+#include <time.h>
+#include "random.h"
+#include "dictBuilder.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#include "io.h"
+#include "util.h"
+#include "zdict.h"
+
+
+
+/*-*************************************
+*  Console display
+***************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
+#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (displayLevel>=4) fflush(stderr); } } }
+
+
+/*-*************************************
+*  Exceptions
+***************************************/
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...)                                             \
+{                                                                         \
+    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+    DISPLAY("Error %i : ", error);                                        \
+    DISPLAY(__VA_ARGS__);                                                 \
+    DISPLAY("\n");                                                        \
+    exit(error);                                                          \
+}
+
+
+/*-*************************************
+*  Constants
+***************************************/
+static const unsigned g_defaultMaxDictSize = 110 KB;
+#define DEFAULT_CLEVEL 3
+#define DEFAULT_DISPLAYLEVEL 2
+
+
+/*-*************************************
+*  Struct
+***************************************/
+typedef struct {
+  const void* dictBuffer;
+  size_t dictSize;
+} dictInfo;
+
+
+/*-*************************************
+* Dictionary related operations
+***************************************/
+/** createDictFromFiles() :
+ *  Based on type of param given, train dictionary using the corresponding algorithm
+ *  @return dictInfo containing dictionary buffer and dictionary size
+ */
+dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
+                  ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams,
+                  ZDICT_legacy_params_t *legacyParams, ZDICT_fastCover_params_t *fastParams) {
+    unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel :
+                                  coverParams ? coverParams->zParams.notificationLevel :
+                                  legacyParams ? legacyParams->zParams.notificationLevel :
+                                  fastParams ? fastParams->zParams.notificationLevel :
+                                  DEFAULT_DISPLAYLEVEL;   /* no dict */
+    void* const dictBuffer = malloc(maxDictSize);
+
+    dictInfo* dInfo = NULL;
+
+    /* Checks */
+    if (!dictBuffer)
+        EXM_THROW(12, "not enough memory for trainFromFiles");   /* should not happen */
+
+    {   size_t dictSize;
+        if(randomParams) {
+          dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer,
+                                               info->samplesSizes, info->nbSamples, *randomParams);
+        }else if(coverParams) {
+          /* Run the optimize version if either k or d is not provided */
+          if (!coverParams->d || !coverParams->k){
+            dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, coverParams);
+          } else {
+            dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, *coverParams);
+          }
+        } else if(legacyParams) {
+          dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer,
+                                               info->samplesSizes, info->nbSamples, *legacyParams);
+        } else if(fastParams) {
+          /* Run the optimize version if either k or d is not provided */
+          if (!fastParams->d || !fastParams->k) {
+            dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, fastParams);
+          } else {
+            dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, *fastParams);
+          }
+        } else {
+          dictSize = 0;
+        }
+        if (ZDICT_isError(dictSize)) {
+            DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize));   /* should not happen */
+            free(dictBuffer);
+            return dInfo;
+        }
+        dInfo = (dictInfo *)malloc(sizeof(dictInfo));
+        dInfo->dictBuffer = dictBuffer;
+        dInfo->dictSize = dictSize;
+    }
+    return dInfo;
+}
+
+
+/** compressWithDict() :
+ *  Compress samples from sample buffer given dicionary stored on dictionary buffer and compression level
+ *  @return compression ratio
+ */
+double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLevel, int displayLevel) {
+  /* Local variables */
+  size_t totalCompressedSize = 0;
+  size_t totalOriginalSize = 0;
+  const unsigned hasDict = dInfo->dictSize > 0 ? 1 : 0;
+  double cRatio;
+  size_t dstCapacity;
+  int i;
+
+  /* Pointers */
+  ZSTD_CDict *cdict = NULL;
+  ZSTD_CCtx* cctx = NULL;
+  size_t *offsets = NULL;
+  void* dst = NULL;
+
+  /* Allocate dst with enough space to compress the maximum sized sample */
+  {
+    size_t maxSampleSize = 0;
+    for (i = 0; i < srcInfo->nbSamples; i++) {
+      maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize);
+    }
+    dstCapacity = ZSTD_compressBound(maxSampleSize);
+    dst = malloc(dstCapacity);
+  }
+
+  /* Calculate offset for each sample */
+  offsets = (size_t *)malloc((srcInfo->nbSamples + 1) * sizeof(size_t));
+  offsets[0] = 0;
+  for (i = 1; i <= srcInfo->nbSamples; i++) {
+    offsets[i] = offsets[i - 1] + srcInfo->samplesSizes[i - 1];
+  }
+
+  /* Create the cctx */
+  cctx = ZSTD_createCCtx();
+  if(!cctx || !dst) {
+    cRatio = -1;
+    goto _cleanup;
+  }
+
+  /* Create CDict if there's a dictionary stored on buffer */
+  if (hasDict) {
+    cdict = ZSTD_createCDict(dInfo->dictBuffer, dInfo->dictSize, compressionLevel);
+    if(!cdict) {
+      cRatio = -1;
+      goto _cleanup;
+    }
+  }
+
+  /* Compress each sample and sum their sizes*/
+  const BYTE *const samples = (const BYTE *)srcInfo->srcBuffer;
+  for (i = 0; i < srcInfo->nbSamples; i++) {
+    size_t compressedSize;
+    if(hasDict) {
+      compressedSize = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, samples + offsets[i], srcInfo->samplesSizes[i], cdict);
+    } else {
+      compressedSize = ZSTD_compressCCtx(cctx, dst, dstCapacity,samples + offsets[i], srcInfo->samplesSizes[i], compressionLevel);
+    }
+    if (ZSTD_isError(compressedSize)) {
+      cRatio = -1;
+      goto _cleanup;
+    }
+    totalCompressedSize += compressedSize;
+  }
+
+  /* Sum orignal sizes */
+  for (i = 0; i<srcInfo->nbSamples; i++) {
+    totalOriginalSize += srcInfo->samplesSizes[i];
+  }
+
+  /* Calculate compression ratio */
+  DISPLAYLEVEL(2, "original size is %lu\n", totalOriginalSize);
+  DISPLAYLEVEL(2, "compressed size is %lu\n", totalCompressedSize);
+  cRatio = (double)totalOriginalSize/(double)totalCompressedSize;
+
+_cleanup:
+  free(dst);
+  free(offsets);
+  ZSTD_freeCCtx(cctx);
+  ZSTD_freeCDict(cdict);
+  return cRatio;
+}
+
+
+/** FreeDictInfo() :
+ *  Free memory allocated for dictInfo
+ */
+void freeDictInfo(dictInfo* info) {
+  if (!info) return;
+  if (info->dictBuffer) free((void*)(info->dictBuffer));
+  free(info);
+}
+
+
+
+/*-********************************************************
+  *  Benchmarking functions
+**********************************************************/
+/** benchmarkDictBuilder() :
+ *  Measure how long a dictionary builder takes and compression ratio with the dictionary built
+ *  @return 0 if benchmark successfully, 1 otherwise
+ */
+int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam,
+                        ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam,
+                        ZDICT_fastCover_params_t *fastParam) {
+  /* Local variables */
+  const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel :
+                                coverParam ? coverParam->zParams.notificationLevel :
+                                legacyParam ? legacyParam->zParams.notificationLevel :
+                                fastParam ? fastParam->zParams.notificationLevel:
+                                DEFAULT_DISPLAYLEVEL;   /* no dict */
+  const char* name = randomParam ? "RANDOM" :
+                    coverParam ? "COVER" :
+                    legacyParam ? "LEGACY" :
+                    fastParam ? "FAST":
+                    "NODICT";    /* no dict */
+  const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel :
+                          coverParam ? coverParam->zParams.compressionLevel :
+                          legacyParam ? legacyParam->zParams.compressionLevel :
+                          fastParam ? fastParam->zParams.compressionLevel:
+                          DEFAULT_CLEVEL;   /* no dict */
+  int result = 0;
+
+  /* Calculate speed */
+  const UTIL_time_t begin = UTIL_getTime();
+  dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam, fastParam);
+  const U64 timeMicro = UTIL_clockSpanMicro(begin);
+  const double timeSec = timeMicro / (double)SEC_TO_MICRO;
+  if (!dInfo) {
+    DISPLAYLEVEL(1, "%s does not train successfully\n", name);
+    result = 1;
+    goto _cleanup;
+  }
+  DISPLAYLEVEL(1, "%s took %f seconds to execute \n", name, timeSec);
+
+  /* Calculate compression ratio */
+  const double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel);
+  if (cRatio < 0) {
+    DISPLAYLEVEL(1, "Compressing with %s dictionary does not work\n", name);
+    result = 1;
+    goto _cleanup;
+
+  }
+  DISPLAYLEVEL(1, "Compression ratio with %s dictionary is %f\n", name, cRatio);
+
+_cleanup:
+  freeDictInfo(dInfo);
+  return result;
+}
+
+
+
+int main(int argCount, const char* argv[])
+{
+  const int displayLevel = DEFAULT_DISPLAYLEVEL;
+  const char* programName = argv[0];
+  int result = 0;
+
+  /* Initialize arguments to default values */
+  unsigned k = 200;
+  unsigned d = 8;
+  unsigned f;
+  unsigned accel;
+  unsigned i;
+  const unsigned cLevel = DEFAULT_CLEVEL;
+  const unsigned dictID = 0;
+  const unsigned maxDictSize = g_defaultMaxDictSize;
+
+  /* Initialize table to store input files */
+  const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));
+  unsigned filenameIdx = 0;
+
+  char* fileNamesBuf = NULL;
+  unsigned fileNamesNb = filenameIdx;
+  const int followLinks = 0;
+  const char** extendedFileList = NULL;
+
+  /* Parse arguments */
+  for (i = 1; i < argCount; i++) {
+    const char* argument = argv[i];
+    if (longCommandWArg(&argument, "in=")) {
+      filenameTable[filenameIdx] = argument;
+      filenameIdx++;
+      continue;
+    }
+    DISPLAYLEVEL(1, "benchmark: Incorrect parameters\n");
+    return 1;
+  }
+
+  /* Get the list of all files recursively (because followLinks==0)*/
+  extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf,
+                                        &fileNamesNb, followLinks);
+  if (extendedFileList) {
+    unsigned u;
+    for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
+    free((void*)filenameTable);
+    filenameTable = extendedFileList;
+    filenameIdx = fileNamesNb;
+  }
+
+  /* get sampleInfo */
+  size_t blockSize = 0;
+  sampleInfo* srcInfo= getSampleInfo(filenameTable,
+                    filenameIdx, blockSize, maxDictSize, displayLevel);
+
+  /* set up zParams */
+  ZDICT_params_t zParams;
+  zParams.compressionLevel = cLevel;
+  zParams.notificationLevel = displayLevel;
+  zParams.dictID = dictID;
+
+  /* with no dict */
+  {
+    const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, NULL);
+    if(noDictResult) {
+      result = 1;
+      goto _cleanup;
+    }
+  }
+
+  /* for random */
+  {
+    ZDICT_random_params_t randomParam;
+    randomParam.zParams = zParams;
+    randomParam.k = k;
+    const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL);
+    DISPLAYLEVEL(2, "k=%u\n", randomParam.k);
+    if(randomResult) {
+      result = 1;
+      goto _cleanup;
+    }
+  }
+
+  /* for legacy */
+  {
+    ZDICT_legacy_params_t legacyParam;
+    legacyParam.zParams = zParams;
+    legacyParam.selectivityLevel = 9;
+    const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL);
+    DISPLAYLEVEL(2, "selectivityLevel=%u\n", legacyParam.selectivityLevel);
+    if(legacyResult) {
+      result = 1;
+      goto _cleanup;
+    }
+  }
+
+  /* for cover */
+  {
+    /* for cover (optimizing k and d) */
+    ZDICT_cover_params_t coverParam;
+    memset(&coverParam, 0, sizeof(coverParam));
+    coverParam.zParams = zParams;
+    coverParam.splitPoint = 1.0;
+    coverParam.steps = 40;
+    coverParam.nbThreads = 1;
+    const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
+    DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
+    if(coverOptResult) {
+      result = 1;
+      goto _cleanup;
+    }
+
+    /* for cover (with k and d provided) */
+    const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
+    DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
+    if(coverResult) {
+      result = 1;
+      goto _cleanup;
+    }
+
+  }
+
+  /* for fastCover */
+  for (f = 15; f < 25; f++){
+    DISPLAYLEVEL(2, "current f is %u\n", f);
+    for (accel = 1; accel < 11; accel++) {
+      DISPLAYLEVEL(2, "current accel is %u\n", accel);
+      /* for fastCover (optimizing k and d) */
+      ZDICT_fastCover_params_t fastParam;
+      memset(&fastParam, 0, sizeof(fastParam));
+      fastParam.zParams = zParams;
+      fastParam.f = f;
+      fastParam.steps = 40;
+      fastParam.nbThreads = 1;
+      fastParam.accel = accel;
+      const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
+      DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
+      if(fastOptResult) {
+        result = 1;
+        goto _cleanup;
+      }
+
+      /* for fastCover (with k and d provided) */
+      for (i = 0; i < 5; i++) {
+        const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
+        DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
+        if(fastResult) {
+          result = 1;
+          goto _cleanup;
+        }
+      }
+    }
+  }
+
+
+  /* Free allocated memory */
+_cleanup:
+  UTIL_freeFileList(extendedFileList, fileNamesBuf);
+  freeSampleInfo(srcInfo);
+  return result;
+}

contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h

+/* ZDICT_trainFromBuffer_legacy() :
+ * issue : samplesBuffer need to be followed by a noisy guard band.
+ * work around : duplicate the buffer, and add the noise */
+size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
+                                    const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+                                    ZDICT_legacy_params_t params);

contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh

+echo "Benchmark with in=../../lib/common"
+./benchmark in=../../../lib/common

contrib/experimental_dict_builders/fastCover/Makefile

+ARG :=
+
+CC ?= gcc
+CFLAGS ?= -O3 -g
+INCLUDES := -I ../../../programs -I ../randomDictBuilder -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
+
+IO_FILE := ../randomDictBuilder/io.c
+
+TEST_INPUT := ../../../lib
+TEST_OUTPUT := fastCoverDict
+
+all: main run clean
+
+.PHONY: test
+test: main testrun testshell clean
+
+.PHONY: run
+run:
+	echo "Building a fastCover dictionary with given arguments"
+	./main $(ARG)
+
+main: main.o io.o fastCover.o libzstd.a
+	$(CC) $(CFLAGS) main.o io.o fastCover.o libzstd.a -o main
+
+main.o: main.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c main.c
+
+fastCover.o: fastCover.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c fastCover.c
+
+io.o: $(IO_FILE)
+	$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
+
+libzstd.a:
+	$(MAKE) MOREFLAGS=-g -C ../../../lib libzstd.a
+	mv ../../../lib/libzstd.a .
+
+.PHONY: testrun
+testrun: main
+	echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) "
+	./main in=$(TEST_INPUT) out=$(TEST_OUTPUT)
+	zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q
+	rm -f $(TEST_OUTPUT)
+
+.PHONY: testshell
+testshell: test.sh
+	sh test.sh
+	echo "Finish running test.sh"
+
+.PHONY: clean
+clean:
+	rm -f *.o main libzstd.a
+	$(MAKE) -C ../../../lib clean
+	echo "Cleaning is completed"

contrib/experimental_dict_builders/fastCover/README.md

+FastCover Dictionary Builder
+
+### Permitted Arguments:
+Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in="
+Output Dictionary (out=dictName): if not provided, default to fastCoverDict
+Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0
+Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB
+Size of Selected Segment (k=#): positive number; in bytes; if not provided, default to 200
+Size of Dmer (d=#): either 6 or 8; if not provided, default to 8
+Number of steps (steps=#): positive number, if not provided, default to 32
+Percentage of samples used for training(split=#): positive number; if not provided, default to 100
+
+
+###Running Test:
+make test
+
+
+###Usage:
+To build a FASTCOVER dictionary with the provided arguments: make ARG= followed by arguments
+If k or d is not provided, the optimize version of FASTCOVER is run.
+
+### Examples:
+make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520"
+make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"

contrib/experimental_dict_builders/fastCover/fastCover.h

+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h> /* memset */
+#include <time.h>   /* clock */
+#include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#define ZDICT_STATIC_LINKING_ONLY
+#endif
+#include "zdict.h"
+
+
+typedef struct {
+    unsigned k;                  /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+    unsigned d;                  /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+    unsigned f;                  /* log of size of frequency array */
+    unsigned steps;              /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
+    unsigned nbThreads;          /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+    double splitPoint;           /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
+    ZDICT_params_t zParams;
+} ZDICT_fastCover_params_t;
+
+
+/*! ZDICT_optimizeTrainFromBuffer_fastCover():
+ *  Train a dictionary from an array of samples using a modified version of the COVER algorithm.
+ *  Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ *  supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ *  The resulting dictionary will be saved into `dictBuffer`.
+ *  All of the parameters except for f are optional.
+ *  If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
+ *  if steps is zero it defaults to its default value.
+ *  If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
+ *
+ *  @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ *           or an error code, which can be tested with ZDICT_isError().
+ *           On success `*parameters` contains the parameters selected.
+ */
+ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(
+     void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+     const size_t *samplesSizes, unsigned nbSamples,
+     ZDICT_fastCover_params_t *parameters);
+
+
+/*! ZDICT_trainFromBuffer_fastCover():
+ *  Train a dictionary from an array of samples using a modified version of the COVER algorithm.
+ *  Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ *  supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ *  The resulting dictionary will be saved into `dictBuffer`.
+ *  d, k, and f are required.
+ *  @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ *           or an error code, which can be tested with ZDICT_isError().
+ */
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
+    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+    const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters);

contrib/experimental_dict_builders/fastCover/main.c

+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h>   /* strcmp, strlen */
+#include <errno.h>    /* errno */
+#include <ctype.h>
+#include "fastCover.h"
+#include "io.h"
+#include "util.h"
+#include "zdict.h"
+
+
+/*-*************************************
+*  Console display
+***************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
+#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (displayLevel>=4) fflush(stderr); } } }
+
+
+/*-*************************************
+*  Exceptions
+***************************************/
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...)                                             \
+{                                                                         \
+    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+    DISPLAY("Error %i : ", error);                                        \
+    DISPLAY(__VA_ARGS__);                                                 \
+    DISPLAY("\n");                                                        \
+    exit(error);                                                          \
+}
+
+
+/*-*************************************
+*  Constants
+***************************************/
+static const unsigned g_defaultMaxDictSize = 110 KB;
+#define DEFAULT_CLEVEL 3
+
+
+/*-*************************************
+*  FASTCOVER
+***************************************/
+int FASTCOVER_trainFromFiles(const char* dictFileName, sampleInfo *info,
+                          unsigned maxDictSize,
+                          ZDICT_fastCover_params_t *params) {
+    unsigned const displayLevel = params->zParams.notificationLevel;
+    void* const dictBuffer = malloc(maxDictSize);
+
+    int result = 0;
+
+    /* Checks */
+    if (!dictBuffer)
+        EXM_THROW(12, "not enough memory for trainFromFiles");   /* should not happen */
+
+    {   size_t dictSize;
+        /* Run the optimize version if either k or d is not provided */
+        if (!params->d || !params->k) {
+          dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
+                                               info->samplesSizes, info->nbSamples, params);
+        } else {
+          dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
+                                               info->samplesSizes, info->nbSamples, *params);
+        }
+        DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint*100));
+        if (ZDICT_isError(dictSize)) {
+            DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize));   /* should not happen */
+            result = 1;
+            goto _done;
+        }
+        /* save dict */
+        DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
+        saveDict(dictFileName, dictBuffer, dictSize);
+    }
+
+    /* clean up */
+_done:
+    free(dictBuffer);
+    return result;
+}
+
+
+
+int main(int argCount, const char* argv[])
+{
+  int displayLevel = 2;
+  const char* programName = argv[0];
+  int operationResult = 0;
+
+  /* Initialize arguments to default values */
+  unsigned k = 0;
+  unsigned d = 0;
+  unsigned f = 23;
+  unsigned steps = 32;
+  unsigned nbThreads = 1;
+  unsigned split = 100;
+  const char* outputFile = "fastCoverDict";
+  unsigned dictID = 0;
+  unsigned maxDictSize = g_defaultMaxDictSize;
+
+  /* Initialize table to store input files */
+  const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));
+  unsigned filenameIdx = 0;
+
+  char* fileNamesBuf = NULL;
+  unsigned fileNamesNb = filenameIdx;
+  int followLinks = 0; /* follow directory recursively */
+  const char** extendedFileList = NULL;
+
+  /* Parse arguments */
+  for (int i = 1; i < argCount; i++) {
+    const char* argument = argv[i];
+    if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; }
+    if (longCommandWArg(&argument, "d=")) { d = readU32FromChar(&argument); continue; }
+    if (longCommandWArg(&argument, "f=")) { f = readU32FromChar(&argument); continue; }
+    if (longCommandWArg(&argument, "steps=")) { steps = readU32FromChar(&argument); continue; }
+    if (longCommandWArg(&argument, "split=")) { split = readU32FromChar(&argument); continue; }
+    if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; }
+    if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
+    if (longCommandWArg(&argument, "in=")) {
+      filenameTable[filenameIdx] = argument;
+      filenameIdx++;
+      continue;
+    }
+    if (longCommandWArg(&argument, "out=")) {
+      outputFile = argument;
+      continue;
+    }
+    DISPLAYLEVEL(1, "Incorrect parameters\n");
+    operationResult = 1;
+    return operationResult;
+  }
+
+  /* Get the list of all files recursively (because followLinks==0)*/
+  extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf,
+                                        &fileNamesNb, followLinks);
+  if (extendedFileList) {
+      unsigned u;
+      for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
+      free((void*)filenameTable);
+      filenameTable = extendedFileList;
+      filenameIdx = fileNamesNb;
+  }
+
+  size_t blockSize = 0;
+
+  /* Set up zParams */
+  ZDICT_params_t zParams;
+  zParams.compressionLevel = DEFAULT_CLEVEL;
+  zParams.notificationLevel = displayLevel;
+  zParams.dictID = dictID;
+
+  /* Set up fastCover params */
+  ZDICT_fastCover_params_t params;
+  params.zParams = zParams;
+  params.k = k;
+  params.d = d;
+  params.f = f;
+  params.steps = steps;
+  params.nbThreads = nbThreads;
+  params.splitPoint = (double)split/100;
+
+  /* Build dictionary */
+  sampleInfo* info = getSampleInfo(filenameTable,
+                    filenameIdx, blockSize, maxDictSize, zParams.notificationLevel);
+  operationResult = FASTCOVER_trainFromFiles(outputFile, info, maxDictSize, &params);
+
+  /* Free allocated memory */
+  UTIL_freeFileList(extendedFileList, fileNamesBuf);
+  freeSampleInfo(info);
+
+  return operationResult;
+}

contrib/experimental_dict_builders/fastCover/test.sh

+echo "Building fastCover dictionary with in=../../lib/common f=20 out=dict1"
+./main in=../../../lib/common f=20 out=dict1
+zstd -be3 -D dict1 -r ../../../lib/common -q
+echo "Building fastCover dictionary with in=../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000"
+./main in=../../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000
+zstd -be3 -D dict2 -r ../../../lib/common -q
+echo "Building fastCover dictionary with 2 sample sources"
+./main in=../../../lib/common in=../../../lib/compress out=dict3
+zstd -be3 -D dict3 -r ../../../lib/common -q
+echo "Removing dict1 dict2 dict3"
+rm -f dict1 dict2 dict3
+
+echo "Testing with invalid parameters, should fail"
+! ./main in=../../../lib/common r=10
+! ./main in=../../../lib/common d=10

contrib/experimental_dict_builders/randomDictBuilder/Makefile

+ARG :=
+
+CC ?= gcc
+CFLAGS ?= -O3
+INCLUDES := -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
+
+TEST_INPUT := ../../../lib
+TEST_OUTPUT := randomDict
+
+all: main run clean
+
+.PHONY: test
+test: main testrun testshell clean
+
+.PHONY: run
+run:
+	echo "Building a random dictionary with given arguments"
+	./main $(ARG)
+
+main: main.o io.o random.o libzstd.a
+	$(CC) $(CFLAGS) main.o io.o random.o libzstd.a -o main
+
+main.o: main.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c main.c
+
+random.o: random.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c random.c
+
+io.o: io.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c io.c
+
+libzstd.a:
+	$(MAKE) -C ../../../lib libzstd.a
+	mv ../../../lib/libzstd.a .
+
+.PHONY: testrun
+testrun: main
+	echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) "
+	./main in=$(TEST_INPUT) out=$(TEST_OUTPUT)
+	zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q
+	rm -f $(TEST_OUTPUT)
+
+.PHONY: testshell
+testshell: test.sh
+	sh test.sh
+	echo "Finish running test.sh"
+
+.PHONY: clean
+clean:
+	rm -f *.o main libzstd.a
+	$(MAKE) -C ../../../lib clean
+	echo "Cleaning is completed"