add zstd

zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+if [ false ]; then
+	for seed in $(seq 100); do
+		datagen -g1000 -s$seed > file$seed
+	done
+
+	zstd --train -r . -o dict0 -qq
+
+	for seed in $(seq 101 200); do
+		datagen -g1000 -s$seed > file$seed
+	done
+
+	zstd --train -r . -o dict1 -qq
+
+	[ "$($MD5SUM < dict0)" != "$($MD5SUM < dict1)" ] || die "dictionaries must not match"
+
+	datagen -g1000 -s0 > file0
+fi
+
+set -v
+zstd files/0 -D dicts/0 -q
+zstd -t files/0.zst -D dicts/0
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
+zstd -t files/0.zst            && die "Must fail" ||:

zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact

@@ -0,0 +1,7 @@
+zstd files/0 -D dicts/0 -q
+zstd -t files/0.zst -D dicts/0
+files/0.zst         : 1000 bytes 
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
+files/0.zst : Decoding error (36) : Dictionary mismatch 
+zstd -t files/0.zst            && die "Must fail" ||:
+files/0.zst : Decoding error (36) : Dictionary mismatch 

zstd/tests/cli-tests/dictionaries/golden.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/"
+
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst

zstd/tests/cli-tests/dictionaries/setup

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+cp -r ../files .
+cp -r ../dicts .

zstd/tests/cli-tests/dictionaries/setup_once

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+
+mkdir files/ dicts/
+
+for seed in $(seq 50); do
+	datagen -g1000 -s$seed > files/$seed
+done
+
+zstd --train -r files -o dicts/0 -qq
+
+for seed in $(seq 51 100); do
+	datagen -g1000 -s$seed > files/$seed
+done
+
+zstd --train -r files -o dicts/1 -qq
+
+cmp dicts/0 dicts/1 && die "dictionaries must not match!"
+
+datagen -g1000 > files/0