aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig11
-rw-r--r--lib/Kconfig.debug3
-rw-r--r--lib/Makefile3
-rw-r--r--lib/idr.c2
-rw-r--r--lib/iov_iter.c6
-rw-r--r--lib/rhashtable.c9
-rw-r--r--lib/string_helpers.c4
-rw-r--r--lib/test_bitmap.c8
-rw-r--r--lib/xxhash.c500
-rw-r--r--lib/zstd/Makefile18
-rw-r--r--lib/zstd/bitstream.h374
-rw-r--r--lib/zstd/compress.c3484
-rw-r--r--lib/zstd/decompress.c2528
-rw-r--r--lib/zstd/entropy_common.c243
-rw-r--r--lib/zstd/error_private.h53
-rw-r--r--lib/zstd/fse.h575
-rw-r--r--lib/zstd/fse_compress.c795
-rw-r--r--lib/zstd/fse_decompress.c332
-rw-r--r--lib/zstd/huf.h212
-rw-r--r--lib/zstd/huf_compress.c770
-rw-r--r--lib/zstd/huf_decompress.c960
-rw-r--r--lib/zstd/mem.h151
-rw-r--r--lib/zstd/zstd_common.c75
-rw-r--r--lib/zstd/zstd_internal.h263
-rw-r--r--lib/zstd/zstd_opt.h1014
25 files changed, 12379 insertions, 14 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index a85e6f76add5..b1445b22a6de 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -192,6 +192,9 @@ config CRC8
192 when they need to do cyclic redundancy check according CRC8 192 when they need to do cyclic redundancy check according CRC8
193 algorithm. Module will be called crc8. 193 algorithm. Module will be called crc8.
194 194
195config XXHASH
196 tristate
197
195config AUDIT_GENERIC 198config AUDIT_GENERIC
196 bool 199 bool
197 depends on AUDIT && !AUDIT_ARCH 200 depends on AUDIT && !AUDIT_ARCH
@@ -246,6 +249,14 @@ config LZ4HC_COMPRESS
246config LZ4_DECOMPRESS 249config LZ4_DECOMPRESS
247 tristate 250 tristate
248 251
252config ZSTD_COMPRESS
253 select XXHASH
254 tristate
255
256config ZSTD_DECOMPRESS
257 select XXHASH
258 tristate
259
249source "lib/xz/Kconfig" 260source "lib/xz/Kconfig"
250 261
251# 262#
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index b19c491cbc4e..2689b7c50c52 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -219,7 +219,8 @@ config FRAME_WARN
219 range 0 8192 219 range 0 8192
220 default 0 if KASAN 220 default 0 if KASAN
221 default 2048 if GCC_PLUGIN_LATENT_ENTROPY 221 default 2048 if GCC_PLUGIN_LATENT_ENTROPY
222 default 1024 if !64BIT 222 default 1280 if (!64BIT && PARISC)
223 default 1024 if (!64BIT && !PARISC)
223 default 2048 if 64BIT 224 default 2048 if 64BIT
224 help 225 help
225 Tell gcc to warn at build time for stack frames larger than this. 226 Tell gcc to warn at build time for stack frames larger than this.
diff --git a/lib/Makefile b/lib/Makefile
index 469ce5e24e4f..dafa79613fb4 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -103,6 +103,7 @@ obj-$(CONFIG_CRC4) += crc4.o
103obj-$(CONFIG_CRC7) += crc7.o 103obj-$(CONFIG_CRC7) += crc7.o
104obj-$(CONFIG_LIBCRC32C) += libcrc32c.o 104obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
105obj-$(CONFIG_CRC8) += crc8.o 105obj-$(CONFIG_CRC8) += crc8.o
106obj-$(CONFIG_XXHASH) += xxhash.o
106obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o 107obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
107 108
108obj-$(CONFIG_842_COMPRESS) += 842/ 109obj-$(CONFIG_842_COMPRESS) += 842/
@@ -116,6 +117,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
116obj-$(CONFIG_LZ4_COMPRESS) += lz4/ 117obj-$(CONFIG_LZ4_COMPRESS) += lz4/
117obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/ 118obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
118obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ 119obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
120obj-$(CONFIG_ZSTD_COMPRESS) += zstd/
121obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/
119obj-$(CONFIG_XZ_DEC) += xz/ 122obj-$(CONFIG_XZ_DEC) += xz/
120obj-$(CONFIG_RAID6_PQ) += raid6/ 123obj-$(CONFIG_RAID6_PQ) += raid6/
121 124
diff --git a/lib/idr.c b/lib/idr.c
index 082778cf883e..f9adf4805fd7 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -151,7 +151,7 @@ EXPORT_SYMBOL(idr_get_next_ext);
151 */ 151 */
152void *idr_replace(struct idr *idr, void *ptr, int id) 152void *idr_replace(struct idr *idr, void *ptr, int id)
153{ 153{
154 if (WARN_ON_ONCE(id < 0)) 154 if (id < 0)
155 return ERR_PTR(-EINVAL); 155 return ERR_PTR(-EINVAL);
156 156
157 return idr_replace_ext(idr, ptr, id); 157 return idr_replace_ext(idr, ptr, id);
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 52c8dd6d8e82..1c1c06ddc20a 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -687,8 +687,10 @@ EXPORT_SYMBOL(_copy_from_iter_full_nocache);
687 687
688static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) 688static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
689{ 689{
690 size_t v = n + offset; 690 struct page *head = compound_head(page);
691 if (likely(n <= v && v <= (PAGE_SIZE << compound_order(page)))) 691 size_t v = n + offset + page_address(page) - page_address(head);
692
693 if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
692 return true; 694 return true;
693 WARN_ON(1); 695 WARN_ON(1);
694 return false; 696 return false;
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 707ca5d677c6..ddd7dde87c3c 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -735,9 +735,9 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit);
735 * rhashtable_walk_start - Start a hash table walk 735 * rhashtable_walk_start - Start a hash table walk
736 * @iter: Hash table iterator 736 * @iter: Hash table iterator
737 * 737 *
738 * Start a hash table walk. Note that we take the RCU lock in all 738 * Start a hash table walk at the current iterator position. Note that we take
739 * cases including when we return an error. So you must always call 739 * the RCU lock in all cases including when we return an error. So you must
740 * rhashtable_walk_stop to clean up. 740 * always call rhashtable_walk_stop to clean up.
741 * 741 *
742 * Returns zero if successful. 742 * Returns zero if successful.
743 * 743 *
@@ -846,7 +846,8 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_next);
846 * rhashtable_walk_stop - Finish a hash table walk 846 * rhashtable_walk_stop - Finish a hash table walk
847 * @iter: Hash table iterator 847 * @iter: Hash table iterator
848 * 848 *
849 * Finish a hash table walk. 849 * Finish a hash table walk. Does not reset the iterator to the start of the
850 * hash table.
850 */ 851 */
851void rhashtable_walk_stop(struct rhashtable_iter *iter) 852void rhashtable_walk_stop(struct rhashtable_iter *iter)
852 __releases(RCU) 853 __releases(RCU)
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index ecaac2c0526f..29c490e5d478 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -576,7 +576,7 @@ char *kstrdup_quotable_cmdline(struct task_struct *task, gfp_t gfp)
576 char *buffer, *quoted; 576 char *buffer, *quoted;
577 int i, res; 577 int i, res;
578 578
579 buffer = kmalloc(PAGE_SIZE, GFP_TEMPORARY); 579 buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
580 if (!buffer) 580 if (!buffer)
581 return NULL; 581 return NULL;
582 582
@@ -612,7 +612,7 @@ char *kstrdup_quotable_file(struct file *file, gfp_t gfp)
612 return kstrdup("<unknown>", gfp); 612 return kstrdup("<unknown>", gfp);
613 613
614 /* We add 11 spaces for ' (deleted)' to be appended */ 614 /* We add 11 spaces for ' (deleted)' to be appended */
615 temp = kmalloc(PATH_MAX + 11, GFP_TEMPORARY); 615 temp = kmalloc(PATH_MAX + 11, GFP_KERNEL);
616 if (!temp) 616 if (!temp)
617 return kstrdup("<no_memory>", gfp); 617 return kstrdup("<no_memory>", gfp);
618 618
diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c
index 599c6713f2a2..aa1f2669bdd5 100644
--- a/lib/test_bitmap.c
+++ b/lib/test_bitmap.c
@@ -186,13 +186,13 @@ static const unsigned long exp[] __initconst = {
186 BITMAP_FROM_U64(0x22222222), 186 BITMAP_FROM_U64(0x22222222),
187 BITMAP_FROM_U64(0xffffffff), 187 BITMAP_FROM_U64(0xffffffff),
188 BITMAP_FROM_U64(0xfffffffe), 188 BITMAP_FROM_U64(0xfffffffe),
189 BITMAP_FROM_U64(0x3333333311111111), 189 BITMAP_FROM_U64(0x3333333311111111ULL),
190 BITMAP_FROM_U64(0xffffffff77777777) 190 BITMAP_FROM_U64(0xffffffff77777777ULL)
191}; 191};
192 192
193static const unsigned long exp2[] __initconst = { 193static const unsigned long exp2[] __initconst = {
194 BITMAP_FROM_U64(0x3333333311111111), 194 BITMAP_FROM_U64(0x3333333311111111ULL),
195 BITMAP_FROM_U64(0xffffffff77777777) 195 BITMAP_FROM_U64(0xffffffff77777777ULL)
196}; 196};
197 197
198static const struct test_bitmap_parselist parselist_tests[] __initconst = { 198static const struct test_bitmap_parselist parselist_tests[] __initconst = {
diff --git a/lib/xxhash.c b/lib/xxhash.c
new file mode 100644
index 000000000000..aa61e2a3802f
--- /dev/null
+++ b/lib/xxhash.c
@@ -0,0 +1,500 @@
1/*
2 * xxHash - Extremely Fast Hash algorithm
3 * Copyright (C) 2012-2016, Yann Collet.
4 *
5 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * This program is free software; you can redistribute it and/or modify it under
31 * the terms of the GNU General Public License version 2 as published by the
32 * Free Software Foundation. This program is dual-licensed; you may select
33 * either version 2 of the GNU General Public License ("GPL") or BSD license
34 * ("BSD").
35 *
36 * You can contact the author at:
37 * - xxHash homepage: http://cyan4973.github.io/xxHash/
38 * - xxHash source repository: https://github.com/Cyan4973/xxHash
39 */
40
41#include <asm/unaligned.h>
42#include <linux/errno.h>
43#include <linux/compiler.h>
44#include <linux/kernel.h>
45#include <linux/module.h>
46#include <linux/string.h>
47#include <linux/xxhash.h>
48
49/*-*************************************
50 * Macros
51 **************************************/
52#define xxh_rotl32(x, r) ((x << r) | (x >> (32 - r)))
53#define xxh_rotl64(x, r) ((x << r) | (x >> (64 - r)))
54
55#ifdef __LITTLE_ENDIAN
56# define XXH_CPU_LITTLE_ENDIAN 1
57#else
58# define XXH_CPU_LITTLE_ENDIAN 0
59#endif
60
61/*-*************************************
62 * Constants
63 **************************************/
64static const uint32_t PRIME32_1 = 2654435761U;
65static const uint32_t PRIME32_2 = 2246822519U;
66static const uint32_t PRIME32_3 = 3266489917U;
67static const uint32_t PRIME32_4 = 668265263U;
68static const uint32_t PRIME32_5 = 374761393U;
69
70static const uint64_t PRIME64_1 = 11400714785074694791ULL;
71static const uint64_t PRIME64_2 = 14029467366897019727ULL;
72static const uint64_t PRIME64_3 = 1609587929392839161ULL;
73static const uint64_t PRIME64_4 = 9650029242287828579ULL;
74static const uint64_t PRIME64_5 = 2870177450012600261ULL;
75
76/*-**************************
77 * Utils
78 ***************************/
79void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src)
80{
81 memcpy(dst, src, sizeof(*dst));
82}
83EXPORT_SYMBOL(xxh32_copy_state);
84
85void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src)
86{
87 memcpy(dst, src, sizeof(*dst));
88}
89EXPORT_SYMBOL(xxh64_copy_state);
90
91/*-***************************
92 * Simple Hash Functions
93 ****************************/
94static uint32_t xxh32_round(uint32_t seed, const uint32_t input)
95{
96 seed += input * PRIME32_2;
97 seed = xxh_rotl32(seed, 13);
98 seed *= PRIME32_1;
99 return seed;
100}
101
102uint32_t xxh32(const void *input, const size_t len, const uint32_t seed)
103{
104 const uint8_t *p = (const uint8_t *)input;
105 const uint8_t *b_end = p + len;
106 uint32_t h32;
107
108 if (len >= 16) {
109 const uint8_t *const limit = b_end - 16;
110 uint32_t v1 = seed + PRIME32_1 + PRIME32_2;
111 uint32_t v2 = seed + PRIME32_2;
112 uint32_t v3 = seed + 0;
113 uint32_t v4 = seed - PRIME32_1;
114
115 do {
116 v1 = xxh32_round(v1, get_unaligned_le32(p));
117 p += 4;
118 v2 = xxh32_round(v2, get_unaligned_le32(p));
119 p += 4;
120 v3 = xxh32_round(v3, get_unaligned_le32(p));
121 p += 4;
122 v4 = xxh32_round(v4, get_unaligned_le32(p));
123 p += 4;
124 } while (p <= limit);
125
126 h32 = xxh_rotl32(v1, 1) + xxh_rotl32(v2, 7) +
127 xxh_rotl32(v3, 12) + xxh_rotl32(v4, 18);
128 } else {
129 h32 = seed + PRIME32_5;
130 }
131
132 h32 += (uint32_t)len;
133
134 while (p + 4 <= b_end) {
135 h32 += get_unaligned_le32(p) * PRIME32_3;
136 h32 = xxh_rotl32(h32, 17) * PRIME32_4;
137 p += 4;
138 }
139
140 while (p < b_end) {
141 h32 += (*p) * PRIME32_5;
142 h32 = xxh_rotl32(h32, 11) * PRIME32_1;
143 p++;
144 }
145
146 h32 ^= h32 >> 15;
147 h32 *= PRIME32_2;
148 h32 ^= h32 >> 13;
149 h32 *= PRIME32_3;
150 h32 ^= h32 >> 16;
151
152 return h32;
153}
154EXPORT_SYMBOL(xxh32);
155
156static uint64_t xxh64_round(uint64_t acc, const uint64_t input)
157{
158 acc += input * PRIME64_2;
159 acc = xxh_rotl64(acc, 31);
160 acc *= PRIME64_1;
161 return acc;
162}
163
164static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val)
165{
166 val = xxh64_round(0, val);
167 acc ^= val;
168 acc = acc * PRIME64_1 + PRIME64_4;
169 return acc;
170}
171
172uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
173{
174 const uint8_t *p = (const uint8_t *)input;
175 const uint8_t *const b_end = p + len;
176 uint64_t h64;
177
178 if (len >= 32) {
179 const uint8_t *const limit = b_end - 32;
180 uint64_t v1 = seed + PRIME64_1 + PRIME64_2;
181 uint64_t v2 = seed + PRIME64_2;
182 uint64_t v3 = seed + 0;
183 uint64_t v4 = seed - PRIME64_1;
184
185 do {
186 v1 = xxh64_round(v1, get_unaligned_le64(p));
187 p += 8;
188 v2 = xxh64_round(v2, get_unaligned_le64(p));
189 p += 8;
190 v3 = xxh64_round(v3, get_unaligned_le64(p));
191 p += 8;
192 v4 = xxh64_round(v4, get_unaligned_le64(p));
193 p += 8;
194 } while (p <= limit);
195
196 h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
197 xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
198 h64 = xxh64_merge_round(h64, v1);
199 h64 = xxh64_merge_round(h64, v2);
200 h64 = xxh64_merge_round(h64, v3);
201 h64 = xxh64_merge_round(h64, v4);
202
203 } else {
204 h64 = seed + PRIME64_5;
205 }
206
207 h64 += (uint64_t)len;
208
209 while (p + 8 <= b_end) {
210 const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
211
212 h64 ^= k1;
213 h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
214 p += 8;
215 }
216
217 if (p + 4 <= b_end) {
218 h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
219 h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
220 p += 4;
221 }
222
223 while (p < b_end) {
224 h64 ^= (*p) * PRIME64_5;
225 h64 = xxh_rotl64(h64, 11) * PRIME64_1;
226 p++;
227 }
228
229 h64 ^= h64 >> 33;
230 h64 *= PRIME64_2;
231 h64 ^= h64 >> 29;
232 h64 *= PRIME64_3;
233 h64 ^= h64 >> 32;
234
235 return h64;
236}
237EXPORT_SYMBOL(xxh64);
238
239/*-**************************************************
240 * Advanced Hash Functions
241 ***************************************************/
242void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed)
243{
244 /* use a local state for memcpy() to avoid strict-aliasing warnings */
245 struct xxh32_state state;
246
247 memset(&state, 0, sizeof(state));
248 state.v1 = seed + PRIME32_1 + PRIME32_2;
249 state.v2 = seed + PRIME32_2;
250 state.v3 = seed + 0;
251 state.v4 = seed - PRIME32_1;
252 memcpy(statePtr, &state, sizeof(state));
253}
254EXPORT_SYMBOL(xxh32_reset);
255
256void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
257{
258 /* use a local state for memcpy() to avoid strict-aliasing warnings */
259 struct xxh64_state state;
260
261 memset(&state, 0, sizeof(state));
262 state.v1 = seed + PRIME64_1 + PRIME64_2;
263 state.v2 = seed + PRIME64_2;
264 state.v3 = seed + 0;
265 state.v4 = seed - PRIME64_1;
266 memcpy(statePtr, &state, sizeof(state));
267}
268EXPORT_SYMBOL(xxh64_reset);
269
270int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
271{
272 const uint8_t *p = (const uint8_t *)input;
273 const uint8_t *const b_end = p + len;
274
275 if (input == NULL)
276 return -EINVAL;
277
278 state->total_len_32 += (uint32_t)len;
279 state->large_len |= (len >= 16) | (state->total_len_32 >= 16);
280
281 if (state->memsize + len < 16) { /* fill in tmp buffer */
282 memcpy((uint8_t *)(state->mem32) + state->memsize, input, len);
283 state->memsize += (uint32_t)len;
284 return 0;
285 }
286
287 if (state->memsize) { /* some data left from previous update */
288 const uint32_t *p32 = state->mem32;
289
290 memcpy((uint8_t *)(state->mem32) + state->memsize, input,
291 16 - state->memsize);
292
293 state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32));
294 p32++;
295 state->v2 = xxh32_round(state->v2, get_unaligned_le32(p32));
296 p32++;
297 state->v3 = xxh32_round(state->v3, get_unaligned_le32(p32));
298 p32++;
299 state->v4 = xxh32_round(state->v4, get_unaligned_le32(p32));
300 p32++;
301
302 p += 16-state->memsize;
303 state->memsize = 0;
304 }
305
306 if (p <= b_end - 16) {
307 const uint8_t *const limit = b_end - 16;
308 uint32_t v1 = state->v1;
309 uint32_t v2 = state->v2;
310 uint32_t v3 = state->v3;
311 uint32_t v4 = state->v4;
312
313 do {
314 v1 = xxh32_round(v1, get_unaligned_le32(p));
315 p += 4;
316 v2 = xxh32_round(v2, get_unaligned_le32(p));
317 p += 4;
318 v3 = xxh32_round(v3, get_unaligned_le32(p));
319 p += 4;
320 v4 = xxh32_round(v4, get_unaligned_le32(p));
321 p += 4;
322 } while (p <= limit);
323
324 state->v1 = v1;
325 state->v2 = v2;
326 state->v3 = v3;
327 state->v4 = v4;
328 }
329
330 if (p < b_end) {
331 memcpy(state->mem32, p, (size_t)(b_end-p));
332 state->memsize = (uint32_t)(b_end-p);
333 }
334
335 return 0;
336}
337EXPORT_SYMBOL(xxh32_update);
338
339uint32_t xxh32_digest(const struct xxh32_state *state)
340{
341 const uint8_t *p = (const uint8_t *)state->mem32;
342 const uint8_t *const b_end = (const uint8_t *)(state->mem32) +
343 state->memsize;
344 uint32_t h32;
345
346 if (state->large_len) {
347 h32 = xxh_rotl32(state->v1, 1) + xxh_rotl32(state->v2, 7) +
348 xxh_rotl32(state->v3, 12) + xxh_rotl32(state->v4, 18);
349 } else {
350 h32 = state->v3 /* == seed */ + PRIME32_5;
351 }
352
353 h32 += state->total_len_32;
354
355 while (p + 4 <= b_end) {
356 h32 += get_unaligned_le32(p) * PRIME32_3;
357 h32 = xxh_rotl32(h32, 17) * PRIME32_4;
358 p += 4;
359 }
360
361 while (p < b_end) {
362 h32 += (*p) * PRIME32_5;
363 h32 = xxh_rotl32(h32, 11) * PRIME32_1;
364 p++;
365 }
366
367 h32 ^= h32 >> 15;
368 h32 *= PRIME32_2;
369 h32 ^= h32 >> 13;
370 h32 *= PRIME32_3;
371 h32 ^= h32 >> 16;
372
373 return h32;
374}
375EXPORT_SYMBOL(xxh32_digest);
376
377int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
378{
379 const uint8_t *p = (const uint8_t *)input;
380 const uint8_t *const b_end = p + len;
381
382 if (input == NULL)
383 return -EINVAL;
384
385 state->total_len += len;
386
387 if (state->memsize + len < 32) { /* fill in tmp buffer */
388 memcpy(((uint8_t *)state->mem64) + state->memsize, input, len);
389 state->memsize += (uint32_t)len;
390 return 0;
391 }
392
393 if (state->memsize) { /* tmp buffer is full */
394 uint64_t *p64 = state->mem64;
395
396 memcpy(((uint8_t *)p64) + state->memsize, input,
397 32 - state->memsize);
398
399 state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64));
400 p64++;
401 state->v2 = xxh64_round(state->v2, get_unaligned_le64(p64));
402 p64++;
403 state->v3 = xxh64_round(state->v3, get_unaligned_le64(p64));
404 p64++;
405 state->v4 = xxh64_round(state->v4, get_unaligned_le64(p64));
406
407 p += 32 - state->memsize;
408 state->memsize = 0;
409 }
410
411 if (p + 32 <= b_end) {
412 const uint8_t *const limit = b_end - 32;
413 uint64_t v1 = state->v1;
414 uint64_t v2 = state->v2;
415 uint64_t v3 = state->v3;
416 uint64_t v4 = state->v4;
417
418 do {
419 v1 = xxh64_round(v1, get_unaligned_le64(p));
420 p += 8;
421 v2 = xxh64_round(v2, get_unaligned_le64(p));
422 p += 8;
423 v3 = xxh64_round(v3, get_unaligned_le64(p));
424 p += 8;
425 v4 = xxh64_round(v4, get_unaligned_le64(p));
426 p += 8;
427 } while (p <= limit);
428
429 state->v1 = v1;
430 state->v2 = v2;
431 state->v3 = v3;
432 state->v4 = v4;
433 }
434
435 if (p < b_end) {
436 memcpy(state->mem64, p, (size_t)(b_end-p));
437 state->memsize = (uint32_t)(b_end - p);
438 }
439
440 return 0;
441}
442EXPORT_SYMBOL(xxh64_update);
443
444uint64_t xxh64_digest(const struct xxh64_state *state)
445{
446 const uint8_t *p = (const uint8_t *)state->mem64;
447 const uint8_t *const b_end = (const uint8_t *)state->mem64 +
448 state->memsize;
449 uint64_t h64;
450
451 if (state->total_len >= 32) {
452 const uint64_t v1 = state->v1;
453 const uint64_t v2 = state->v2;
454 const uint64_t v3 = state->v3;
455 const uint64_t v4 = state->v4;
456
457 h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
458 xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
459 h64 = xxh64_merge_round(h64, v1);
460 h64 = xxh64_merge_round(h64, v2);
461 h64 = xxh64_merge_round(h64, v3);
462 h64 = xxh64_merge_round(h64, v4);
463 } else {
464 h64 = state->v3 + PRIME64_5;
465 }
466
467 h64 += (uint64_t)state->total_len;
468
469 while (p + 8 <= b_end) {
470 const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
471
472 h64 ^= k1;
473 h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
474 p += 8;
475 }
476
477 if (p + 4 <= b_end) {
478 h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
479 h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
480 p += 4;
481 }
482
483 while (p < b_end) {
484 h64 ^= (*p) * PRIME64_5;
485 h64 = xxh_rotl64(h64, 11) * PRIME64_1;
486 p++;
487 }
488
489 h64 ^= h64 >> 33;
490 h64 *= PRIME64_2;
491 h64 ^= h64 >> 29;
492 h64 *= PRIME64_3;
493 h64 ^= h64 >> 32;
494
495 return h64;
496}
497EXPORT_SYMBOL(xxh64_digest);
498
499MODULE_LICENSE("Dual BSD/GPL");
500MODULE_DESCRIPTION("xxHash");
diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile
new file mode 100644
index 000000000000..dd0a359c135b
--- /dev/null
+++ b/lib/zstd/Makefile
@@ -0,0 +1,18 @@
1obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
2obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
3
4ccflags-y += -O3
5
6# Object files unique to zstd_compress and zstd_decompress
7zstd_compress-y := fse_compress.o huf_compress.o compress.o
8zstd_decompress-y := huf_decompress.o decompress.o
9
10# These object files are shared between the modules.
11# Always add them to zstd_compress.
12# Unless both zstd_compress and zstd_decompress are built in
13# then also add them to zstd_decompress.
14zstd_compress-y += entropy_common.o fse_decompress.o zstd_common.o
15
16ifneq ($(CONFIG_ZSTD_COMPRESS)$(CONFIG_ZSTD_DECOMPRESS),yy)
17 zstd_decompress-y += entropy_common.o fse_decompress.o zstd_common.o
18endif
diff --git a/lib/zstd/bitstream.h b/lib/zstd/bitstream.h
new file mode 100644
index 000000000000..a826b99e1d63
--- /dev/null
+++ b/lib/zstd/bitstream.h
@@ -0,0 +1,374 @@
1/*
2 * bitstream
3 * Part of FSE library
4 * header file (to include)
5 * Copyright (C) 2013-2016, Yann Collet.
6 *
7 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions are
11 * met:
12 *
13 * * Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 *
32 * This program is free software; you can redistribute it and/or modify it under
33 * the terms of the GNU General Public License version 2 as published by the
34 * Free Software Foundation. This program is dual-licensed; you may select
35 * either version 2 of the GNU General Public License ("GPL") or BSD license
36 * ("BSD").
37 *
38 * You can contact the author at :
39 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
40 */
41#ifndef BITSTREAM_H_MODULE
42#define BITSTREAM_H_MODULE
43
44/*
45* This API consists of small unitary functions, which must be inlined for best performance.
46* Since link-time-optimization is not available for all compilers,
47* these functions are defined into a .h to be included.
48*/
49
50/*-****************************************
51* Dependencies
52******************************************/
53#include "error_private.h" /* error codes and messages */
54#include "mem.h" /* unaligned access routines */
55
56/*=========================================
57* Target specific
58=========================================*/
59#define STREAM_ACCUMULATOR_MIN_32 25
60#define STREAM_ACCUMULATOR_MIN_64 57
61#define STREAM_ACCUMULATOR_MIN ((U32)(ZSTD_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
62
63/*-******************************************
64* bitStream encoding API (write forward)
65********************************************/
66/* bitStream can mix input from multiple sources.
67* A critical property of these streams is that they encode and decode in **reverse** direction.
68* So the first bit sequence you add will be the last to be read, like a LIFO stack.
69*/
70typedef struct {
71 size_t bitContainer;
72 int bitPos;
73 char *startPtr;
74 char *ptr;
75 char *endPtr;
76} BIT_CStream_t;
77
78ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size_t dstCapacity);
79ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
80ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC);
81ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC);
82
83/* Start with initCStream, providing the size of buffer to write into.
84* bitStream will never write outside of this buffer.
85* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
86*
87* bits are first added to a local register.
88* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
89* Writing data into memory is an explicit operation, performed by the flushBits function.
90* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
91* After a flushBits, a maximum of 7 bits might still be stored into local register.
92*
93* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
94*
95* Last operation is to close the bitStream.
96* The function returns the final size of CStream in bytes.
97* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
98*/
99
100/*-********************************************
101* bitStream decoding API (read backward)
102**********************************************/
103typedef struct {
104 size_t bitContainer;
105 unsigned bitsConsumed;
106 const char *ptr;
107 const char *start;
108} BIT_DStream_t;
109
110typedef enum {
111 BIT_DStream_unfinished = 0,
112 BIT_DStream_endOfBuffer = 1,
113 BIT_DStream_completed = 2,
114 BIT_DStream_overflow = 3
115} BIT_DStream_status; /* result of BIT_reloadDStream() */
116/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
117
118ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize);
119ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, unsigned nbBits);
120ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD);
121ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD);
122
123/* Start by invoking BIT_initDStream().
124* A chunk of the bitStream is then stored into a local register.
125* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
126* You can then retrieve bitFields stored into the local register, **in reverse order**.
127* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
128* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
129* Otherwise, it can be less than that, so proceed accordingly.
130* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
131*/
132
133/*-****************************************
134* unsafe API
135******************************************/
136ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
137/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
138
139ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC);
140/* unsafe version; does not check buffer overflow */
141
142ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits);
143/* faster, but works only if nbBits >= 1 */
144
145/*-**************************************************************
146* Internal functions
147****************************************************************/
148ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); }
149
150/*===== Local Constants =====*/
151static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF,
152 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
153 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */
154
155/*-**************************************************************
156* bitStream encoding
157****************************************************************/
158/*! BIT_initCStream() :
159 * `dstCapacity` must be > sizeof(void*)
160 * @return : 0 if success,
161 otherwise an error code (can be tested using ERR_isError() ) */
162ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *startPtr, size_t dstCapacity)
163{
164 bitC->bitContainer = 0;
165 bitC->bitPos = 0;
166 bitC->startPtr = (char *)startPtr;
167 bitC->ptr = bitC->startPtr;
168 bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
169 if (dstCapacity <= sizeof(bitC->ptr))
170 return ERROR(dstSize_tooSmall);
171 return 0;
172}
173
174/*! BIT_addBits() :
175 can add up to 26 bits into `bitC`.
176 Does not check for register overflow ! */
177ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
178{
179 bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
180 bitC->bitPos += nbBits;
181}
182
183/*! BIT_addBitsFast() :
184 * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
185ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
186{
187 bitC->bitContainer |= value << bitC->bitPos;
188 bitC->bitPos += nbBits;
189}
190
191/*! BIT_flushBitsFast() :
192 * unsafe version; does not check buffer overflow */
193ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC)
194{
195 size_t const nbBytes = bitC->bitPos >> 3;
196 ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
197 bitC->ptr += nbBytes;
198 bitC->bitPos &= 7;
199 bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
200}
201
202/*! BIT_flushBits() :
203 * safe version; check for buffer overflow, and prevents it.
204 * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
205ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC)
206{
207 size_t const nbBytes = bitC->bitPos >> 3;
208 ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
209 bitC->ptr += nbBytes;
210 if (bitC->ptr > bitC->endPtr)
211 bitC->ptr = bitC->endPtr;
212 bitC->bitPos &= 7;
213 bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
214}
215
216/*! BIT_closeCStream() :
217 * @return : size of CStream, in bytes,
218 or 0 if it could not fit into dstBuffer */
219ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC)
220{
221 BIT_addBitsFast(bitC, 1, 1); /* endMark */
222 BIT_flushBits(bitC);
223
224 if (bitC->ptr >= bitC->endPtr)
225 return 0; /* doesn't fit within authorized budget : cancel */
226
227 return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
228}
229
230/*-********************************************************
231* bitStream decoding
232**********************************************************/
233/*! BIT_initDStream() :
234* Initialize a BIT_DStream_t.
235* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
236* `srcSize` must be the *exact* size of the bitStream, in bytes.
237* @return : size of stream (== srcSize) or an errorCode if a problem is detected
238*/
239ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize)
240{
241 if (srcSize < 1) {
242 memset(bitD, 0, sizeof(*bitD));
243 return ERROR(srcSize_wrong);
244 }
245
246 if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
247 bitD->start = (const char *)srcBuffer;
248 bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer);
249 bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
250 {
251 BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
252 bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
253 if (lastByte == 0)
254 return ERROR(GENERIC); /* endMark not present */
255 }
256 } else {
257 bitD->start = (const char *)srcBuffer;
258 bitD->ptr = bitD->start;
259 bitD->bitContainer = *(const BYTE *)(bitD->start);
260 switch (srcSize) {
261 case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16);
262 case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24);
263 case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32);
264 case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24;
265 case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16;
266 case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8;
267 default:;
268 }
269 {
270 BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
271 bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
272 if (lastByte == 0)
273 return ERROR(GENERIC); /* endMark not present */
274 }
275 bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8;
276 }
277
278 return srcSize;
279}
280
281ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; }
282
283ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; }
284
285ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; }
286
287/*! BIT_lookBits() :
288 * Provides next n bits from local register.
289 * local register is not modified.
290 * On 32-bits, maxNbBits==24.
291 * On 64-bits, maxNbBits==56.
292 * @return : value extracted
293 */
294ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t *bitD, U32 nbBits)
295{
296 U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
297 return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask - nbBits) & bitMask);
298}
299
300/*! BIT_lookBitsFast() :
301* unsafe version; only works only if nbBits >= 1 */
302ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t *bitD, U32 nbBits)
303{
304 U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
305 return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask);
306}
307
308ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; }
309
310/*! BIT_readBits() :
311 * Read (consume) next n bits from local register and update.
312 * Pay attention to not read more than nbBits contained into local register.
313 * @return : extracted value.
314 */
315ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, U32 nbBits)
316{
317 size_t const value = BIT_lookBits(bitD, nbBits);
318 BIT_skipBits(bitD, nbBits);
319 return value;
320}
321
322/*! BIT_readBitsFast() :
323* unsafe version; only works only if nbBits >= 1 */
324ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, U32 nbBits)
325{
326 size_t const value = BIT_lookBitsFast(bitD, nbBits);
327 BIT_skipBits(bitD, nbBits);
328 return value;
329}
330
331/*! BIT_reloadDStream() :
332* Refill `bitD` from buffer previously set in BIT_initDStream() .
333* This function is safe, it guarantees it will not read beyond src buffer.
334* @return : status of `BIT_DStream_t` internal register.
335 if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
336ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD)
337{
338 if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* should not happen => corruption detected */
339 return BIT_DStream_overflow;
340
341 if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
342 bitD->ptr -= bitD->bitsConsumed >> 3;
343 bitD->bitsConsumed &= 7;
344 bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
345 return BIT_DStream_unfinished;
346 }
347 if (bitD->ptr == bitD->start) {
348 if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8)
349 return BIT_DStream_endOfBuffer;
350 return BIT_DStream_completed;
351 }
352 {
353 U32 nbBytes = bitD->bitsConsumed >> 3;
354 BIT_DStream_status result = BIT_DStream_unfinished;
355 if (bitD->ptr - nbBytes < bitD->start) {
356 nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
357 result = BIT_DStream_endOfBuffer;
358 }
359 bitD->ptr -= nbBytes;
360 bitD->bitsConsumed -= nbBytes * 8;
361 bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
362 return result;
363 }
364}
365
366/*! BIT_endOfDStream() :
367* @return Tells if DStream has exactly reached its end (all bits consumed).
368*/
369ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *DStream)
370{
371 return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8));
372}
373
374#endif /* BITSTREAM_H_MODULE */
diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c
new file mode 100644
index 000000000000..f9166cf4f7a9
--- /dev/null
+++ b/lib/zstd/compress.c
@@ -0,0 +1,3484 @@
1/**
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of https://github.com/facebook/zstd.
7 * An additional grant of patent rights can be found in the PATENTS file in the
8 * same directory.
9 *
10 * This program is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU General Public License version 2 as published by the
12 * Free Software Foundation. This program is dual-licensed; you may select
13 * either version 2 of the GNU General Public License ("GPL") or BSD license
14 * ("BSD").
15 */
16
17/*-*************************************
18* Dependencies
19***************************************/
20#include "fse.h"
21#include "huf.h"
22#include "mem.h"
23#include "zstd_internal.h" /* includes zstd.h */
24#include <linux/kernel.h>
25#include <linux/module.h>
26#include <linux/string.h> /* memset */
27
28/*-*************************************
29* Constants
30***************************************/
31static const U32 g_searchStrength = 8; /* control skip over incompressible data */
32#define HASH_READ_SIZE 8
33typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
34
35/*-*************************************
36* Helper functions
37***************************************/
38size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
39
40/*-*************************************
41* Sequence storage
42***************************************/
43static void ZSTD_resetSeqStore(seqStore_t *ssPtr)
44{
45 ssPtr->lit = ssPtr->litStart;
46 ssPtr->sequences = ssPtr->sequencesStart;
47 ssPtr->longLengthID = 0;
48}
49
50/*-*************************************
51* Context memory management
52***************************************/
53struct ZSTD_CCtx_s {
54 const BYTE *nextSrc; /* next block here to continue on curr prefix */
55 const BYTE *base; /* All regular indexes relative to this position */
56 const BYTE *dictBase; /* extDict indexes relative to this position */
57 U32 dictLimit; /* below that point, need extDict */
58 U32 lowLimit; /* below that point, no more data */
59 U32 nextToUpdate; /* index from which to continue dictionary update */
60 U32 nextToUpdate3; /* index from which to continue dictionary update */
61 U32 hashLog3; /* dispatch table : larger == faster, more memory */
62 U32 loadedDictEnd; /* index of end of dictionary */
63 U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
64 U32 forceRawDict; /* Force loading dictionary in "content-only" mode (no header analysis) */
65 ZSTD_compressionStage_e stage;
66 U32 rep[ZSTD_REP_NUM];
67 U32 repToConfirm[ZSTD_REP_NUM];
68 U32 dictID;
69 ZSTD_parameters params;
70 void *workSpace;
71 size_t workSpaceSize;
72 size_t blockSize;
73 U64 frameContentSize;
74 struct xxh64_state xxhState;
75 ZSTD_customMem customMem;
76
77 seqStore_t seqStore; /* sequences storage ptrs */
78 U32 *hashTable;
79 U32 *hashTable3;
80 U32 *chainTable;
81 HUF_CElt *hufTable;
82 U32 flagStaticTables;
83 HUF_repeat flagStaticHufTable;
84 FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
85 FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
86 FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
87 unsigned tmpCounters[HUF_COMPRESS_WORKSPACE_SIZE_U32];
88};
89
90size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams)
91{
92 size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
93 U32 const divider = (cParams.searchLength == 3) ? 3 : 4;
94 size_t const maxNbSeq = blockSize / divider;
95 size_t const tokenSpace = blockSize + 11 * maxNbSeq;
96 size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog);
97 size_t const hSize = ((size_t)1) << cParams.hashLog;
98 U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
99 size_t const h3Size = ((size_t)1) << hashLog3;
100 size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
101 size_t const optSpace =
102 ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
103 size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
104 (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
105
106 return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize);
107}
108
109static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
110{
111 ZSTD_CCtx *cctx;
112 if (!customMem.customAlloc || !customMem.customFree)
113 return NULL;
114 cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
115 if (!cctx)
116 return NULL;
117 memset(cctx, 0, sizeof(ZSTD_CCtx));
118 cctx->customMem = customMem;
119 return cctx;
120}
121
122ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize)
123{
124 ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
125 ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(stackMem);
126 if (cctx) {
127 cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize);
128 }
129 return cctx;
130}
131
132size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx)
133{
134 if (cctx == NULL)
135 return 0; /* support free on NULL */
136 ZSTD_free(cctx->workSpace, cctx->customMem);
137 ZSTD_free(cctx, cctx->customMem);
138 return 0; /* reserved as a potential error code in the future */
139}
140
141const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); }
142
143static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; }
144
145/** ZSTD_checkParams() :
146 ensure param values remain within authorized range.
147 @return : 0, or an error code if one value is beyond authorized range */
148size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
149{
150#define CLAMPCHECK(val, min, max) \
151 { \
152 if ((val < min) | (val > max)) \
153 return ERROR(compressionParameter_unsupported); \
154 }
155 CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
156 CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
157 CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
158 CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
159 CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
160 CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
161 if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2)
162 return ERROR(compressionParameter_unsupported);
163 return 0;
164}
165
166/** ZSTD_cycleLog() :
167 * condition for correct operation : hashLog > 1 */
168static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
169{
170 U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
171 return hashLog - btScale;
172}
173
174/** ZSTD_adjustCParams() :
175 optimize `cPar` for a given input (`srcSize` and `dictSize`).
176 mostly downsizing to reduce memory consumption and initialization.
177 Both `srcSize` and `dictSize` are optional (use 0 if unknown),
178 but if both are 0, no optimization can be done.
179 Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
180ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
181{
182 if (srcSize + dictSize == 0)
183 return cPar; /* no size information available : no adjustment */
184
185 /* resize params, to use less memory when necessary */
186 {
187 U32 const minSrcSize = (srcSize == 0) ? 500 : 0;
188 U64 const rSize = srcSize + dictSize + minSrcSize;
189 if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) {
190 U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1);
191 if (cPar.windowLog > srcLog)
192 cPar.windowLog = srcLog;
193 }
194 }
195 if (cPar.hashLog > cPar.windowLog)
196 cPar.hashLog = cPar.windowLog;
197 {
198 U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
199 if (cycleLog > cPar.windowLog)
200 cPar.chainLog -= (cycleLog - cPar.windowLog);
201 }
202
203 if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
204 cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
205
206 return cPar;
207}
208
209static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
210{
211 return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) &
212 (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3));
213}
214
215/*! ZSTD_continueCCtx() :
216 reuse CCtx without reset (note : requires no dictionary) */
217static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_parameters params, U64 frameContentSize)
218{
219 U32 const end = (U32)(cctx->nextSrc - cctx->base);
220 cctx->params = params;
221 cctx->frameContentSize = frameContentSize;
222 cctx->lowLimit = end;
223 cctx->dictLimit = end;
224 cctx->nextToUpdate = end + 1;
225 cctx->stage = ZSTDcs_init;
226 cctx->dictID = 0;
227 cctx->loadedDictEnd = 0;
228 {
229 int i;
230 for (i = 0; i < ZSTD_REP_NUM; i++)
231 cctx->rep[i] = repStartValue[i];
232 }
233 cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */
234 xxh64_reset(&cctx->xxhState, 0);
235 return 0;
236}
237
238typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
239
240/*! ZSTD_resetCCtx_advanced() :
241 note : `params` must be validated */
242static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp)
243{
244 if (crp == ZSTDcrp_continue)
245 if (ZSTD_equivalentParams(params, zc->params)) {
246 zc->flagStaticTables = 0;
247 zc->flagStaticHufTable = HUF_repeat_none;
248 return ZSTD_continueCCtx(zc, params, frameContentSize);
249 }
250
251 {
252 size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
253 U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4;
254 size_t const maxNbSeq = blockSize / divider;
255 size_t const tokenSpace = blockSize + 11 * maxNbSeq;
256 size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
257 size_t const hSize = ((size_t)1) << params.cParams.hashLog;
258 U32 const hashLog3 = (params.cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
259 size_t const h3Size = ((size_t)1) << hashLog3;
260 size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
261 void *ptr;
262
263 /* Check if workSpace is large enough, alloc a new one if needed */
264 {
265 size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) +
266 (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
267 size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
268 (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
269 if (zc->workSpaceSize < neededSpace) {
270 ZSTD_free(zc->workSpace, zc->customMem);
271 zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
272 if (zc->workSpace == NULL)
273 return ERROR(memory_allocation);
274 zc->workSpaceSize = neededSpace;
275 }
276 }
277
278 if (crp != ZSTDcrp_noMemset)
279 memset(zc->workSpace, 0, tableSpace); /* reset tables only */
280 xxh64_reset(&zc->xxhState, 0);
281 zc->hashLog3 = hashLog3;
282 zc->hashTable = (U32 *)(zc->workSpace);
283 zc->chainTable = zc->hashTable + hSize;
284 zc->hashTable3 = zc->chainTable + chainSize;
285 ptr = zc->hashTable3 + h3Size;
286 zc->hufTable = (HUF_CElt *)ptr;
287 zc->flagStaticTables = 0;
288 zc->flagStaticHufTable = HUF_repeat_none;
289 ptr = ((U32 *)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
290
291 zc->nextToUpdate = 1;
292 zc->nextSrc = NULL;
293 zc->base = NULL;
294 zc->dictBase = NULL;
295 zc->dictLimit = 0;
296 zc->lowLimit = 0;
297 zc->params = params;
298 zc->blockSize = blockSize;
299 zc->frameContentSize = frameContentSize;
300 {
301 int i;
302 for (i = 0; i < ZSTD_REP_NUM; i++)
303 zc->rep[i] = repStartValue[i];
304 }
305
306 if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
307 zc->seqStore.litFreq = (U32 *)ptr;
308 zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1 << Litbits);
309 zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL + 1);
310 zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML + 1);
311 ptr = zc->seqStore.offCodeFreq + (MaxOff + 1);
312 zc->seqStore.matchTable = (ZSTD_match_t *)ptr;
313 ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM + 1;
314 zc->seqStore.priceTable = (ZSTD_optimal_t *)ptr;
315 ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM + 1;
316 zc->seqStore.litLengthSum = 0;
317 }
318 zc->seqStore.sequencesStart = (seqDef *)ptr;
319 ptr = zc->seqStore.sequencesStart + maxNbSeq;
320 zc->seqStore.llCode = (BYTE *)ptr;
321 zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
322 zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
323 zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
324
325 zc->stage = ZSTDcs_init;
326 zc->dictID = 0;
327 zc->loadedDictEnd = 0;
328
329 return 0;
330 }
331}
332
333/* ZSTD_invalidateRepCodes() :
334 * ensures next compression will not use repcodes from previous block.
335 * Note : only works with regular variant;
336 * do not use with extDict variant ! */
337void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx)
338{
339 int i;
340 for (i = 0; i < ZSTD_REP_NUM; i++)
341 cctx->rep[i] = 0;
342}
343
344/*! ZSTD_copyCCtx() :
345* Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
346* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
347* @return : 0, or an error code */
348size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize)
349{
350 if (srcCCtx->stage != ZSTDcs_init)
351 return ERROR(stage_wrong);
352
353 memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
354 {
355 ZSTD_parameters params = srcCCtx->params;
356 params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
357 ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
358 }
359
360 /* copy tables */
361 {
362 size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
363 size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
364 size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
365 size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
366 memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
367 }
368
369 /* copy dictionary offsets */
370 dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
371 dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3;
372 dstCCtx->nextSrc = srcCCtx->nextSrc;
373 dstCCtx->base = srcCCtx->base;
374 dstCCtx->dictBase = srcCCtx->dictBase;
375 dstCCtx->dictLimit = srcCCtx->dictLimit;
376 dstCCtx->lowLimit = srcCCtx->lowLimit;
377 dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd;
378 dstCCtx->dictID = srcCCtx->dictID;
379
380 /* copy entropy tables */
381 dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
382 dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
383 if (srcCCtx->flagStaticTables) {
384 memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
385 memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
386 memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
387 }
388 if (srcCCtx->flagStaticHufTable) {
389 memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256 * 4);
390 }
391
392 return 0;
393}
394
395/*! ZSTD_reduceTable() :
396* reduce table indexes by `reducerValue` */
397static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue)
398{
399 U32 u;
400 for (u = 0; u < size; u++) {
401 if (table[u] < reducerValue)
402 table[u] = 0;
403 else
404 table[u] -= reducerValue;
405 }
406}
407
408/*! ZSTD_reduceIndex() :
409* rescale all indexes to avoid future overflow (indexes are U32) */
410static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue)
411{
412 {
413 U32 const hSize = 1 << zc->params.cParams.hashLog;
414 ZSTD_reduceTable(zc->hashTable, hSize, reducerValue);
415 }
416
417 {
418 U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
419 ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue);
420 }
421
422 {
423 U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
424 ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue);
425 }
426}
427
428/*-*******************************************************
429* Block entropic compression
430*********************************************************/
431
432/* See doc/zstd_compression_format.md for detailed format description */
433
434size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
435{
436 if (srcSize + ZSTD_blockHeaderSize > dstCapacity)
437 return ERROR(dstSize_tooSmall);
438 memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize);
439 ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw);
440 return ZSTD_blockHeaderSize + srcSize;
441}
442
443static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
444{
445 BYTE *const ostart = (BYTE * const)dst;
446 U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
447
448 if (srcSize + flSize > dstCapacity)
449 return ERROR(dstSize_tooSmall);
450
451 switch (flSize) {
452 case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break;
453 case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break;
454 default: /*note : should not be necessary : flSize is within {1,2,3} */
455 case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break;
456 }
457
458 memcpy(ostart + flSize, src, srcSize);
459 return srcSize + flSize;
460}
461
462static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
463{
464 BYTE *const ostart = (BYTE * const)dst;
465 U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
466
467 (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
468
469 switch (flSize) {
470 case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break;
471 case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break;
472 default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */
473 case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break;
474 }
475
476 ostart[flSize] = *(const BYTE *)src;
477 return flSize + 1;
478}
479
480static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
481
482static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
483{
484 size_t const minGain = ZSTD_minGain(srcSize);
485 size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
486 BYTE *const ostart = (BYTE *)dst;
487 U32 singleStream = srcSize < 256;
488 symbolEncodingType_e hType = set_compressed;
489 size_t cLitSize;
490
491/* small ? don't even attempt compression (speed opt) */
492#define LITERAL_NOENTROPY 63
493 {
494 size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
495 if (srcSize <= minLitSize)
496 return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
497 }
498
499 if (dstCapacity < lhSize + 1)
500 return ERROR(dstSize_tooSmall); /* not enough space for compression */
501 {
502 HUF_repeat repeat = zc->flagStaticHufTable;
503 int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
504 if (repeat == HUF_repeat_valid && lhSize == 3)
505 singleStream = 1;
506 cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
507 sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
508 : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
509 sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
510 if (repeat != HUF_repeat_none) {
511 hType = set_repeat;
512 } /* reused the existing table */
513 else {
514 zc->flagStaticHufTable = HUF_repeat_check;
515 } /* now have a table to reuse */
516 }
517
518 if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) {
519 zc->flagStaticHufTable = HUF_repeat_none;
520 return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
521 }
522 if (cLitSize == 1) {
523 zc->flagStaticHufTable = HUF_repeat_none;
524 return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
525 }
526
527 /* Build header */
528 switch (lhSize) {
529 case 3: /* 2 - 2 - 10 - 10 */
530 {
531 U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14);
532 ZSTD_writeLE24(ostart, lhc);
533 break;
534 }
535 case 4: /* 2 - 2 - 14 - 14 */
536 {
537 U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18);
538 ZSTD_writeLE32(ostart, lhc);
539 break;
540 }
541 default: /* should not be necessary, lhSize is only {3,4,5} */
542 case 5: /* 2 - 2 - 18 - 18 */
543 {
544 U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22);
545 ZSTD_writeLE32(ostart, lhc);
546 ostart[4] = (BYTE)(cLitSize >> 10);
547 break;
548 }
549 }
550 return lhSize + cLitSize;
551}
552
553static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18,
554 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23,
555 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24};
556
557static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
558 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38,
559 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
560 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42,
561 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42};
562
563void ZSTD_seqToCodes(const seqStore_t *seqStorePtr)
564{
565 BYTE const LL_deltaCode = 19;
566 BYTE const ML_deltaCode = 36;
567 const seqDef *const sequences = seqStorePtr->sequencesStart;
568 BYTE *const llCodeTable = seqStorePtr->llCode;
569 BYTE *const ofCodeTable = seqStorePtr->ofCode;
570 BYTE *const mlCodeTable = seqStorePtr->mlCode;
571 U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
572 U32 u;
573 for (u = 0; u < nbSeq; u++) {
574 U32 const llv = sequences[u].litLength;
575 U32 const mlv = sequences[u].matchLength;
576 llCodeTable[u] = (llv > 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv];
577 ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset);
578 mlCodeTable[u] = (mlv > 127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv];
579 }
580 if (seqStorePtr->longLengthID == 1)
581 llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
582 if (seqStorePtr->longLengthID == 2)
583 mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
584}
585
586ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity)
587{
588 const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
589 const seqStore_t *seqStorePtr = &(zc->seqStore);
590 FSE_CTable *CTable_LitLength = zc->litlengthCTable;
591 FSE_CTable *CTable_OffsetBits = zc->offcodeCTable;
592 FSE_CTable *CTable_MatchLength = zc->matchlengthCTable;
593 U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
594 const seqDef *const sequences = seqStorePtr->sequencesStart;
595 const BYTE *const ofCodeTable = seqStorePtr->ofCode;
596 const BYTE *const llCodeTable = seqStorePtr->llCode;
597 const BYTE *const mlCodeTable = seqStorePtr->mlCode;
598 BYTE *const ostart = (BYTE *)dst;
599 BYTE *const oend = ostart + dstCapacity;
600 BYTE *op = ostart;
601 size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
602 BYTE *seqHead;
603
604 U32 *count;
605 S16 *norm;
606 U32 *workspace;
607 size_t workspaceSize = sizeof(zc->tmpCounters);
608 {
609 size_t spaceUsed32 = 0;
610 count = (U32 *)zc->tmpCounters + spaceUsed32;
611 spaceUsed32 += MaxSeq + 1;
612 norm = (S16 *)((U32 *)zc->tmpCounters + spaceUsed32);
613 spaceUsed32 += ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2;
614
615 workspace = (U32 *)zc->tmpCounters + spaceUsed32;
616 workspaceSize -= (spaceUsed32 << 2);
617 }
618
619 /* Compress literals */
620 {
621 const BYTE *const literals = seqStorePtr->litStart;
622 size_t const litSize = seqStorePtr->lit - literals;
623 size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize);
624 if (ZSTD_isError(cSize))
625 return cSize;
626 op += cSize;
627 }
628
629 /* Sequences Header */
630 if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */)
631 return ERROR(dstSize_tooSmall);
632 if (nbSeq < 0x7F)
633 *op++ = (BYTE)nbSeq;
634 else if (nbSeq < LONGNBSEQ)
635 op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2;
636 else
637 op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3;
638 if (nbSeq == 0)
639 return op - ostart;
640
641 /* seqHead : flags for FSE encoding type */
642 seqHead = op++;
643
644#define MIN_SEQ_FOR_DYNAMIC_FSE 64
645#define MAX_SEQ_FOR_STATIC_FSE 1000
646
647 /* convert length/distances into codes */
648 ZSTD_seqToCodes(seqStorePtr);
649
650 /* CTable for Literal Lengths */
651 {
652 U32 max = MaxLL;
653 size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace);
654 if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
655 *op++ = llCodeTable[0];
656 FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
657 LLtype = set_rle;
658 } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
659 LLtype = set_repeat;
660 } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog - 1)))) {
661 FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, workspace, workspaceSize);
662 LLtype = set_basic;
663 } else {
664 size_t nbSeq_1 = nbSeq;
665 const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
666 if (count[llCodeTable[nbSeq - 1]] > 1) {
667 count[llCodeTable[nbSeq - 1]]--;
668 nbSeq_1--;
669 }
670 FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
671 {
672 size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
673 if (FSE_isError(NCountSize))
674 return NCountSize;
675 op += NCountSize;
676 }
677 FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, workspace, workspaceSize);
678 LLtype = set_compressed;
679 }
680 }
681
682 /* CTable for Offsets */
683 {
684 U32 max = MaxOff;
685 size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace);
686 if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
687 *op++ = ofCodeTable[0];
688 FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
689 Offtype = set_rle;
690 } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
691 Offtype = set_repeat;
692 } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog - 1)))) {
693 FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, workspace, workspaceSize);
694 Offtype = set_basic;
695 } else {
696 size_t nbSeq_1 = nbSeq;
697 const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
698 if (count[ofCodeTable[nbSeq - 1]] > 1) {
699 count[ofCodeTable[nbSeq - 1]]--;
700 nbSeq_1--;
701 }
702 FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
703 {
704 size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
705 if (FSE_isError(NCountSize))
706 return NCountSize;
707 op += NCountSize;
708 }
709 FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, workspace, workspaceSize);
710 Offtype = set_compressed;
711 }
712 }
713
714 /* CTable for MatchLengths */
715 {
716 U32 max = MaxML;
717 size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace);
718 if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
719 *op++ = *mlCodeTable;
720 FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
721 MLtype = set_rle;
722 } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
723 MLtype = set_repeat;
724 } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog - 1)))) {
725 FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, workspace, workspaceSize);
726 MLtype = set_basic;
727 } else {
728 size_t nbSeq_1 = nbSeq;
729 const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
730 if (count[mlCodeTable[nbSeq - 1]] > 1) {
731 count[mlCodeTable[nbSeq - 1]]--;
732 nbSeq_1--;
733 }
734 FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
735 {
736 size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
737 if (FSE_isError(NCountSize))
738 return NCountSize;
739 op += NCountSize;
740 }
741 FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, workspace, workspaceSize);
742 MLtype = set_compressed;
743 }
744 }
745
746 *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2));
747 zc->flagStaticTables = 0;
748
749 /* Encoding Sequences */
750 {
751 BIT_CStream_t blockStream;
752 FSE_CState_t stateMatchLength;
753 FSE_CState_t stateOffsetBits;
754 FSE_CState_t stateLitLength;
755
756 CHECK_E(BIT_initCStream(&blockStream, op, oend - op), dstSize_tooSmall); /* not enough space remaining */
757
758 /* first symbols */
759 FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]);
760 FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]);
761 FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]);
762 BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]);
763 if (ZSTD_32bits())
764 BIT_flushBits(&blockStream);
765 BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]);
766 if (ZSTD_32bits())
767 BIT_flushBits(&blockStream);
768 if (longOffsets) {
769 U32 const ofBits = ofCodeTable[nbSeq - 1];
770 int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
771 if (extraBits) {
772 BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits);
773 BIT_flushBits(&blockStream);
774 }
775 BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits);
776 } else {
777 BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]);
778 }
779 BIT_flushBits(&blockStream);
780
781 {
782 size_t n;
783 for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */
784 BYTE const llCode = llCodeTable[n];
785 BYTE const ofCode = ofCodeTable[n];
786 BYTE const mlCode = mlCodeTable[n];
787 U32 const llBits = LL_bits[llCode];
788 U32 const ofBits = ofCode; /* 32b*/ /* 64b*/
789 U32 const mlBits = ML_bits[mlCode];
790 /* (7)*/ /* (7)*/
791 FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
792 FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
793 if (ZSTD_32bits())
794 BIT_flushBits(&blockStream); /* (7)*/
795 FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
796 if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
797 BIT_flushBits(&blockStream); /* (7)*/
798 BIT_addBits(&blockStream, sequences[n].litLength, llBits);
799 if (ZSTD_32bits() && ((llBits + mlBits) > 24))
800 BIT_flushBits(&blockStream);
801 BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
802 if (ZSTD_32bits())
803 BIT_flushBits(&blockStream); /* (7)*/
804 if (longOffsets) {
805 int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
806 if (extraBits) {
807 BIT_addBits(&blockStream, sequences[n].offset, extraBits);
808 BIT_flushBits(&blockStream); /* (7)*/
809 }
810 BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */
811 } else {
812 BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
813 }
814 BIT_flushBits(&blockStream); /* (7)*/
815 }
816 }
817
818 FSE_flushCState(&blockStream, &stateMatchLength);
819 FSE_flushCState(&blockStream, &stateOffsetBits);
820 FSE_flushCState(&blockStream, &stateLitLength);
821
822 {
823 size_t const streamSize = BIT_closeCStream(&blockStream);
824 if (streamSize == 0)
825 return ERROR(dstSize_tooSmall); /* not enough space */
826 op += streamSize;
827 }
828 }
829 return op - ostart;
830}
831
832ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize)
833{
834 size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity);
835 size_t const minGain = ZSTD_minGain(srcSize);
836 size_t const maxCSize = srcSize - minGain;
837 /* If the srcSize <= dstCapacity, then there is enough space to write a
838 * raw uncompressed block. Since we ran out of space, the block must not
839 * be compressible, so fall back to a raw uncompressed block.
840 */
841 int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
842 int i;
843
844 if (ZSTD_isError(cSize) && !uncompressibleError)
845 return cSize;
846 if (cSize >= maxCSize || uncompressibleError) {
847 zc->flagStaticHufTable = HUF_repeat_none;
848 return 0;
849 }
850 /* confirm repcodes */
851 for (i = 0; i < ZSTD_REP_NUM; i++)
852 zc->rep[i] = zc->repToConfirm[i];
853 return cSize;
854}
855
856/*! ZSTD_storeSeq() :
857 Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
858 `offsetCode` : distance to match, or 0 == repCode.
859 `matchCode` : matchLength - MINMATCH
860*/
861ZSTD_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t matchCode)
862{
863 /* copy Literals */
864 ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
865 seqStorePtr->lit += litLength;
866
867 /* literal Length */
868 if (litLength > 0xFFFF) {
869 seqStorePtr->longLengthID = 1;
870 seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
871 }
872 seqStorePtr->sequences[0].litLength = (U16)litLength;
873
874 /* match offset */
875 seqStorePtr->sequences[0].offset = offsetCode + 1;
876
877 /* match Length */
878 if (matchCode > 0xFFFF) {
879 seqStorePtr->longLengthID = 2;
880 seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
881 }
882 seqStorePtr->sequences[0].matchLength = (U16)matchCode;
883
884 seqStorePtr->sequences++;
885}
886
887/*-*************************************
888* Match length counter
889***************************************/
890static unsigned ZSTD_NbCommonBytes(register size_t val)
891{
892 if (ZSTD_isLittleEndian()) {
893 if (ZSTD_64bits()) {
894 return (__builtin_ctzll((U64)val) >> 3);
895 } else { /* 32 bits */
896 return (__builtin_ctz((U32)val) >> 3);
897 }
898 } else { /* Big Endian CPU */
899 if (ZSTD_64bits()) {
900 return (__builtin_clzll(val) >> 3);
901 } else { /* 32 bits */
902 return (__builtin_clz((U32)val) >> 3);
903 }
904 }
905}
906
907static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit)
908{
909 const BYTE *const pStart = pIn;
910 const BYTE *const pInLoopLimit = pInLimit - (sizeof(size_t) - 1);
911
912 while (pIn < pInLoopLimit) {
913 size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn);
914 if (!diff) {
915 pIn += sizeof(size_t);
916 pMatch += sizeof(size_t);
917 continue;
918 }
919 pIn += ZSTD_NbCommonBytes(diff);
920 return (size_t)(pIn - pStart);
921 }
922 if (ZSTD_64bits())
923 if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) {
924 pIn += 4;
925 pMatch += 4;
926 }
927 if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) {
928 pIn += 2;
929 pMatch += 2;
930 }
931 if ((pIn < pInLimit) && (*pMatch == *pIn))
932 pIn++;
933 return (size_t)(pIn - pStart);
934}
935
936/** ZSTD_count_2segments() :
937* can count match length with `ip` & `match` in 2 different segments.
938* convention : on reaching mEnd, match count continue starting from iStart
939*/
940static size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart)
941{
942 const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd);
943 size_t const matchLength = ZSTD_count(ip, match, vEnd);
944 if (match + matchLength != mEnd)
945 return matchLength;
946 return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd);
947}
948
949/*-*************************************
950* Hashes
951***************************************/
952static const U32 prime3bytes = 506832829U;
953static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32 - 24)) * prime3bytes) >> (32 - h); }
954ZSTD_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */
955
956static const U32 prime4bytes = 2654435761U;
957static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32 - h); }
958static size_t ZSTD_hash4Ptr(const void *ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); }
959
960static const U64 prime5bytes = 889523592379ULL;
961static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); }
962static size_t ZSTD_hash5Ptr(const void *p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); }
963
964static const U64 prime6bytes = 227718039650203ULL;
965static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); }
966static size_t ZSTD_hash6Ptr(const void *p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); }
967
968static const U64 prime7bytes = 58295818150454627ULL;
969static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); }
970static size_t ZSTD_hash7Ptr(const void *p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); }
971
972static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
973static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u)*prime8bytes) >> (64 - h)); }
974static size_t ZSTD_hash8Ptr(const void *p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); }
975
976static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls)
977{
978 switch (mls) {
979 // case 3: return ZSTD_hash3Ptr(p, hBits);
980 default:
981 case 4: return ZSTD_hash4Ptr(p, hBits);
982 case 5: return ZSTD_hash5Ptr(p, hBits);
983 case 6: return ZSTD_hash6Ptr(p, hBits);
984 case 7: return ZSTD_hash7Ptr(p, hBits);
985 case 8: return ZSTD_hash8Ptr(p, hBits);
986 }
987}
988
989/*-*************************************
990* Fast Scan
991***************************************/
992static void ZSTD_fillHashTable(ZSTD_CCtx *zc, const void *end, const U32 mls)
993{
994 U32 *const hashTable = zc->hashTable;
995 U32 const hBits = zc->params.cParams.hashLog;
996 const BYTE *const base = zc->base;
997 const BYTE *ip = base + zc->nextToUpdate;
998 const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
999 const size_t fastHashFillStep = 3;
1000
1001 while (ip <= iend) {
1002 hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
1003 ip += fastHashFillStep;
1004 }
1005}
1006
1007FORCE_INLINE
1008void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
1009{
1010 U32 *const hashTable = cctx->hashTable;
1011 U32 const hBits = cctx->params.cParams.hashLog;
1012 seqStore_t *seqStorePtr = &(cctx->seqStore);
1013 const BYTE *const base = cctx->base;
1014 const BYTE *const istart = (const BYTE *)src;
1015 const BYTE *ip = istart;
1016 const BYTE *anchor = istart;
1017 const U32 lowestIndex = cctx->dictLimit;
1018 const BYTE *const lowest = base + lowestIndex;
1019 const BYTE *const iend = istart + srcSize;
1020 const BYTE *const ilimit = iend - HASH_READ_SIZE;
1021 U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
1022 U32 offsetSaved = 0;
1023
1024 /* init */
1025 ip += (ip == lowest);
1026 {
1027 U32 const maxRep = (U32)(ip - lowest);
1028 if (offset_2 > maxRep)
1029 offsetSaved = offset_2, offset_2 = 0;
1030 if (offset_1 > maxRep)
1031 offsetSaved = offset_1, offset_1 = 0;
1032 }
1033
1034 /* Main Search Loop */
1035 while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
1036 size_t mLength;
1037 size_t const h = ZSTD_hashPtr(ip, hBits, mls);
1038 U32 const curr = (U32)(ip - base);
1039 U32 const matchIndex = hashTable[h];
1040 const BYTE *match = base + matchIndex;
1041 hashTable[h] = curr; /* update hash table */
1042
1043 if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) {
1044 mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
1045 ip++;
1046 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
1047 } else {
1048 U32 offset;
1049 if ((matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
1050 ip += ((ip - anchor) >> g_searchStrength) + 1;
1051 continue;
1052 }
1053 mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
1054 offset = (U32)(ip - match);
1055 while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
1056 ip--;
1057 match--;
1058 mLength++;
1059 } /* catch up */
1060 offset_2 = offset_1;
1061 offset_1 = offset;
1062
1063 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
1064 }
1065
1066 /* match found */
1067 ip += mLength;
1068 anchor = ip;
1069
1070 if (ip <= ilimit) {
1071 /* Fill Table */
1072 hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */
1073 hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
1074 /* check immediate repcode */
1075 while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
1076 /* store sequence */
1077 size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
1078 {
1079 U32 const tmpOff = offset_2;
1080 offset_2 = offset_1;
1081 offset_1 = tmpOff;
1082 } /* swap offset_2 <=> offset_1 */
1083 hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
1084 ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
1085 ip += rLength;
1086 anchor = ip;
1087 continue; /* faster when present ... (?) */
1088 }
1089 }
1090 }
1091
1092 /* save reps for next block */
1093 cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
1094 cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
1095
1096 /* Last Literals */
1097 {
1098 size_t const lastLLSize = iend - anchor;
1099 memcpy(seqStorePtr->lit, anchor, lastLLSize);
1100 seqStorePtr->lit += lastLLSize;
1101 }
1102}
1103
1104static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
1105{
1106 const U32 mls = ctx->params.cParams.searchLength;
1107 switch (mls) {
1108 default: /* includes case 3 */
1109 case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return;
1110 case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return;
1111 case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return;
1112 case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return;
1113 }
1114}
1115
1116static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
1117{
1118 U32 *hashTable = ctx->hashTable;
1119 const U32 hBits = ctx->params.cParams.hashLog;
1120 seqStore_t *seqStorePtr = &(ctx->seqStore);
1121 const BYTE *const base = ctx->base;
1122 const BYTE *const dictBase = ctx->dictBase;
1123 const BYTE *const istart = (const BYTE *)src;
1124 const BYTE *ip = istart;
1125 const BYTE *anchor = istart;
1126 const U32 lowestIndex = ctx->lowLimit;
1127 const BYTE *const dictStart = dictBase + lowestIndex;
1128 const U32 dictLimit = ctx->dictLimit;
1129 const BYTE *const lowPrefixPtr = base + dictLimit;
1130 const BYTE *const dictEnd = dictBase + dictLimit;
1131 const BYTE *const iend = istart + srcSize;
1132 const BYTE *const ilimit = iend - 8;
1133 U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
1134
1135 /* Search Loop */
1136 while (ip < ilimit) { /* < instead of <=, because (ip+1) */
1137 const size_t h = ZSTD_hashPtr(ip, hBits, mls);
1138 const U32 matchIndex = hashTable[h];
1139 const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
1140 const BYTE *match = matchBase + matchIndex;
1141 const U32 curr = (U32)(ip - base);
1142 const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
1143 const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
1144 const BYTE *repMatch = repBase + repIndex;
1145 size_t mLength;
1146 hashTable[h] = curr; /* update hash table */
1147
1148 if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
1149 (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
1150 const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
1151 mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
1152 ip++;
1153 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
1154 } else {
1155 if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
1156 ip += ((ip - anchor) >> g_searchStrength) + 1;
1157 continue;
1158 }
1159 {
1160 const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
1161 const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
1162 U32 offset;
1163 mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
1164 while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
1165 ip--;
1166 match--;
1167 mLength++;
1168 } /* catch up */
1169 offset = curr - matchIndex;
1170 offset_2 = offset_1;
1171 offset_1 = offset;
1172 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
1173 }
1174 }
1175
1176 /* found a match : store it */
1177 ip += mLength;
1178 anchor = ip;
1179
1180 if (ip <= ilimit) {
1181 /* Fill Table */
1182 hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2;
1183 hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
1184 /* check immediate repcode */
1185 while (ip <= ilimit) {
1186 U32 const curr2 = (U32)(ip - base);
1187 U32 const repIndex2 = curr2 - offset_2;
1188 const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
1189 if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
1190 && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
1191 const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
1192 size_t repLength2 =
1193 ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
1194 U32 tmpOffset = offset_2;
1195 offset_2 = offset_1;
1196 offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
1197 ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
1198 hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2;
1199 ip += repLength2;
1200 anchor = ip;
1201 continue;
1202 }
1203 break;
1204 }
1205 }
1206 }
1207
1208 /* save reps for next block */
1209 ctx->repToConfirm[0] = offset_1;
1210 ctx->repToConfirm[1] = offset_2;
1211
1212 /* Last Literals */
1213 {
1214 size_t const lastLLSize = iend - anchor;
1215 memcpy(seqStorePtr->lit, anchor, lastLLSize);
1216 seqStorePtr->lit += lastLLSize;
1217 }
1218}
1219
1220static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
1221{
1222 U32 const mls = ctx->params.cParams.searchLength;
1223 switch (mls) {
1224 default: /* includes case 3 */
1225 case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return;
1226 case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return;
1227 case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return;
1228 case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return;
1229 }
1230}
1231
1232/*-*************************************
1233* Double Fast
1234***************************************/
1235static void ZSTD_fillDoubleHashTable(ZSTD_CCtx *cctx, const void *end, const U32 mls)
1236{
1237 U32 *const hashLarge = cctx->hashTable;
1238 U32 const hBitsL = cctx->params.cParams.hashLog;
1239 U32 *const hashSmall = cctx->chainTable;
1240 U32 const hBitsS = cctx->params.cParams.chainLog;
1241 const BYTE *const base = cctx->base;
1242 const BYTE *ip = base + cctx->nextToUpdate;
1243 const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
1244 const size_t fastHashFillStep = 3;
1245
1246 while (ip <= iend) {
1247 hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
1248 hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
1249 ip += fastHashFillStep;
1250 }
1251}
1252
1253FORCE_INLINE
1254void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
1255{
1256 U32 *const hashLong = cctx->hashTable;
1257 const U32 hBitsL = cctx->params.cParams.hashLog;
1258 U32 *const hashSmall = cctx->chainTable;
1259 const U32 hBitsS = cctx->params.cParams.chainLog;
1260 seqStore_t *seqStorePtr = &(cctx->seqStore);
1261 const BYTE *const base = cctx->base;
1262 const BYTE *const istart = (const BYTE *)src;
1263 const BYTE *ip = istart;
1264 const BYTE *anchor = istart;
1265 const U32 lowestIndex = cctx->dictLimit;
1266 const BYTE *const lowest = base + lowestIndex;
1267 const BYTE *const iend = istart + srcSize;
1268 const BYTE *const ilimit = iend - HASH_READ_SIZE;
1269 U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
1270 U32 offsetSaved = 0;
1271
1272 /* init */
1273 ip += (ip == lowest);
1274 {
1275 U32 const maxRep = (U32)(ip - lowest);
1276 if (offset_2 > maxRep)
1277 offsetSaved = offset_2, offset_2 = 0;
1278 if (offset_1 > maxRep)
1279 offsetSaved = offset_1, offset_1 = 0;
1280 }
1281
1282 /* Main Search Loop */
1283 while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
1284 size_t mLength;
1285 size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
1286 size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
1287 U32 const curr = (U32)(ip - base);
1288 U32 const matchIndexL = hashLong[h2];
1289 U32 const matchIndexS = hashSmall[h];
1290 const BYTE *matchLong = base + matchIndexL;
1291 const BYTE *match = base + matchIndexS;
1292 hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
1293
1294 if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { /* note : by construction, offset_1 <= curr */
1295 mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
1296 ip++;
1297 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
1298 } else {
1299 U32 offset;
1300 if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
1301 mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8;
1302 offset = (U32)(ip - matchLong);
1303 while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) {
1304 ip--;
1305 matchLong--;
1306 mLength++;
1307 } /* catch up */
1308 } else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
1309 size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
1310 U32 const matchIndex3 = hashLong[h3];
1311 const BYTE *match3 = base + matchIndex3;
1312 hashLong[h3] = curr + 1;
1313 if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
1314 mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8;
1315 ip++;
1316 offset = (U32)(ip - match3);
1317 while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) {
1318 ip--;
1319 match3--;
1320 mLength++;
1321 } /* catch up */
1322 } else {
1323 mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
1324 offset = (U32)(ip - match);
1325 while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
1326 ip--;
1327 match--;
1328 mLength++;
1329 } /* catch up */
1330 }
1331 } else {
1332 ip += ((ip - anchor) >> g_searchStrength) + 1;
1333 continue;
1334 }
1335
1336 offset_2 = offset_1;
1337 offset_1 = offset;
1338
1339 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
1340 }
1341
1342 /* match found */
1343 ip += mLength;
1344 anchor = ip;
1345
1346 if (ip <= ilimit) {
1347 /* Fill Table */
1348 hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] =
1349 curr + 2; /* here because curr+2 could be > iend-8 */
1350 hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
1351
1352 /* check immediate repcode */
1353 while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
1354 /* store sequence */
1355 size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
1356 {
1357 U32 const tmpOff = offset_2;
1358 offset_2 = offset_1;
1359 offset_1 = tmpOff;
1360 } /* swap offset_2 <=> offset_1 */
1361 hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
1362 hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
1363 ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
1364 ip += rLength;
1365 anchor = ip;
1366 continue; /* faster when present ... (?) */
1367 }
1368 }
1369 }
1370
1371 /* save reps for next block */
1372 cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
1373 cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
1374
1375 /* Last Literals */
1376 {
1377 size_t const lastLLSize = iend - anchor;
1378 memcpy(seqStorePtr->lit, anchor, lastLLSize);
1379 seqStorePtr->lit += lastLLSize;
1380 }
1381}
1382
1383static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
1384{
1385 const U32 mls = ctx->params.cParams.searchLength;
1386 switch (mls) {
1387 default: /* includes case 3 */
1388 case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return;
1389 case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return;
1390 case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return;
1391 case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return;
1392 }
1393}
1394
1395static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
1396{
1397 U32 *const hashLong = ctx->hashTable;
1398 U32 const hBitsL = ctx->params.cParams.hashLog;
1399 U32 *const hashSmall = ctx->chainTable;
1400 U32 const hBitsS = ctx->params.cParams.chainLog;
1401 seqStore_t *seqStorePtr = &(ctx->seqStore);
1402 const BYTE *const base = ctx->base;
1403 const BYTE *const dictBase = ctx->dictBase;
1404 const BYTE *const istart = (const BYTE *)src;
1405 const BYTE *ip = istart;
1406 const BYTE *anchor = istart;
1407 const U32 lowestIndex = ctx->lowLimit;
1408 const BYTE *const dictStart = dictBase + lowestIndex;
1409 const U32 dictLimit = ctx->dictLimit;
1410 const BYTE *const lowPrefixPtr = base + dictLimit;
1411 const BYTE *const dictEnd = dictBase + dictLimit;
1412 const BYTE *const iend = istart + srcSize;
1413 const BYTE *const ilimit = iend - 8;
1414 U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
1415
1416 /* Search Loop */
1417 while (ip < ilimit) { /* < instead of <=, because (ip+1) */
1418 const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
1419 const U32 matchIndex = hashSmall[hSmall];
1420 const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
1421 const BYTE *match = matchBase + matchIndex;
1422
1423 const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
1424 const U32 matchLongIndex = hashLong[hLong];
1425 const BYTE *matchLongBase = matchLongIndex < dictLimit ? dictBase : base;
1426 const BYTE *matchLong = matchLongBase + matchLongIndex;
1427
1428 const U32 curr = (U32)(ip - base);
1429 const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
1430 const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
1431 const BYTE *repMatch = repBase + repIndex;
1432 size_t mLength;
1433 hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
1434
1435 if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
1436 (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
1437 const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
1438 mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4;
1439 ip++;
1440 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
1441 } else {
1442 if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
1443 const BYTE *matchEnd = matchLongIndex < dictLimit ? dictEnd : iend;
1444 const BYTE *lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr;
1445 U32 offset;
1446 mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8;
1447 offset = curr - matchLongIndex;
1448 while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) {
1449 ip--;
1450 matchLong--;
1451 mLength++;
1452 } /* catch up */
1453 offset_2 = offset_1;
1454 offset_1 = offset;
1455 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
1456
1457 } else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
1458 size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
1459 U32 const matchIndex3 = hashLong[h3];
1460 const BYTE *const match3Base = matchIndex3 < dictLimit ? dictBase : base;
1461 const BYTE *match3 = match3Base + matchIndex3;
1462 U32 offset;
1463 hashLong[h3] = curr + 1;
1464 if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
1465 const BYTE *matchEnd = matchIndex3 < dictLimit ? dictEnd : iend;
1466 const BYTE *lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr;
1467 mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8;
1468 ip++;
1469 offset = curr + 1 - matchIndex3;
1470 while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) {
1471 ip--;
1472 match3--;
1473 mLength++;
1474 } /* catch up */
1475 } else {
1476 const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
1477 const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
1478 mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4;
1479 offset = curr - matchIndex;
1480 while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
1481 ip--;
1482 match--;
1483 mLength++;
1484 } /* catch up */
1485 }
1486 offset_2 = offset_1;
1487 offset_1 = offset;
1488 ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
1489
1490 } else {
1491 ip += ((ip - anchor) >> g_searchStrength) + 1;
1492 continue;
1493 }
1494 }
1495
1496 /* found a match : store it */
1497 ip += mLength;
1498 anchor = ip;
1499
1500 if (ip <= ilimit) {
1501 /* Fill Table */
1502 hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = curr + 2;
1503 hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = curr + 2;
1504 hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
1505 hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base);
1506 /* check immediate repcode */
1507 while (ip <= ilimit) {
1508 U32 const curr2 = (U32)(ip - base);
1509 U32 const repIndex2 = curr2 - offset_2;
1510 const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
1511 if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
1512 && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
1513 const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
1514 size_t const repLength2 =
1515 ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
1516 U32 tmpOffset = offset_2;
1517 offset_2 = offset_1;
1518 offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
1519 ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
1520 hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2;
1521 hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2;
1522 ip += repLength2;
1523 anchor = ip;
1524 continue;
1525 }
1526 break;
1527 }
1528 }
1529 }
1530
1531 /* save reps for next block */
1532 ctx->repToConfirm[0] = offset_1;
1533 ctx->repToConfirm[1] = offset_2;
1534
1535 /* Last Literals */
1536 {
1537 size_t const lastLLSize = iend - anchor;
1538 memcpy(seqStorePtr->lit, anchor, lastLLSize);
1539 seqStorePtr->lit += lastLLSize;
1540 }
1541}
1542
1543static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
1544{
1545 U32 const mls = ctx->params.cParams.searchLength;
1546 switch (mls) {
1547 default: /* includes case 3 */
1548 case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return;
1549 case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return;
1550 case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return;
1551 case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return;
1552 }
1553}
1554
1555/*-*************************************
1556* Binary Tree search
1557***************************************/
1558/** ZSTD_insertBt1() : add one or multiple positions to tree.
1559* ip : assumed <= iend-8 .
1560* @return : nb of positions added */
1561static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict)
1562{
1563 U32 *const hashTable = zc->hashTable;
1564 U32 const hashLog = zc->params.cParams.hashLog;
1565 size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
1566 U32 *const bt = zc->chainTable;
1567 U32 const btLog = zc->params.cParams.chainLog - 1;
1568 U32 const btMask = (1 << btLog) - 1;
1569 U32 matchIndex = hashTable[h];
1570 size_t commonLengthSmaller = 0, commonLengthLarger = 0;
1571 const BYTE *const base = zc->base;
1572 const BYTE *const dictBase = zc->dictBase;
1573 const U32 dictLimit = zc->dictLimit;
1574 const BYTE *const dictEnd = dictBase + dictLimit;
1575 const BYTE *const prefixStart = base + dictLimit;
1576 const BYTE *match;
1577 const U32 curr = (U32)(ip - base);
1578 const U32 btLow = btMask >= curr ? 0 : curr - btMask;
1579 U32 *smallerPtr = bt + 2 * (curr & btMask);
1580 U32 *largerPtr = smallerPtr + 1;
1581 U32 dummy32; /* to be nullified at the end */
1582 U32 const windowLow = zc->lowLimit;
1583 U32 matchEndIdx = curr + 8;
1584 size_t bestLength = 8;
1585
1586 hashTable[h] = curr; /* Update Hash Table */
1587
1588 while (nbCompares-- && (matchIndex > windowLow)) {
1589 U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
1590 size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
1591
1592 if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
1593 match = base + matchIndex;
1594 if (match[matchLength] == ip[matchLength])
1595 matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
1596 } else {
1597 match = dictBase + matchIndex;
1598 matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
1599 if (matchIndex + matchLength >= dictLimit)
1600 match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
1601 }
1602
1603 if (matchLength > bestLength) {
1604 bestLength = matchLength;
1605 if (matchLength > matchEndIdx - matchIndex)
1606 matchEndIdx = matchIndex + (U32)matchLength;
1607 }
1608
1609 if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
1610 break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
1611
1612 if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */
1613 /* match is smaller than curr */
1614 *smallerPtr = matchIndex; /* update smaller idx */
1615 commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
1616 if (matchIndex <= btLow) {
1617 smallerPtr = &dummy32;
1618 break;
1619 } /* beyond tree size, stop the search */
1620 smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
1621 matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
1622 } else {
1623 /* match is larger than curr */
1624 *largerPtr = matchIndex;
1625 commonLengthLarger = matchLength;
1626 if (matchIndex <= btLow) {
1627 largerPtr = &dummy32;
1628 break;
1629 } /* beyond tree size, stop the search */
1630 largerPtr = nextPtr;
1631 matchIndex = nextPtr[0];
1632 }
1633 }
1634
1635 *smallerPtr = *largerPtr = 0;
1636 if (bestLength > 384)
1637 return MIN(192, (U32)(bestLength - 384)); /* speed optimization */
1638 if (matchEndIdx > curr + 8)
1639 return matchEndIdx - curr - 8;
1640 return 1;
1641}
1642
1643static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls,
1644 U32 extDict)
1645{
1646 U32 *const hashTable = zc->hashTable;
1647 U32 const hashLog = zc->params.cParams.hashLog;
1648 size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
1649 U32 *const bt = zc->chainTable;
1650 U32 const btLog = zc->params.cParams.chainLog - 1;
1651 U32 const btMask = (1 << btLog) - 1;
1652 U32 matchIndex = hashTable[h];
1653 size_t commonLengthSmaller = 0, commonLengthLarger = 0;
1654 const BYTE *const base = zc->base;
1655 const BYTE *const dictBase = zc->dictBase;
1656 const U32 dictLimit = zc->dictLimit;
1657 const BYTE *const dictEnd = dictBase + dictLimit;
1658 const BYTE *const prefixStart = base + dictLimit;
1659 const U32 curr = (U32)(ip - base);
1660 const U32 btLow = btMask >= curr ? 0 : curr - btMask;
1661 const U32 windowLow = zc->lowLimit;
1662 U32 *smallerPtr = bt + 2 * (curr & btMask);
1663 U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
1664 U32 matchEndIdx = curr + 8;
1665 U32 dummy32; /* to be nullified at the end */
1666 size_t bestLength = 0;
1667
1668 hashTable[h] = curr; /* Update Hash Table */
1669
1670 while (nbCompares-- && (matchIndex > windowLow)) {
1671 U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
1672 size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
1673 const BYTE *match;
1674
1675 if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
1676 match = base + matchIndex;
1677 if (match[matchLength] == ip[matchLength])
1678 matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
1679 } else {
1680 match = dictBase + matchIndex;
1681 matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
1682 if (matchIndex + matchLength >= dictLimit)
1683 match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
1684 }
1685
1686 if (matchLength > bestLength) {
1687 if (matchLength > matchEndIdx - matchIndex)
1688 matchEndIdx = matchIndex + (U32)matchLength;
1689 if ((4 * (int)(matchLength - bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1)))
1690 bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex;
1691 if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
1692 break; /* drop, to guarantee consistency (miss a little bit of compression) */
1693 }
1694
1695 if (match[matchLength] < ip[matchLength]) {
1696 /* match is smaller than curr */
1697 *smallerPtr = matchIndex; /* update smaller idx */
1698 commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
1699 if (matchIndex <= btLow) {
1700 smallerPtr = &dummy32;
1701 break;
1702 } /* beyond tree size, stop the search */
1703 smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
1704 matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
1705 } else {
1706 /* match is larger than curr */
1707 *largerPtr = matchIndex;
1708 commonLengthLarger = matchLength;
1709 if (matchIndex <= btLow) {
1710 largerPtr = &dummy32;
1711 break;
1712 } /* beyond tree size, stop the search */
1713 largerPtr = nextPtr;
1714 matchIndex = nextPtr[0];
1715 }
1716 }
1717
1718 *smallerPtr = *largerPtr = 0;
1719
1720 zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
1721 return bestLength;
1722}
1723
1724static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
1725{
1726 const BYTE *const base = zc->base;
1727 const U32 target = (U32)(ip - base);
1728 U32 idx = zc->nextToUpdate;
1729
1730 while (idx < target)
1731 idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 0);
1732}
1733
1734/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
1735static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls)
1736{
1737 if (ip < zc->base + zc->nextToUpdate)
1738 return 0; /* skipped area */
1739 ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
1740 return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0);
1741}
1742
1743static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
1744 const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch)
1745{
1746 switch (matchLengthSearch) {
1747 default: /* includes case 3 */
1748 case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
1749 case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
1750 case 7:
1751 case 6: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
1752 }
1753}
1754
1755static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
1756{
1757 const BYTE *const base = zc->base;
1758 const U32 target = (U32)(ip - base);
1759 U32 idx = zc->nextToUpdate;
1760
1761 while (idx < target)
1762 idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1);
1763}
1764
1765/** Tree updater, providing best match */
1766static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
1767 const U32 mls)
1768{
1769 if (ip < zc->base + zc->nextToUpdate)
1770 return 0; /* skipped area */
1771 ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
1772 return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1);
1773}
1774
1775static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
1776 const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
1777 const U32 matchLengthSearch)
1778{
1779 switch (matchLengthSearch) {
1780 default: /* includes case 3 */
1781 case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
1782 case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
1783 case 7:
1784 case 6: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
1785 }
1786}
1787
1788/* *********************************
1789* Hash Chain
1790***********************************/
1791#define NEXT_IN_CHAIN(d, mask) chainTable[(d)&mask]
1792
1793/* Update chains up to ip (excluded)
1794 Assumption : always within prefix (i.e. not within extDict) */
1795FORCE_INLINE
1796U32 ZSTD_insertAndFindFirstIndex(ZSTD_CCtx *zc, const BYTE *ip, U32 mls)
1797{
1798 U32 *const hashTable = zc->hashTable;
1799 const U32 hashLog = zc->params.cParams.hashLog;
1800 U32 *const chainTable = zc->chainTable;
1801 const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1;
1802 const BYTE *const base = zc->base;
1803 const U32 target = (U32)(ip - base);
1804 U32 idx = zc->nextToUpdate;
1805
1806 while (idx < target) { /* catch up */
1807 size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls);
1808 NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
1809 hashTable[h] = idx;
1810 idx++;
1811 }
1812
1813 zc->nextToUpdate = target;
1814 return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
1815}
1816
1817/* inlining is important to hardwire a hot branch (template emulation) */
1818FORCE_INLINE
1819size_t ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */
1820 const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls,
1821 const U32 extDict)
1822{
1823 U32 *const chainTable = zc->chainTable;
1824 const U32 chainSize = (1 << zc->params.cParams.chainLog);
1825 const U32 chainMask = chainSize - 1;
1826 const BYTE *const base = zc->base;
1827 const BYTE *const dictBase = zc->dictBase;
1828 const U32 dictLimit = zc->dictLimit;
1829 const BYTE *const prefixStart = base + dictLimit;
1830 const BYTE *const dictEnd = dictBase + dictLimit;
1831 const U32 lowLimit = zc->lowLimit;
1832 const U32 curr = (U32)(ip - base);
1833 const U32 minChain = curr > chainSize ? curr - chainSize : 0;
1834 int nbAttempts = maxNbAttempts;
1835 size_t ml = EQUAL_READ32 - 1;
1836
1837 /* HC4 match finder */
1838 U32 matchIndex = ZSTD_insertAndFindFirstIndex(zc, ip, mls);
1839
1840 for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) {
1841 const BYTE *match;
1842 size_t currMl = 0;
1843 if ((!extDict) || matchIndex >= dictLimit) {
1844 match = base + matchIndex;
1845 if (match[ml] == ip[ml]) /* potentially better */
1846 currMl = ZSTD_count(ip, match, iLimit);
1847 } else {
1848 match = dictBase + matchIndex;
1849 if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
1850 currMl = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32;
1851 }
1852
1853 /* save best solution */
1854 if (currMl > ml) {
1855 ml = currMl;
1856 *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE;
1857 if (ip + currMl == iLimit)
1858 break; /* best possible, and avoid read overflow*/
1859 }
1860
1861 if (matchIndex <= minChain)
1862 break;
1863 matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
1864 }
1865
1866 return ml;
1867}
1868
1869FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
1870 const U32 matchLengthSearch)
1871{
1872 switch (matchLengthSearch) {
1873 default: /* includes case 3 */
1874 case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0);
1875 case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0);
1876 case 7:
1877 case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0);
1878 }
1879}
1880
1881FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
1882 const U32 matchLengthSearch)
1883{
1884 switch (matchLengthSearch) {
1885 default: /* includes case 3 */
1886 case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1);
1887 case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1);
1888 case 7:
1889 case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1);
1890 }
1891}
1892
1893/* *******************************
1894* Common parser - lazy strategy
1895*********************************/
1896FORCE_INLINE
1897void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
1898{
1899 seqStore_t *seqStorePtr = &(ctx->seqStore);
1900 const BYTE *const istart = (const BYTE *)src;
1901 const BYTE *ip = istart;
1902 const BYTE *anchor = istart;
1903 const BYTE *const iend = istart + srcSize;
1904 const BYTE *const ilimit = iend - 8;
1905 const BYTE *const base = ctx->base + ctx->dictLimit;
1906
1907 U32 const maxSearches = 1 << ctx->params.cParams.searchLog;
1908 U32 const mls = ctx->params.cParams.searchLength;
1909
1910 typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
1911 searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
1912 U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0;
1913
1914 /* init */
1915 ip += (ip == base);
1916 ctx->nextToUpdate3 = ctx->nextToUpdate;
1917 {
1918 U32 const maxRep = (U32)(ip - base);
1919 if (offset_2 > maxRep)
1920 savedOffset = offset_2, offset_2 = 0;
1921 if (offset_1 > maxRep)
1922 savedOffset = offset_1, offset_1 = 0;
1923 }
1924
1925 /* Match Loop */
1926 while (ip < ilimit) {
1927 size_t matchLength = 0;
1928 size_t offset = 0;
1929 const BYTE *start = ip + 1;
1930
1931 /* check repCode */
1932 if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) {
1933 /* repcode : we take it */
1934 matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
1935 if (depth == 0)
1936 goto _storeSequence;
1937 }
1938
1939 /* first search (depth 0) */
1940 {
1941 size_t offsetFound = 99999999;
1942 size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
1943 if (ml2 > matchLength)
1944 matchLength = ml2, start = ip, offset = offsetFound;
1945 }
1946
1947 if (matchLength < EQUAL_READ32) {
1948 ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
1949 continue;
1950 }
1951
1952 /* let's try to find a better solution */
1953 if (depth >= 1)
1954 while (ip < ilimit) {
1955 ip++;
1956 if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
1957 size_t const mlRep = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
1958 int const gain2 = (int)(mlRep * 3);
1959 int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
1960 if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
1961 matchLength = mlRep, offset = 0, start = ip;
1962 }
1963 {
1964 size_t offset2 = 99999999;
1965 size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
1966 int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
1967 int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
1968 if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
1969 matchLength = ml2, offset = offset2, start = ip;
1970 continue; /* search a better one */
1971 }
1972 }
1973
1974 /* let's find an even better one */
1975 if ((depth == 2) && (ip < ilimit)) {
1976 ip++;
1977 if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
1978 size_t const ml2 = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
1979 int const gain2 = (int)(ml2 * 4);
1980 int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
1981 if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
1982 matchLength = ml2, offset = 0, start = ip;
1983 }
1984 {
1985 size_t offset2 = 99999999;
1986 size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
1987 int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
1988 int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
1989 if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
1990 matchLength = ml2, offset = offset2, start = ip;
1991 continue;
1992 }
1993 }
1994 }
1995 break; /* nothing found : store previous solution */
1996 }
1997
1998 /* NOTE:
1999 * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior.
2000 * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which
2001 * overflows the pointer, which is undefined behavior.
2002 */
2003 /* catch up */
2004 if (offset) {
2005 while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
2006 (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */
2007 {
2008 start--;
2009 matchLength++;
2010 }
2011 offset_2 = offset_1;
2012 offset_1 = (U32)(offset - ZSTD_REP_MOVE);
2013 }
2014
2015 /* store sequence */
2016_storeSequence:
2017 {
2018 size_t const litLength = start - anchor;
2019 ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
2020 anchor = ip = start + matchLength;
2021 }
2022
2023 /* check immediate repcode */
2024 while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
2025 /* store sequence */
2026 matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32;
2027 offset = offset_2;
2028 offset_2 = offset_1;
2029 offset_1 = (U32)offset; /* swap repcodes */
2030 ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
2031 ip += matchLength;
2032 anchor = ip;
2033 continue; /* faster when present ... (?) */
2034 }
2035 }
2036
2037 /* Save reps for next block */
2038 ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
2039 ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
2040
2041 /* Last Literals */
2042 {
2043 size_t const lastLLSize = iend - anchor;
2044 memcpy(seqStorePtr->lit, anchor, lastLLSize);
2045 seqStorePtr->lit += lastLLSize;
2046 }
2047}
2048
2049static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); }
2050
2051static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); }
2052
2053static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); }
2054
2055static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); }
2056
2057FORCE_INLINE
2058void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
2059{
2060 seqStore_t *seqStorePtr = &(ctx->seqStore);
2061 const BYTE *const istart = (const BYTE *)src;
2062 const BYTE *ip = istart;
2063 const BYTE *anchor = istart;
2064 const BYTE *const iend = istart + srcSize;
2065 const BYTE *const ilimit = iend - 8;
2066 const BYTE *const base = ctx->base;
2067 const U32 dictLimit = ctx->dictLimit;
2068 const U32 lowestIndex = ctx->lowLimit;
2069 const BYTE *const prefixStart = base + dictLimit;
2070 const BYTE *const dictBase = ctx->dictBase;
2071 const BYTE *const dictEnd = dictBase + dictLimit;
2072 const BYTE *const dictStart = dictBase + ctx->lowLimit;
2073
2074 const U32 maxSearches = 1 << ctx->params.cParams.searchLog;
2075 const U32 mls = ctx->params.cParams.searchLength;
2076
2077 typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
2078 searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;
2079
2080 U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
2081
2082 /* init */
2083 ctx->nextToUpdate3 = ctx->nextToUpdate;
2084 ip += (ip == prefixStart);
2085
2086 /* Match Loop */
2087 while (ip < ilimit) {
2088 size_t matchLength = 0;
2089 size_t offset = 0;
2090 const BYTE *start = ip + 1;
2091 U32 curr = (U32)(ip - base);
2092
2093 /* check repCode */
2094 {
2095 const U32 repIndex = (U32)(curr + 1 - offset_1);
2096 const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
2097 const BYTE *const repMatch = repBase + repIndex;
2098 if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
2099 if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) {
2100 /* repcode detected we should take it */
2101 const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
2102 matchLength =
2103 ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
2104 if (depth == 0)
2105 goto _storeSequence;
2106 }
2107 }
2108
2109 /* first search (depth 0) */
2110 {
2111 size_t offsetFound = 99999999;
2112 size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
2113 if (ml2 > matchLength)
2114 matchLength = ml2, start = ip, offset = offsetFound;
2115 }
2116
2117 if (matchLength < EQUAL_READ32) {
2118 ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
2119 continue;
2120 }
2121
2122 /* let's try to find a better solution */
2123 if (depth >= 1)
2124 while (ip < ilimit) {
2125 ip++;
2126 curr++;
2127 /* check repCode */
2128 if (offset) {
2129 const U32 repIndex = (U32)(curr - offset_1);
2130 const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
2131 const BYTE *const repMatch = repBase + repIndex;
2132 if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
2133 if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
2134 /* repcode detected */
2135 const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
2136 size_t const repLength =
2137 ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) +
2138 EQUAL_READ32;
2139 int const gain2 = (int)(repLength * 3);
2140 int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
2141 if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
2142 matchLength = repLength, offset = 0, start = ip;
2143 }
2144 }
2145
2146 /* search match, depth 1 */
2147 {
2148 size_t offset2 = 99999999;
2149 size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
2150 int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
2151 int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
2152 if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
2153 matchLength = ml2, offset = offset2, start = ip;
2154 continue; /* search a better one */
2155 }
2156 }
2157
2158 /* let's find an even better one */
2159 if ((depth == 2) && (ip < ilimit)) {
2160 ip++;
2161 curr++;
2162 /* check repCode */
2163 if (offset) {
2164 const U32 repIndex = (U32)(curr - offset_1);
2165 const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
2166 const BYTE *const repMatch = repBase + repIndex;
2167 if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
2168 if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
2169 /* repcode detected */
2170 const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
2171 size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend,
2172 repEnd, prefixStart) +
2173 EQUAL_READ32;
2174 int gain2 = (int)(repLength * 4);
2175 int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
2176 if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
2177 matchLength = repLength, offset = 0, start = ip;
2178 }
2179 }
2180
2181 /* search match, depth 2 */
2182 {
2183 size_t offset2 = 99999999;
2184 size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
2185 int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
2186 int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
2187 if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
2188 matchLength = ml2, offset = offset2, start = ip;
2189 continue;
2190 }
2191 }
2192 }
2193 break; /* nothing found : store previous solution */
2194 }
2195
2196 /* catch up */
2197 if (offset) {
2198 U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE));
2199 const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
2200 const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
2201 while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) {
2202 start--;
2203 match--;
2204 matchLength++;
2205 } /* catch up */
2206 offset_2 = offset_1;
2207 offset_1 = (U32)(offset - ZSTD_REP_MOVE);
2208 }
2209
2210 /* store sequence */
2211 _storeSequence : {
2212 size_t const litLength = start - anchor;
2213 ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
2214 anchor = ip = start + matchLength;
2215 }
2216
2217 /* check immediate repcode */
2218 while (ip <= ilimit) {
2219 const U32 repIndex = (U32)((ip - base) - offset_2);
2220 const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
2221 const BYTE *const repMatch = repBase + repIndex;
2222 if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
2223 if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
2224 /* repcode detected we should take it */
2225 const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
2226 matchLength =
2227 ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
2228 offset = offset_2;
2229 offset_2 = offset_1;
2230 offset_1 = (U32)offset; /* swap offset history */
2231 ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
2232 ip += matchLength;
2233 anchor = ip;
2234 continue; /* faster when present ... (?) */
2235 }
2236 break;
2237 }
2238 }
2239
2240 /* Save reps for next block */
2241 ctx->repToConfirm[0] = offset_1;
2242 ctx->repToConfirm[1] = offset_2;
2243
2244 /* Last Literals */
2245 {
2246 size_t const lastLLSize = iend - anchor;
2247 memcpy(seqStorePtr->lit, anchor, lastLLSize);
2248 seqStorePtr->lit += lastLLSize;
2249 }
2250}
2251
2252void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); }
2253
2254static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
2255{
2256 ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1);
2257}
2258
2259static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
2260{
2261 ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2);
2262}
2263
2264static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
2265{
2266 ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2);
2267}
2268
2269/* The optimal parser */
2270#include "zstd_opt.h"
2271
2272static void ZSTD_compressBlock_btopt(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
2273{
2274#ifdef ZSTD_OPT_H_91842398743
2275 ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
2276#else
2277 (void)ctx;
2278 (void)src;
2279 (void)srcSize;
2280 return;
2281#endif
2282}
2283
2284static void ZSTD_compressBlock_btopt2(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
2285{
2286#ifdef ZSTD_OPT_H_91842398743
2287 ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
2288#else
2289 (void)ctx;
2290 (void)src;
2291 (void)srcSize;
2292 return;
2293#endif
2294}
2295
2296static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
2297{
2298#ifdef ZSTD_OPT_H_91842398743
2299 ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
2300#else
2301 (void)ctx;
2302 (void)src;
2303 (void)srcSize;
2304 return;
2305#endif
2306}
2307
2308static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
2309{
2310#ifdef ZSTD_OPT_H_91842398743
2311 ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
2312#else
2313 (void)ctx;
2314 (void)src;
2315 (void)srcSize;
2316 return;
2317#endif
2318}
2319
2320typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize);
2321
2322static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
2323{
2324 static const ZSTD_blockCompressor blockCompressor[2][8] = {
2325 {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2,
2326 ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2},
2327 {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,
2328 ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}};
2329
2330 return blockCompressor[extDict][(U32)strat];
2331}
2332
2333static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
2334{
2335 ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
2336 const BYTE *const base = zc->base;
2337 const BYTE *const istart = (const BYTE *)src;
2338 const U32 curr = (U32)(istart - base);
2339 if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1)
2340 return 0; /* don't even attempt compression below a certain srcSize */
2341 ZSTD_resetSeqStore(&(zc->seqStore));
2342 if (curr > zc->nextToUpdate + 384)
2343 zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */
2344 blockCompressor(zc, src, srcSize);
2345 return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
2346}
2347
2348/*! ZSTD_compress_generic() :
2349* Compress a chunk of data into one or multiple blocks.
2350* All blocks will be terminated, all input will be consumed.
2351* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
2352* Frame is supposed already started (header already produced)
2353* @return : compressed size, or an error code
2354*/
2355static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk)
2356{
2357 size_t blockSize = cctx->blockSize;
2358 size_t remaining = srcSize;
2359 const BYTE *ip = (const BYTE *)src;
2360 BYTE *const ostart = (BYTE *)dst;
2361 BYTE *op = ostart;
2362 U32 const maxDist = 1 << cctx->params.cParams.windowLog;
2363
2364 if (cctx->params.fParams.checksumFlag && srcSize)
2365 xxh64_update(&cctx->xxhState, src, srcSize);
2366
2367 while (remaining) {
2368 U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
2369 size_t cSize;
2370
2371 if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
2372 return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
2373 if (remaining < blockSize)
2374 blockSize = remaining;
2375
2376 /* preemptive overflow correction */
2377 if (cctx->lowLimit > (3U << 29)) {
2378 U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
2379 U32 const curr = (U32)(ip - cctx->base);
2380 U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog);
2381 U32 const correction = curr - newCurr;
2382 ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
2383 ZSTD_reduceIndex(cctx, correction);
2384 cctx->base += correction;
2385 cctx->dictBase += correction;
2386 cctx->lowLimit -= correction;
2387 cctx->dictLimit -= correction;
2388 if (cctx->nextToUpdate < correction)
2389 cctx->nextToUpdate = 0;
2390 else
2391 cctx->nextToUpdate -= correction;
2392 }
2393
2394 if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
2395 /* enforce maxDist */
2396 U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist;
2397 if (cctx->lowLimit < newLowLimit)
2398 cctx->lowLimit = newLowLimit;
2399 if (cctx->dictLimit < cctx->lowLimit)
2400 cctx->dictLimit = cctx->lowLimit;
2401 }
2402
2403 cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize);
2404 if (ZSTD_isError(cSize))
2405 return cSize;
2406
2407 if (cSize == 0) { /* block is not compressible */
2408 U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3);
2409 if (blockSize + ZSTD_blockHeaderSize > dstCapacity)
2410 return ERROR(dstSize_tooSmall);
2411 ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */
2412 memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
2413 cSize = ZSTD_blockHeaderSize + blockSize;
2414 } else {
2415 U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3);
2416 ZSTD_writeLE24(op, cBlockHeader24);
2417 cSize += ZSTD_blockHeaderSize;
2418 }
2419
2420 remaining -= blockSize;
2421 dstCapacity -= cSize;
2422 ip += blockSize;
2423 op += cSize;
2424 }
2425
2426 if (lastFrameChunk && (op > ostart))
2427 cctx->stage = ZSTDcs_ending;
2428 return op - ostart;
2429}
2430
2431static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
2432{
2433 BYTE *const op = (BYTE *)dst;
2434 U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */
2435 U32 const checksumFlag = params.fParams.checksumFlag > 0;
2436 U32 const windowSize = 1U << params.cParams.windowLog;
2437 U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
2438 BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
2439 U32 const fcsCode =
2440 params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
2441 BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
2442 size_t pos;
2443
2444 if (dstCapacity < ZSTD_frameHeaderSize_max)
2445 return ERROR(dstSize_tooSmall);
2446
2447 ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
2448 op[4] = frameHeaderDecriptionByte;
2449 pos = 5;
2450 if (!singleSegment)
2451 op[pos++] = windowLogByte;
2452 switch (dictIDSizeCode) {
2453 default: /* impossible */
2454 case 0: break;
2455 case 1:
2456 op[pos] = (BYTE)(dictID);
2457 pos++;
2458 break;
2459 case 2:
2460 ZSTD_writeLE16(op + pos, (U16)dictID);
2461 pos += 2;
2462 break;
2463 case 3:
2464 ZSTD_writeLE32(op + pos, dictID);
2465 pos += 4;
2466 break;
2467 }
2468 switch (fcsCode) {
2469 default: /* impossible */
2470 case 0:
2471 if (singleSegment)
2472 op[pos++] = (BYTE)(pledgedSrcSize);
2473 break;
2474 case 1:
2475 ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256));
2476 pos += 2;
2477 break;
2478 case 2:
2479 ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize));
2480 pos += 4;
2481 break;
2482 case 3:
2483 ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize));
2484 pos += 8;
2485 break;
2486 }
2487 return pos;
2488}
2489
2490static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk)
2491{
2492 const BYTE *const ip = (const BYTE *)src;
2493 size_t fhSize = 0;
2494
2495 if (cctx->stage == ZSTDcs_created)
2496 return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
2497
2498 if (frame && (cctx->stage == ZSTDcs_init)) {
2499 fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
2500 if (ZSTD_isError(fhSize))
2501 return fhSize;
2502 dstCapacity -= fhSize;
2503 dst = (char *)dst + fhSize;
2504 cctx->stage = ZSTDcs_ongoing;
2505 }
2506
2507 /* Check if blocks follow each other */
2508 if (src != cctx->nextSrc) {
2509 /* not contiguous */
2510 ptrdiff_t const delta = cctx->nextSrc - ip;
2511 cctx->lowLimit = cctx->dictLimit;
2512 cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
2513 cctx->dictBase = cctx->base;
2514 cctx->base -= delta;
2515 cctx->nextToUpdate = cctx->dictLimit;
2516 if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE)
2517 cctx->lowLimit = cctx->dictLimit; /* too small extDict */
2518 }
2519
2520 /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
2521 if ((ip + srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
2522 ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
2523 U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
2524 cctx->lowLimit = lowLimitMax;
2525 }
2526
2527 cctx->nextSrc = ip + srcSize;
2528
2529 if (srcSize) {
2530 size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk)
2531 : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize);
2532 if (ZSTD_isError(cSize))
2533 return cSize;
2534 return cSize + fhSize;
2535 } else
2536 return fhSize;
2537}
2538
2539size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
2540{
2541 return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0);
2542}
2543
2544size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); }
2545
2546size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
2547{
2548 size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
2549 if (srcSize > blockSizeMax)
2550 return ERROR(srcSize_wrong);
2551 return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
2552}
2553
2554/*! ZSTD_loadDictionaryContent() :
2555 * @return : 0, or an error code
2556 */
2557static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx *zc, const void *src, size_t srcSize)
2558{
2559 const BYTE *const ip = (const BYTE *)src;
2560 const BYTE *const iend = ip + srcSize;
2561
2562 /* input becomes curr prefix */
2563 zc->lowLimit = zc->dictLimit;
2564 zc->dictLimit = (U32)(zc->nextSrc - zc->base);
2565 zc->dictBase = zc->base;
2566 zc->base += ip - zc->nextSrc;
2567 zc->nextToUpdate = zc->dictLimit;
2568 zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
2569
2570 zc->nextSrc = iend;
2571 if (srcSize <= HASH_READ_SIZE)
2572 return 0;
2573
2574 switch (zc->params.cParams.strategy) {
2575 case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break;
2576
2577 case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break;
2578
2579 case ZSTD_greedy:
2580 case ZSTD_lazy:
2581 case ZSTD_lazy2:
2582 if (srcSize >= HASH_READ_SIZE)
2583 ZSTD_insertAndFindFirstIndex(zc, iend - HASH_READ_SIZE, zc->params.cParams.searchLength);
2584 break;
2585
2586 case ZSTD_btlazy2:
2587 case ZSTD_btopt:
2588 case ZSTD_btopt2:
2589 if (srcSize >= HASH_READ_SIZE)
2590 ZSTD_updateTree(zc, iend - HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
2591 break;
2592
2593 default:
2594 return ERROR(GENERIC); /* strategy doesn't exist; impossible */
2595 }
2596
2597 zc->nextToUpdate = (U32)(iend - zc->base);
2598 return 0;
2599}
2600
2601/* Dictionaries that assign zero probability to symbols that show up causes problems
2602 when FSE encoding. Refuse dictionaries that assign zero probability to symbols
2603 that we may encounter during compression.
2604 NOTE: This behavior is not standard and could be improved in the future. */
2605static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
2606{
2607 U32 s;
2608 if (dictMaxSymbolValue < maxSymbolValue)
2609 return ERROR(dictionary_corrupted);
2610 for (s = 0; s <= maxSymbolValue; ++s) {
2611 if (normalizedCounter[s] == 0)
2612 return ERROR(dictionary_corrupted);
2613 }
2614 return 0;
2615}
2616
2617/* Dictionary format :
2618 * See :
2619 * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
2620 */
2621/*! ZSTD_loadZstdDictionary() :
2622 * @return : 0, or an error code
2623 * assumptions : magic number supposed already checked
2624 * dictSize supposed > 8
2625 */
2626static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
2627{
2628 const BYTE *dictPtr = (const BYTE *)dict;
2629 const BYTE *const dictEnd = dictPtr + dictSize;
2630 short offcodeNCount[MaxOff + 1];
2631 unsigned offcodeMaxValue = MaxOff;
2632
2633 dictPtr += 4; /* skip magic number */
2634 cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr);
2635 dictPtr += 4;
2636
2637 {
2638 size_t const hufHeaderSize = HUF_readCTable_wksp(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr, cctx->tmpCounters, sizeof(cctx->tmpCounters));
2639 if (HUF_isError(hufHeaderSize))
2640 return ERROR(dictionary_corrupted);
2641 dictPtr += hufHeaderSize;
2642 }
2643
2644 {
2645 unsigned offcodeLog;
2646 size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
2647 if (FSE_isError(offcodeHeaderSize))
2648 return ERROR(dictionary_corrupted);
2649 if (offcodeLog > OffFSELog)
2650 return ERROR(dictionary_corrupted);
2651 /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
2652 CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
2653 dictionary_corrupted);
2654 dictPtr += offcodeHeaderSize;
2655 }
2656
2657 {
2658 short matchlengthNCount[MaxML + 1];
2659 unsigned matchlengthMaxValue = MaxML, matchlengthLog;
2660 size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
2661 if (FSE_isError(matchlengthHeaderSize))
2662 return ERROR(dictionary_corrupted);
2663 if (matchlengthLog > MLFSELog)
2664 return ERROR(dictionary_corrupted);
2665 /* Every match length code must have non-zero probability */
2666 CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
2667 CHECK_E(
2668 FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
2669 dictionary_corrupted);
2670 dictPtr += matchlengthHeaderSize;
2671 }
2672
2673 {
2674 short litlengthNCount[MaxLL + 1];
2675 unsigned litlengthMaxValue = MaxLL, litlengthLog;
2676 size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
2677 if (FSE_isError(litlengthHeaderSize))
2678 return ERROR(dictionary_corrupted);
2679 if (litlengthLog > LLFSELog)
2680 return ERROR(dictionary_corrupted);
2681 /* Every literal length code must have non-zero probability */
2682 CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
2683 CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
2684 dictionary_corrupted);
2685 dictPtr += litlengthHeaderSize;
2686 }
2687
2688 if (dictPtr + 12 > dictEnd)
2689 return ERROR(dictionary_corrupted);
2690 cctx->rep[0] = ZSTD_readLE32(dictPtr + 0);
2691 cctx->rep[1] = ZSTD_readLE32(dictPtr + 4);
2692 cctx->rep[2] = ZSTD_readLE32(dictPtr + 8);
2693 dictPtr += 12;
2694
2695 {
2696 size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
2697 U32 offcodeMax = MaxOff;
2698 if (dictContentSize <= ((U32)-1) - 128 KB) {
2699 U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */
2700 offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
2701 }
2702 /* All offset values <= dictContentSize + 128 KB must be representable */
2703 CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
2704 /* All repCodes must be <= dictContentSize and != 0*/
2705 {
2706 U32 u;
2707 for (u = 0; u < 3; u++) {
2708 if (cctx->rep[u] == 0)
2709 return ERROR(dictionary_corrupted);
2710 if (cctx->rep[u] > dictContentSize)
2711 return ERROR(dictionary_corrupted);
2712 }
2713 }
2714
2715 cctx->flagStaticTables = 1;
2716 cctx->flagStaticHufTable = HUF_repeat_valid;
2717 return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize);
2718 }
2719}
2720
2721/** ZSTD_compress_insertDictionary() :
2722* @return : 0, or an error code */
2723static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
2724{
2725 if ((dict == NULL) || (dictSize <= 8))
2726 return 0;
2727
2728 /* dict as pure content */
2729 if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict))
2730 return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
2731
2732 /* dict as zstd dictionary */
2733 return ZSTD_loadZstdDictionary(cctx, dict, dictSize);
2734}
2735
2736/*! ZSTD_compressBegin_internal() :
2737* @return : 0, or an error code */
2738static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize)
2739{
2740 ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
2741 CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
2742 return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
2743}
2744
2745/*! ZSTD_compressBegin_advanced() :
2746* @return : 0, or an error code */
2747size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
2748{
2749 /* compression parameters verification and optimization */
2750 CHECK_F(ZSTD_checkCParams(params.cParams));
2751 return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
2752}
2753
2754size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel)
2755{
2756 ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
2757 return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
2758}
2759
2760size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); }
2761
2762/*! ZSTD_writeEpilogue() :
2763* Ends a frame.
2764* @return : nb of bytes written into dst (or an error code) */
2765static size_t ZSTD_writeEpilogue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity)
2766{
2767 BYTE *const ostart = (BYTE *)dst;
2768 BYTE *op = ostart;
2769 size_t fhSize = 0;
2770
2771 if (cctx->stage == ZSTDcs_created)
2772 return ERROR(stage_wrong); /* init missing */
2773
2774 /* special case : empty frame */
2775 if (cctx->stage == ZSTDcs_init) {
2776 fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
2777 if (ZSTD_isError(fhSize))
2778 return fhSize;
2779 dstCapacity -= fhSize;
2780 op += fhSize;
2781 cctx->stage = ZSTDcs_ongoing;
2782 }
2783
2784 if (cctx->stage != ZSTDcs_ending) {
2785 /* write one last empty block, make it the "last" block */
2786 U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0;
2787 if (dstCapacity < 4)
2788 return ERROR(dstSize_tooSmall);
2789 ZSTD_writeLE32(op, cBlockHeader24);
2790 op += ZSTD_blockHeaderSize;
2791 dstCapacity -= ZSTD_blockHeaderSize;
2792 }
2793
2794 if (cctx->params.fParams.checksumFlag) {
2795 U32 const checksum = (U32)xxh64_digest(&cctx->xxhState);
2796 if (dstCapacity < 4)
2797 return ERROR(dstSize_tooSmall);
2798 ZSTD_writeLE32(op, checksum);
2799 op += 4;
2800 }
2801
2802 cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
2803 return op - ostart;
2804}
2805
2806size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
2807{
2808 size_t endResult;
2809 size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
2810 if (ZSTD_isError(cSize))
2811 return cSize;
2812 endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize);
2813 if (ZSTD_isError(endResult))
2814 return endResult;
2815 return cSize + endResult;
2816}
2817
2818static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
2819 ZSTD_parameters params)
2820{
2821 CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
2822 return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
2823}
2824
2825size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
2826 ZSTD_parameters params)
2827{
2828 return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
2829}
2830
2831size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params)
2832{
2833 return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params);
2834}
2835
2836/* ===== Dictionary API ===== */
2837
2838struct ZSTD_CDict_s {
2839 void *dictBuffer;
2840 const void *dictContent;
2841 size_t dictContentSize;
2842 ZSTD_CCtx *refContext;
2843}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
2844
2845size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); }
2846
2847static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem)
2848{
2849 if (!customMem.customAlloc || !customMem.customFree)
2850 return NULL;
2851
2852 {
2853 ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
2854 ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem);
2855
2856 if (!cdict || !cctx) {
2857 ZSTD_free(cdict, customMem);
2858 ZSTD_freeCCtx(cctx);
2859 return NULL;
2860 }
2861
2862 if ((byReference) || (!dictBuffer) || (!dictSize)) {
2863 cdict->dictBuffer = NULL;
2864 cdict->dictContent = dictBuffer;
2865 } else {
2866 void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
2867 if (!internalBuffer) {
2868 ZSTD_free(cctx, customMem);
2869 ZSTD_free(cdict, customMem);
2870 return NULL;
2871 }
2872 memcpy(internalBuffer, dictBuffer, dictSize);
2873 cdict->dictBuffer = internalBuffer;
2874 cdict->dictContent = internalBuffer;
2875 }
2876
2877 {
2878 size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
2879 if (ZSTD_isError(errorCode)) {
2880 ZSTD_free(cdict->dictBuffer, customMem);
2881 ZSTD_free(cdict, customMem);
2882 ZSTD_freeCCtx(cctx);
2883 return NULL;
2884 }
2885 }
2886
2887 cdict->refContext = cctx;
2888 cdict->dictContentSize = dictSize;
2889 return cdict;
2890 }
2891}
2892
2893ZSTD_CDict *ZSTD_initCDict(const void *dict, size_t dictSize, ZSTD_parameters params, void *workspace, size_t workspaceSize)
2894{
2895 ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
2896 return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem);
2897}
2898
2899size_t ZSTD_freeCDict(ZSTD_CDict *cdict)
2900{
2901 if (cdict == NULL)
2902 return 0; /* support free on NULL */
2903 {
2904 ZSTD_customMem const cMem = cdict->refContext->customMem;
2905 ZSTD_freeCCtx(cdict->refContext);
2906 ZSTD_free(cdict->dictBuffer, cMem);
2907 ZSTD_free(cdict, cMem);
2908 return 0;
2909 }
2910}
2911
2912static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); }
2913
2914size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize)
2915{
2916 if (cdict->dictContentSize)
2917 CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
2918 else {
2919 ZSTD_parameters params = cdict->refContext->params;
2920 params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
2921 CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize));
2922 }
2923 return 0;
2924}
2925
2926/*! ZSTD_compress_usingCDict() :
2927* Compression using a digested Dictionary.
2928* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
2929* Note that compression level is decided during dictionary creation */
2930size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict)
2931{
2932 CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
2933
2934 if (cdict->refContext->params.fParams.contentSizeFlag == 1) {
2935 cctx->params.fParams.contentSizeFlag = 1;
2936 cctx->frameContentSize = srcSize;
2937 } else {
2938 cctx->params.fParams.contentSizeFlag = 0;
2939 }
2940
2941 return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
2942}
2943
2944/* ******************************************************************
2945* Streaming
2946********************************************************************/
2947
2948typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
2949
2950struct ZSTD_CStream_s {
2951 ZSTD_CCtx *cctx;
2952 ZSTD_CDict *cdictLocal;
2953 const ZSTD_CDict *cdict;
2954 char *inBuff;
2955 size_t inBuffSize;
2956 size_t inToCompress;
2957 size_t inBuffPos;
2958 size_t inBuffTarget;
2959 size_t blockSize;
2960 char *outBuff;
2961 size_t outBuffSize;
2962 size_t outBuffContentSize;
2963 size_t outBuffFlushedSize;
2964 ZSTD_cStreamStage stage;
2965 U32 checksum;
2966 U32 frameEnded;
2967 U64 pledgedSrcSize;
2968 U64 inputProcessed;
2969 ZSTD_parameters params;
2970 ZSTD_customMem customMem;
2971}; /* typedef'd to ZSTD_CStream within "zstd.h" */
2972
2973size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams)
2974{
2975 size_t const inBuffSize = (size_t)1 << cParams.windowLog;
2976 size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, inBuffSize);
2977 size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
2978
2979 return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
2980}
2981
2982ZSTD_CStream *ZSTD_createCStream_advanced(ZSTD_customMem customMem)
2983{
2984 ZSTD_CStream *zcs;
2985
2986 if (!customMem.customAlloc || !customMem.customFree)
2987 return NULL;
2988
2989 zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem);
2990 if (zcs == NULL)
2991 return NULL;
2992 memset(zcs, 0, sizeof(ZSTD_CStream));
2993 memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
2994 zcs->cctx = ZSTD_createCCtx_advanced(customMem);
2995 if (zcs->cctx == NULL) {
2996 ZSTD_freeCStream(zcs);
2997 return NULL;
2998 }
2999 return zcs;
3000}
3001
3002size_t ZSTD_freeCStream(ZSTD_CStream *zcs)
3003{
3004 if (zcs == NULL)
3005 return 0; /* support free on NULL */
3006 {
3007 ZSTD_customMem const cMem = zcs->customMem;
3008 ZSTD_freeCCtx(zcs->cctx);
3009 zcs->cctx = NULL;
3010 ZSTD_freeCDict(zcs->cdictLocal);
3011 zcs->cdictLocal = NULL;
3012 ZSTD_free(zcs->inBuff, cMem);
3013 zcs->inBuff = NULL;
3014 ZSTD_free(zcs->outBuff, cMem);
3015 zcs->outBuff = NULL;
3016 ZSTD_free(zcs, cMem);
3017 return 0;
3018 }
3019}
3020
3021/*====== Initialization ======*/
3022
3023size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
3024size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; }
3025
3026static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
3027{
3028 if (zcs->inBuffSize == 0)
3029 return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
3030
3031 if (zcs->cdict)
3032 CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
3033 else
3034 CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
3035
3036 zcs->inToCompress = 0;
3037 zcs->inBuffPos = 0;
3038 zcs->inBuffTarget = zcs->blockSize;
3039 zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
3040 zcs->stage = zcss_load;
3041 zcs->frameEnded = 0;
3042 zcs->pledgedSrcSize = pledgedSrcSize;
3043 zcs->inputProcessed = 0;
3044 return 0; /* ready to go */
3045}
3046
3047size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
3048{
3049
3050 zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
3051
3052 return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
3053}
3054
3055static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
3056{
3057 /* allocate buffers */
3058 {
3059 size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
3060 if (zcs->inBuffSize < neededInBuffSize) {
3061 zcs->inBuffSize = neededInBuffSize;
3062 ZSTD_free(zcs->inBuff, zcs->customMem);
3063 zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem);
3064 if (zcs->inBuff == NULL)
3065 return ERROR(memory_allocation);
3066 }
3067 zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
3068 }
3069 if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize) + 1) {
3070 zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1;
3071 ZSTD_free(zcs->outBuff, zcs->customMem);
3072 zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
3073 if (zcs->outBuff == NULL)
3074 return ERROR(memory_allocation);
3075 }
3076
3077 if (dict && dictSize >= 8) {
3078 ZSTD_freeCDict(zcs->cdictLocal);
3079 zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
3080 if (zcs->cdictLocal == NULL)
3081 return ERROR(memory_allocation);
3082 zcs->cdict = zcs->cdictLocal;
3083 } else
3084 zcs->cdict = NULL;
3085
3086 zcs->checksum = params.fParams.checksumFlag > 0;
3087 zcs->params = params;
3088
3089 return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
3090}
3091
3092ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
3093{
3094 ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
3095 ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem);
3096 if (zcs) {
3097 size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
3098 if (ZSTD_isError(code)) {
3099 return NULL;
3100 }
3101 }
3102 return zcs;
3103}
3104
3105ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
3106{
3107 ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
3108 ZSTD_CStream *const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize);
3109 if (zcs) {
3110 zcs->cdict = cdict;
3111 if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) {
3112 return NULL;
3113 }
3114 }
3115 return zcs;
3116}
3117
3118/*====== Compression ======*/
3119
3120typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
3121
3122ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
3123{
3124 size_t const length = MIN(dstCapacity, srcSize);
3125 memcpy(dst, src, length);
3126 return length;
3127}
3128
3129static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush)
3130{
3131 U32 someMoreWork = 1;
3132 const char *const istart = (const char *)src;
3133 const char *const iend = istart + *srcSizePtr;
3134 const char *ip = istart;
3135 char *const ostart = (char *)dst;
3136 char *const oend = ostart + *dstCapacityPtr;
3137 char *op = ostart;
3138
3139 while (someMoreWork) {
3140 switch (zcs->stage) {
3141 case zcss_init:
3142 return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
3143
3144 case zcss_load:
3145 /* complete inBuffer */
3146 {
3147 size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
3148 size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip);
3149 zcs->inBuffPos += loaded;
3150 ip += loaded;
3151 if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) {
3152 someMoreWork = 0;
3153 break; /* not enough input to get a full block : stop there, wait for more */
3154 }
3155 }
3156 /* compress curr block (note : this stage cannot be stopped in the middle) */
3157 {
3158 void *cDst;
3159 size_t cSize;
3160 size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
3161 size_t oSize = oend - op;
3162 if (oSize >= ZSTD_compressBound(iSize))
3163 cDst = op; /* compress directly into output buffer (avoid flush stage) */
3164 else
3165 cDst = zcs->outBuff, oSize = zcs->outBuffSize;
3166 cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize)
3167 : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
3168 if (ZSTD_isError(cSize))
3169 return cSize;
3170 if (flush == zsf_end)
3171 zcs->frameEnded = 1;
3172 /* prepare next block */
3173 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
3174 if (zcs->inBuffTarget > zcs->inBuffSize)
3175 zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
3176 zcs->inToCompress = zcs->inBuffPos;
3177 if (cDst == op) {
3178 op += cSize;
3179 break;
3180 } /* no need to flush */
3181 zcs->outBuffContentSize = cSize;
3182 zcs->outBuffFlushedSize = 0;
3183 zcs->stage = zcss_flush; /* pass-through to flush stage */
3184 }
3185
3186 case zcss_flush: {
3187 size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
3188 size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
3189 op += flushed;
3190 zcs->outBuffFlushedSize += flushed;
3191 if (toFlush != flushed) {
3192 someMoreWork = 0;
3193 break;
3194 } /* dst too small to store flushed data : stop there */
3195 zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
3196 zcs->stage = zcss_load;
3197 break;
3198 }
3199
3200 case zcss_final:
3201 someMoreWork = 0; /* do nothing */
3202 break;
3203
3204 default:
3205 return ERROR(GENERIC); /* impossible */
3206 }
3207 }
3208
3209 *srcSizePtr = ip - istart;
3210 *dstCapacityPtr = op - ostart;
3211 zcs->inputProcessed += *srcSizePtr;
3212 if (zcs->frameEnded)
3213 return 0;
3214 {
3215 size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
3216 if (hintInSize == 0)
3217 hintInSize = zcs->blockSize;
3218 return hintInSize;
3219 }
3220}
3221
3222size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
3223{
3224 size_t sizeRead = input->size - input->pos;
3225 size_t sizeWritten = output->size - output->pos;
3226 size_t const result =
3227 ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather);
3228 input->pos += sizeRead;
3229 output->pos += sizeWritten;
3230 return result;
3231}
3232
3233/*====== Finalize ======*/
3234
3235/*! ZSTD_flushStream() :
3236* @return : amount of data remaining to flush */
3237size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
3238{
3239 size_t srcSize = 0;
3240 size_t sizeWritten = output->size - output->pos;
3241 size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize,
3242 &srcSize, /* use a valid src address instead of NULL */
3243 zsf_flush);
3244 output->pos += sizeWritten;
3245 if (ZSTD_isError(result))
3246 return result;
3247 return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
3248}
3249
3250size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
3251{
3252 BYTE *const ostart = (BYTE *)(output->dst) + output->pos;
3253 BYTE *const oend = (BYTE *)(output->dst) + output->size;
3254 BYTE *op = ostart;
3255
3256 if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
3257 return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */
3258
3259 if (zcs->stage != zcss_final) {
3260 /* flush whatever remains */
3261 size_t srcSize = 0;
3262 size_t sizeWritten = output->size - output->pos;
3263 size_t const notEnded =
3264 ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
3265 size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
3266 op += sizeWritten;
3267 if (remainingToFlush) {
3268 output->pos += sizeWritten;
3269 return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
3270 }
3271 /* create epilogue */
3272 zcs->stage = zcss_final;
3273 zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL,
3274 0); /* write epilogue, including final empty block, into outBuff */
3275 }
3276
3277 /* flush epilogue */
3278 {
3279 size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
3280 size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
3281 op += flushed;
3282 zcs->outBuffFlushedSize += flushed;
3283 output->pos += op - ostart;
3284 if (toFlush == flushed)
3285 zcs->stage = zcss_init; /* end reached */
3286 return toFlush - flushed;
3287 }
3288}
3289
3290/*-===== Pre-defined compression levels =====-*/
3291
3292#define ZSTD_DEFAULT_CLEVEL 1
3293#define ZSTD_MAX_CLEVEL 22
3294int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
3295
3296static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = {
3297 {
3298 /* "default" */
3299 /* W, C, H, S, L, TL, strat */
3300 {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */
3301 {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */
3302 {19, 15, 16, 1, 6, 16, ZSTD_fast}, /* level 2 */
3303 {20, 16, 17, 1, 5, 16, ZSTD_dfast}, /* level 3.*/
3304 {20, 18, 18, 1, 5, 16, ZSTD_dfast}, /* level 4.*/
3305 {20, 15, 18, 3, 5, 16, ZSTD_greedy}, /* level 5 */
3306 {21, 16, 19, 2, 5, 16, ZSTD_lazy}, /* level 6 */
3307 {21, 17, 20, 3, 5, 16, ZSTD_lazy}, /* level 7 */
3308 {21, 18, 20, 3, 5, 16, ZSTD_lazy2}, /* level 8 */
3309 {21, 20, 20, 3, 5, 16, ZSTD_lazy2}, /* level 9 */
3310 {21, 19, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */
3311 {22, 20, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */
3312 {22, 20, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */
3313 {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 13 */
3314 {22, 21, 22, 6, 5, 16, ZSTD_lazy2}, /* level 14 */
3315 {22, 21, 21, 5, 5, 16, ZSTD_btlazy2}, /* level 15 */
3316 {23, 22, 22, 5, 5, 16, ZSTD_btlazy2}, /* level 16 */
3317 {23, 21, 22, 4, 5, 24, ZSTD_btopt}, /* level 17 */
3318 {23, 23, 22, 6, 5, 32, ZSTD_btopt}, /* level 18 */
3319 {23, 23, 22, 6, 3, 48, ZSTD_btopt}, /* level 19 */
3320 {25, 25, 23, 7, 3, 64, ZSTD_btopt2}, /* level 20 */
3321 {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */
3322 {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */
3323 },
3324 {
3325 /* for srcSize <= 256 KB */
3326 /* W, C, H, S, L, T, strat */
3327 {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */
3328 {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */
3329 {18, 14, 13, 1, 5, 8, ZSTD_dfast}, /* level 2 */
3330 {18, 16, 15, 1, 5, 8, ZSTD_dfast}, /* level 3 */
3331 {18, 15, 17, 1, 5, 8, ZSTD_greedy}, /* level 4.*/
3332 {18, 16, 17, 4, 5, 8, ZSTD_greedy}, /* level 5.*/
3333 {18, 16, 17, 3, 5, 8, ZSTD_lazy}, /* level 6.*/
3334 {18, 17, 17, 4, 4, 8, ZSTD_lazy}, /* level 7 */
3335 {18, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
3336 {18, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
3337 {18, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
3338 {18, 18, 17, 6, 4, 8, ZSTD_lazy2}, /* level 11.*/
3339 {18, 18, 17, 7, 4, 8, ZSTD_lazy2}, /* level 12.*/
3340 {18, 19, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13 */
3341 {18, 18, 18, 4, 4, 16, ZSTD_btopt}, /* level 14.*/
3342 {18, 18, 18, 4, 3, 16, ZSTD_btopt}, /* level 15.*/
3343 {18, 19, 18, 6, 3, 32, ZSTD_btopt}, /* level 16.*/
3344 {18, 19, 18, 8, 3, 64, ZSTD_btopt}, /* level 17.*/
3345 {18, 19, 18, 9, 3, 128, ZSTD_btopt}, /* level 18.*/
3346 {18, 19, 18, 10, 3, 256, ZSTD_btopt}, /* level 19.*/
3347 {18, 19, 18, 11, 3, 512, ZSTD_btopt2}, /* level 20.*/
3348 {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/
3349 {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/
3350 },
3351 {
3352 /* for srcSize <= 128 KB */
3353 /* W, C, H, S, L, T, strat */
3354 {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */
3355 {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */
3356 {17, 13, 16, 1, 5, 8, ZSTD_fast}, /* level 2 */
3357 {17, 16, 16, 2, 5, 8, ZSTD_dfast}, /* level 3 */
3358 {17, 13, 15, 3, 4, 8, ZSTD_greedy}, /* level 4 */
3359 {17, 15, 17, 4, 4, 8, ZSTD_greedy}, /* level 5 */
3360 {17, 16, 17, 3, 4, 8, ZSTD_lazy}, /* level 6 */
3361 {17, 15, 17, 4, 4, 8, ZSTD_lazy2}, /* level 7 */
3362 {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
3363 {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
3364 {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
3365 {17, 17, 17, 7, 4, 8, ZSTD_lazy2}, /* level 11 */
3366 {17, 17, 17, 8, 4, 8, ZSTD_lazy2}, /* level 12 */
3367 {17, 18, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13.*/
3368 {17, 17, 17, 7, 3, 8, ZSTD_btopt}, /* level 14.*/
3369 {17, 17, 17, 7, 3, 16, ZSTD_btopt}, /* level 15.*/
3370 {17, 18, 17, 7, 3, 32, ZSTD_btopt}, /* level 16.*/
3371 {17, 18, 17, 7, 3, 64, ZSTD_btopt}, /* level 17.*/
3372 {17, 18, 17, 7, 3, 256, ZSTD_btopt}, /* level 18.*/
3373 {17, 18, 17, 8, 3, 256, ZSTD_btopt}, /* level 19.*/
3374 {17, 18, 17, 9, 3, 256, ZSTD_btopt2}, /* level 20.*/
3375 {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/
3376 {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/
3377 },
3378 {
3379 /* for srcSize <= 16 KB */
3380 /* W, C, H, S, L, T, strat */
3381 {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */
3382 {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */
3383 {14, 14, 14, 1, 4, 6, ZSTD_fast}, /* level 2 */
3384 {14, 14, 14, 1, 4, 6, ZSTD_dfast}, /* level 3.*/
3385 {14, 14, 14, 4, 4, 6, ZSTD_greedy}, /* level 4.*/
3386 {14, 14, 14, 3, 4, 6, ZSTD_lazy}, /* level 5.*/
3387 {14, 14, 14, 4, 4, 6, ZSTD_lazy2}, /* level 6 */
3388 {14, 14, 14, 5, 4, 6, ZSTD_lazy2}, /* level 7 */
3389 {14, 14, 14, 6, 4, 6, ZSTD_lazy2}, /* level 8.*/
3390 {14, 15, 14, 6, 4, 6, ZSTD_btlazy2}, /* level 9.*/
3391 {14, 15, 14, 3, 3, 6, ZSTD_btopt}, /* level 10.*/
3392 {14, 15, 14, 6, 3, 8, ZSTD_btopt}, /* level 11.*/
3393 {14, 15, 14, 6, 3, 16, ZSTD_btopt}, /* level 12.*/
3394 {14, 15, 14, 6, 3, 24, ZSTD_btopt}, /* level 13.*/
3395 {14, 15, 15, 6, 3, 48, ZSTD_btopt}, /* level 14.*/
3396 {14, 15, 15, 6, 3, 64, ZSTD_btopt}, /* level 15.*/
3397 {14, 15, 15, 6, 3, 96, ZSTD_btopt}, /* level 16.*/
3398 {14, 15, 15, 6, 3, 128, ZSTD_btopt}, /* level 17.*/
3399 {14, 15, 15, 6, 3, 256, ZSTD_btopt}, /* level 18.*/
3400 {14, 15, 15, 7, 3, 256, ZSTD_btopt}, /* level 19.*/
3401 {14, 15, 15, 8, 3, 256, ZSTD_btopt2}, /* level 20.*/
3402 {14, 15, 15, 9, 3, 256, ZSTD_btopt2}, /* level 21.*/
3403 {14, 15, 15, 10, 3, 256, ZSTD_btopt2}, /* level 22.*/
3404 },
3405};
3406
3407/*! ZSTD_getCParams() :
3408* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
3409* Size values are optional, provide 0 if not known or unused */
3410ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
3411{
3412 ZSTD_compressionParameters cp;
3413 size_t const addedSize = srcSize ? 0 : 500;
3414 U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1;
3415 U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
3416 if (compressionLevel <= 0)
3417 compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */
3418 if (compressionLevel > ZSTD_MAX_CLEVEL)
3419 compressionLevel = ZSTD_MAX_CLEVEL;
3420 cp = ZSTD_defaultCParameters[tableID][compressionLevel];
3421 if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */
3422 if (cp.windowLog > ZSTD_WINDOWLOG_MAX)
3423 cp.windowLog = ZSTD_WINDOWLOG_MAX;
3424 if (cp.chainLog > ZSTD_CHAINLOG_MAX)
3425 cp.chainLog = ZSTD_CHAINLOG_MAX;
3426 if (cp.hashLog > ZSTD_HASHLOG_MAX)
3427 cp.hashLog = ZSTD_HASHLOG_MAX;
3428 }
3429 cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
3430 return cp;
3431}
3432
3433/*! ZSTD_getParams() :
3434* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`).
3435* All fields of `ZSTD_frameParameters` are set to default (0) */
3436ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
3437{
3438 ZSTD_parameters params;
3439 ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
3440 memset(&params, 0, sizeof(params));
3441 params.cParams = cParams;
3442 return params;
3443}
3444
3445EXPORT_SYMBOL(ZSTD_maxCLevel);
3446EXPORT_SYMBOL(ZSTD_compressBound);
3447
3448EXPORT_SYMBOL(ZSTD_CCtxWorkspaceBound);
3449EXPORT_SYMBOL(ZSTD_initCCtx);
3450EXPORT_SYMBOL(ZSTD_compressCCtx);
3451EXPORT_SYMBOL(ZSTD_compress_usingDict);
3452
3453EXPORT_SYMBOL(ZSTD_CDictWorkspaceBound);
3454EXPORT_SYMBOL(ZSTD_initCDict);
3455EXPORT_SYMBOL(ZSTD_compress_usingCDict);
3456
3457EXPORT_SYMBOL(ZSTD_CStreamWorkspaceBound);
3458EXPORT_SYMBOL(ZSTD_initCStream);
3459EXPORT_SYMBOL(ZSTD_initCStream_usingCDict);
3460EXPORT_SYMBOL(ZSTD_resetCStream);
3461EXPORT_SYMBOL(ZSTD_compressStream);
3462EXPORT_SYMBOL(ZSTD_flushStream);
3463EXPORT_SYMBOL(ZSTD_endStream);
3464EXPORT_SYMBOL(ZSTD_CStreamInSize);
3465EXPORT_SYMBOL(ZSTD_CStreamOutSize);
3466
3467EXPORT_SYMBOL(ZSTD_getCParams);
3468EXPORT_SYMBOL(ZSTD_getParams);
3469EXPORT_SYMBOL(ZSTD_checkCParams);
3470EXPORT_SYMBOL(ZSTD_adjustCParams);
3471
3472EXPORT_SYMBOL(ZSTD_compressBegin);
3473EXPORT_SYMBOL(ZSTD_compressBegin_usingDict);
3474EXPORT_SYMBOL(ZSTD_compressBegin_advanced);
3475EXPORT_SYMBOL(ZSTD_copyCCtx);
3476EXPORT_SYMBOL(ZSTD_compressBegin_usingCDict);
3477EXPORT_SYMBOL(ZSTD_compressContinue);
3478EXPORT_SYMBOL(ZSTD_compressEnd);
3479
3480EXPORT_SYMBOL(ZSTD_getBlockSizeMax);
3481EXPORT_SYMBOL(ZSTD_compressBlock);
3482
3483MODULE_LICENSE("Dual BSD/GPL");
3484MODULE_DESCRIPTION("Zstd Compressor");
diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c
new file mode 100644
index 000000000000..b17846725ca0
--- /dev/null
+++ b/lib/zstd/decompress.c
@@ -0,0 +1,2528 @@
1/**
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of https://github.com/facebook/zstd.
7 * An additional grant of patent rights can be found in the PATENTS file in the
8 * same directory.
9 *
10 * This program is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU General Public License version 2 as published by the
12 * Free Software Foundation. This program is dual-licensed; you may select
13 * either version 2 of the GNU General Public License ("GPL") or BSD license
14 * ("BSD").
15 */
16
17/* ***************************************************************
18* Tuning parameters
19*****************************************************************/
20/*!
21* MAXWINDOWSIZE_DEFAULT :
22* maximum window size accepted by DStream, by default.
23* Frames requiring more memory will be rejected.
24*/
25#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
26#define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
27#endif
28
29/*-*******************************************************
30* Dependencies
31*********************************************************/
32#include "fse.h"
33#include "huf.h"
34#include "mem.h" /* low level memory routines */
35#include "zstd_internal.h"
36#include <linux/kernel.h>
37#include <linux/module.h>
38#include <linux/string.h> /* memcpy, memmove, memset */
39
40#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
41
42/*-*************************************
43* Macros
44***************************************/
45#define ZSTD_isError ERR_isError /* for inlining */
46#define FSE_isError ERR_isError
47#define HUF_isError ERR_isError
48
49/*_*******************************************************
50* Memory operations
51**********************************************************/
52static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); }
53
54/*-*************************************************************
55* Context management
56***************************************************************/
57typedef enum {
58 ZSTDds_getFrameHeaderSize,
59 ZSTDds_decodeFrameHeader,
60 ZSTDds_decodeBlockHeader,
61 ZSTDds_decompressBlock,
62 ZSTDds_decompressLastBlock,
63 ZSTDds_checkChecksum,
64 ZSTDds_decodeSkippableHeader,
65 ZSTDds_skipFrame
66} ZSTD_dStage;
67
68typedef struct {
69 FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
70 FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
71 FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
72 HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
73 U64 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32 / 2];
74 U32 rep[ZSTD_REP_NUM];
75} ZSTD_entropyTables_t;
76
77struct ZSTD_DCtx_s {
78 const FSE_DTable *LLTptr;
79 const FSE_DTable *MLTptr;
80 const FSE_DTable *OFTptr;
81 const HUF_DTable *HUFptr;
82 ZSTD_entropyTables_t entropy;
83 const void *previousDstEnd; /* detect continuity */
84 const void *base; /* start of curr segment */
85 const void *vBase; /* virtual start of previous segment if it was just before curr one */
86 const void *dictEnd; /* end of previous segment */
87 size_t expected;
88 ZSTD_frameParams fParams;
89 blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
90 ZSTD_dStage stage;
91 U32 litEntropy;
92 U32 fseEntropy;
93 struct xxh64_state xxhState;
94 size_t headerSize;
95 U32 dictID;
96 const BYTE *litPtr;
97 ZSTD_customMem customMem;
98 size_t litSize;
99 size_t rleSize;
100 BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
101 BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
102}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
103
104size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); }
105
106size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx)
107{
108 dctx->expected = ZSTD_frameHeaderSize_prefix;
109 dctx->stage = ZSTDds_getFrameHeaderSize;
110 dctx->previousDstEnd = NULL;
111 dctx->base = NULL;
112 dctx->vBase = NULL;
113 dctx->dictEnd = NULL;
114 dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
115 dctx->litEntropy = dctx->fseEntropy = 0;
116 dctx->dictID = 0;
117 ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
118 memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
119 dctx->LLTptr = dctx->entropy.LLTable;
120 dctx->MLTptr = dctx->entropy.MLTable;
121 dctx->OFTptr = dctx->entropy.OFTable;
122 dctx->HUFptr = dctx->entropy.hufTable;
123 return 0;
124}
125
126ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
127{
128 ZSTD_DCtx *dctx;
129
130 if (!customMem.customAlloc || !customMem.customFree)
131 return NULL;
132
133 dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
134 if (!dctx)
135 return NULL;
136 memcpy(&dctx->customMem, &customMem, sizeof(customMem));
137 ZSTD_decompressBegin(dctx);
138 return dctx;
139}
140
141ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize)
142{
143 ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
144 return ZSTD_createDCtx_advanced(stackMem);
145}
146
147size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx)
148{
149 if (dctx == NULL)
150 return 0; /* support free on NULL */
151 ZSTD_free(dctx, dctx->customMem);
152 return 0; /* reserved as a potential error code in the future */
153}
154
155void ZSTD_copyDCtx(ZSTD_DCtx *dstDCtx, const ZSTD_DCtx *srcDCtx)
156{
157 size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
158 memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
159}
160
161static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict);
162
163/*-*************************************************************
164* Decompression section
165***************************************************************/
166
167/*! ZSTD_isFrame() :
168 * Tells if the content of `buffer` starts with a valid Frame Identifier.
169 * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
170 * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
171 * Note 3 : Skippable Frame Identifiers are considered valid. */
172unsigned ZSTD_isFrame(const void *buffer, size_t size)
173{
174 if (size < 4)
175 return 0;
176 {
177 U32 const magic = ZSTD_readLE32(buffer);
178 if (magic == ZSTD_MAGICNUMBER)
179 return 1;
180 if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START)
181 return 1;
182 }
183 return 0;
184}
185
186/** ZSTD_frameHeaderSize() :
187* srcSize must be >= ZSTD_frameHeaderSize_prefix.
188* @return : size of the Frame Header */
189static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize)
190{
191 if (srcSize < ZSTD_frameHeaderSize_prefix)
192 return ERROR(srcSize_wrong);
193 {
194 BYTE const fhd = ((const BYTE *)src)[4];
195 U32 const dictID = fhd & 3;
196 U32 const singleSegment = (fhd >> 5) & 1;
197 U32 const fcsId = fhd >> 6;
198 return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId);
199 }
200}
201
202/** ZSTD_getFrameParams() :
203* decode Frame Header, or require larger `srcSize`.
204* @return : 0, `fparamsPtr` is correctly filled,
205* >0, `srcSize` is too small, result is expected `srcSize`,
206* or an error code, which can be tested using ZSTD_isError() */
207size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, size_t srcSize)
208{
209 const BYTE *ip = (const BYTE *)src;
210
211 if (srcSize < ZSTD_frameHeaderSize_prefix)
212 return ZSTD_frameHeaderSize_prefix;
213 if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) {
214 if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
215 if (srcSize < ZSTD_skippableHeaderSize)
216 return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
217 memset(fparamsPtr, 0, sizeof(*fparamsPtr));
218 fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4);
219 fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
220 return 0;
221 }
222 return ERROR(prefix_unknown);
223 }
224
225 /* ensure there is enough `srcSize` to fully read/decode frame header */
226 {
227 size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
228 if (srcSize < fhsize)
229 return fhsize;
230 }
231
232 {
233 BYTE const fhdByte = ip[4];
234 size_t pos = 5;
235 U32 const dictIDSizeCode = fhdByte & 3;
236 U32 const checksumFlag = (fhdByte >> 2) & 1;
237 U32 const singleSegment = (fhdByte >> 5) & 1;
238 U32 const fcsID = fhdByte >> 6;
239 U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
240 U32 windowSize = 0;
241 U32 dictID = 0;
242 U64 frameContentSize = 0;
243 if ((fhdByte & 0x08) != 0)
244 return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
245 if (!singleSegment) {
246 BYTE const wlByte = ip[pos++];
247 U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
248 if (windowLog > ZSTD_WINDOWLOG_MAX)
249 return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
250 windowSize = (1U << windowLog);
251 windowSize += (windowSize >> 3) * (wlByte & 7);
252 }
253
254 switch (dictIDSizeCode) {
255 default: /* impossible */
256 case 0: break;
257 case 1:
258 dictID = ip[pos];
259 pos++;
260 break;
261 case 2:
262 dictID = ZSTD_readLE16(ip + pos);
263 pos += 2;
264 break;
265 case 3:
266 dictID = ZSTD_readLE32(ip + pos);
267 pos += 4;
268 break;
269 }
270 switch (fcsID) {
271 default: /* impossible */
272 case 0:
273 if (singleSegment)
274 frameContentSize = ip[pos];
275 break;
276 case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break;
277 case 2: frameContentSize = ZSTD_readLE32(ip + pos); break;
278 case 3: frameContentSize = ZSTD_readLE64(ip + pos); break;
279 }
280 if (!windowSize)
281 windowSize = (U32)frameContentSize;
282 if (windowSize > windowSizeMax)
283 return ERROR(frameParameter_windowTooLarge);
284 fparamsPtr->frameContentSize = frameContentSize;
285 fparamsPtr->windowSize = windowSize;
286 fparamsPtr->dictID = dictID;
287 fparamsPtr->checksumFlag = checksumFlag;
288 }
289 return 0;
290}
291
292/** ZSTD_getFrameContentSize() :
293* compatible with legacy mode
294* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
295* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
296* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
297unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
298{
299 {
300 ZSTD_frameParams fParams;
301 if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0)
302 return ZSTD_CONTENTSIZE_ERROR;
303 if (fParams.windowSize == 0) {
304 /* Either skippable or empty frame, size == 0 either way */
305 return 0;
306 } else if (fParams.frameContentSize != 0) {
307 return fParams.frameContentSize;
308 } else {
309 return ZSTD_CONTENTSIZE_UNKNOWN;
310 }
311 }
312}
313
314/** ZSTD_findDecompressedSize() :
315 * compatible with legacy mode
316 * `srcSize` must be the exact length of some number of ZSTD compressed and/or
317 * skippable frames
318 * @return : decompressed size of the frames contained */
319unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize)
320{
321 {
322 unsigned long long totalDstSize = 0;
323 while (srcSize >= ZSTD_frameHeaderSize_prefix) {
324 const U32 magicNumber = ZSTD_readLE32(src);
325
326 if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
327 size_t skippableSize;
328 if (srcSize < ZSTD_skippableHeaderSize)
329 return ERROR(srcSize_wrong);
330 skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
331 if (srcSize < skippableSize) {
332 return ZSTD_CONTENTSIZE_ERROR;
333 }
334
335 src = (const BYTE *)src + skippableSize;
336 srcSize -= skippableSize;
337 continue;
338 }
339
340 {
341 unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
342 if (ret >= ZSTD_CONTENTSIZE_ERROR)
343 return ret;
344
345 /* check for overflow */
346 if (totalDstSize + ret < totalDstSize)
347 return ZSTD_CONTENTSIZE_ERROR;
348 totalDstSize += ret;
349 }
350 {
351 size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
352 if (ZSTD_isError(frameSrcSize)) {
353 return ZSTD_CONTENTSIZE_ERROR;
354 }
355
356 src = (const BYTE *)src + frameSrcSize;
357 srcSize -= frameSrcSize;
358 }
359 }
360
361 if (srcSize) {
362 return ZSTD_CONTENTSIZE_ERROR;
363 }
364
365 return totalDstSize;
366 }
367}
368
369/** ZSTD_decodeFrameHeader() :
370* `headerSize` must be the size provided by ZSTD_frameHeaderSize().
371* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
372static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize)
373{
374 size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
375 if (ZSTD_isError(result))
376 return result; /* invalid header */
377 if (result > 0)
378 return ERROR(srcSize_wrong); /* headerSize too small */
379 if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
380 return ERROR(dictionary_wrong);
381 if (dctx->fParams.checksumFlag)
382 xxh64_reset(&dctx->xxhState, 0);
383 return 0;
384}
385
386typedef struct {
387 blockType_e blockType;
388 U32 lastBlock;
389 U32 origSize;
390} blockProperties_t;
391
392/*! ZSTD_getcBlockSize() :
393* Provides the size of compressed block from block header `src` */
394size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr)
395{
396 if (srcSize < ZSTD_blockHeaderSize)
397 return ERROR(srcSize_wrong);
398 {
399 U32 const cBlockHeader = ZSTD_readLE24(src);
400 U32 const cSize = cBlockHeader >> 3;
401 bpPtr->lastBlock = cBlockHeader & 1;
402 bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
403 bpPtr->origSize = cSize; /* only useful for RLE */
404 if (bpPtr->blockType == bt_rle)
405 return 1;
406 if (bpPtr->blockType == bt_reserved)
407 return ERROR(corruption_detected);
408 return cSize;
409 }
410}
411
412static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
413{
414 if (srcSize > dstCapacity)
415 return ERROR(dstSize_tooSmall);
416 memcpy(dst, src, srcSize);
417 return srcSize;
418}
419
420static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize)
421{
422 if (srcSize != 1)
423 return ERROR(srcSize_wrong);
424 if (regenSize > dstCapacity)
425 return ERROR(dstSize_tooSmall);
426 memset(dst, *(const BYTE *)src, regenSize);
427 return regenSize;
428}
429
430/*! ZSTD_decodeLiteralsBlock() :
431 @return : nb of bytes read from src (< srcSize ) */
432size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
433{
434 if (srcSize < MIN_CBLOCK_SIZE)
435 return ERROR(corruption_detected);
436
437 {
438 const BYTE *const istart = (const BYTE *)src;
439 symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
440
441 switch (litEncType) {
442 case set_repeat:
443 if (dctx->litEntropy == 0)
444 return ERROR(dictionary_corrupted);
445 /* fall-through */
446 case set_compressed:
447 if (srcSize < 5)
448 return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
449 {
450 size_t lhSize, litSize, litCSize;
451 U32 singleStream = 0;
452 U32 const lhlCode = (istart[0] >> 2) & 3;
453 U32 const lhc = ZSTD_readLE32(istart);
454 switch (lhlCode) {
455 case 0:
456 case 1:
457 default: /* note : default is impossible, since lhlCode into [0..3] */
458 /* 2 - 2 - 10 - 10 */
459 singleStream = !lhlCode;
460 lhSize = 3;
461 litSize = (lhc >> 4) & 0x3FF;
462 litCSize = (lhc >> 14) & 0x3FF;
463 break;
464 case 2:
465 /* 2 - 2 - 14 - 14 */
466 lhSize = 4;
467 litSize = (lhc >> 4) & 0x3FFF;
468 litCSize = lhc >> 18;
469 break;
470 case 3:
471 /* 2 - 2 - 18 - 18 */
472 lhSize = 5;
473 litSize = (lhc >> 4) & 0x3FFFF;
474 litCSize = (lhc >> 22) + (istart[4] << 10);
475 break;
476 }
477 if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
478 return ERROR(corruption_detected);
479 if (litCSize + lhSize > srcSize)
480 return ERROR(corruption_detected);
481
482 if (HUF_isError(
483 (litEncType == set_repeat)
484 ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)
485 : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr))
486 : (singleStream
487 ? HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize,
488 dctx->entropy.workspace, sizeof(dctx->entropy.workspace))
489 : HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize,
490 dctx->entropy.workspace, sizeof(dctx->entropy.workspace)))))
491 return ERROR(corruption_detected);
492
493 dctx->litPtr = dctx->litBuffer;
494 dctx->litSize = litSize;
495 dctx->litEntropy = 1;
496 if (litEncType == set_compressed)
497 dctx->HUFptr = dctx->entropy.hufTable;
498 memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
499 return litCSize + lhSize;
500 }
501
502 case set_basic: {
503 size_t litSize, lhSize;
504 U32 const lhlCode = ((istart[0]) >> 2) & 3;
505 switch (lhlCode) {
506 case 0:
507 case 2:
508 default: /* note : default is impossible, since lhlCode into [0..3] */
509 lhSize = 1;
510 litSize = istart[0] >> 3;
511 break;
512 case 1:
513 lhSize = 2;
514 litSize = ZSTD_readLE16(istart) >> 4;
515 break;
516 case 3:
517 lhSize = 3;
518 litSize = ZSTD_readLE24(istart) >> 4;
519 break;
520 }
521
522 if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
523 if (litSize + lhSize > srcSize)
524 return ERROR(corruption_detected);
525 memcpy(dctx->litBuffer, istart + lhSize, litSize);
526 dctx->litPtr = dctx->litBuffer;
527 dctx->litSize = litSize;
528 memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
529 return lhSize + litSize;
530 }
531 /* direct reference into compressed stream */
532 dctx->litPtr = istart + lhSize;
533 dctx->litSize = litSize;
534 return lhSize + litSize;
535 }
536
537 case set_rle: {
538 U32 const lhlCode = ((istart[0]) >> 2) & 3;
539 size_t litSize, lhSize;
540 switch (lhlCode) {
541 case 0:
542 case 2:
543 default: /* note : default is impossible, since lhlCode into [0..3] */
544 lhSize = 1;
545 litSize = istart[0] >> 3;
546 break;
547 case 1:
548 lhSize = 2;
549 litSize = ZSTD_readLE16(istart) >> 4;
550 break;
551 case 3:
552 lhSize = 3;
553 litSize = ZSTD_readLE24(istart) >> 4;
554 if (srcSize < 4)
555 return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
556 break;
557 }
558 if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
559 return ERROR(corruption_detected);
560 memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
561 dctx->litPtr = dctx->litBuffer;
562 dctx->litSize = litSize;
563 return lhSize + 1;
564 }
565 default:
566 return ERROR(corruption_detected); /* impossible */
567 }
568 }
569}
570
571typedef union {
572 FSE_decode_t realData;
573 U32 alignedBy4;
574} FSE_decode_t4;
575
576static const FSE_decode_t4 LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = {
577 {{LL_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
578 {{0, 0, 4}}, /* 0 : base, symbol, bits */
579 {{16, 0, 4}},
580 {{32, 1, 5}},
581 {{0, 3, 5}},
582 {{0, 4, 5}},
583 {{0, 6, 5}},
584 {{0, 7, 5}},
585 {{0, 9, 5}},
586 {{0, 10, 5}},
587 {{0, 12, 5}},
588 {{0, 14, 6}},
589 {{0, 16, 5}},
590 {{0, 18, 5}},
591 {{0, 19, 5}},
592 {{0, 21, 5}},
593 {{0, 22, 5}},
594 {{0, 24, 5}},
595 {{32, 25, 5}},
596 {{0, 26, 5}},
597 {{0, 27, 6}},
598 {{0, 29, 6}},
599 {{0, 31, 6}},
600 {{32, 0, 4}},
601 {{0, 1, 4}},
602 {{0, 2, 5}},
603 {{32, 4, 5}},
604 {{0, 5, 5}},
605 {{32, 7, 5}},
606 {{0, 8, 5}},
607 {{32, 10, 5}},
608 {{0, 11, 5}},
609 {{0, 13, 6}},
610 {{32, 16, 5}},
611 {{0, 17, 5}},
612 {{32, 19, 5}},
613 {{0, 20, 5}},
614 {{32, 22, 5}},
615 {{0, 23, 5}},
616 {{0, 25, 4}},
617 {{16, 25, 4}},
618 {{32, 26, 5}},
619 {{0, 28, 6}},
620 {{0, 30, 6}},
621 {{48, 0, 4}},
622 {{16, 1, 4}},
623 {{32, 2, 5}},
624 {{32, 3, 5}},
625 {{32, 5, 5}},
626 {{32, 6, 5}},
627 {{32, 8, 5}},
628 {{32, 9, 5}},
629 {{32, 11, 5}},
630 {{32, 12, 5}},
631 {{0, 15, 6}},
632 {{32, 17, 5}},
633 {{32, 18, 5}},
634 {{32, 20, 5}},
635 {{32, 21, 5}},
636 {{32, 23, 5}},
637 {{32, 24, 5}},
638 {{0, 35, 6}},
639 {{0, 34, 6}},
640 {{0, 33, 6}},
641 {{0, 32, 6}},
642}; /* LL_defaultDTable */
643
644static const FSE_decode_t4 ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = {
645 {{ML_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
646 {{0, 0, 6}}, /* 0 : base, symbol, bits */
647 {{0, 1, 4}},
648 {{32, 2, 5}},
649 {{0, 3, 5}},
650 {{0, 5, 5}},
651 {{0, 6, 5}},
652 {{0, 8, 5}},
653 {{0, 10, 6}},
654 {{0, 13, 6}},
655 {{0, 16, 6}},
656 {{0, 19, 6}},
657 {{0, 22, 6}},
658 {{0, 25, 6}},
659 {{0, 28, 6}},
660 {{0, 31, 6}},
661 {{0, 33, 6}},
662 {{0, 35, 6}},
663 {{0, 37, 6}},
664 {{0, 39, 6}},
665 {{0, 41, 6}},
666 {{0, 43, 6}},
667 {{0, 45, 6}},
668 {{16, 1, 4}},
669 {{0, 2, 4}},
670 {{32, 3, 5}},
671 {{0, 4, 5}},
672 {{32, 6, 5}},
673 {{0, 7, 5}},
674 {{0, 9, 6}},
675 {{0, 12, 6}},
676 {{0, 15, 6}},
677 {{0, 18, 6}},
678 {{0, 21, 6}},
679 {{0, 24, 6}},
680 {{0, 27, 6}},
681 {{0, 30, 6}},
682 {{0, 32, 6}},
683 {{0, 34, 6}},
684 {{0, 36, 6}},
685 {{0, 38, 6}},
686 {{0, 40, 6}},
687 {{0, 42, 6}},
688 {{0, 44, 6}},
689 {{32, 1, 4}},
690 {{48, 1, 4}},
691 {{16, 2, 4}},
692 {{32, 4, 5}},
693 {{32, 5, 5}},
694 {{32, 7, 5}},
695 {{32, 8, 5}},
696 {{0, 11, 6}},
697 {{0, 14, 6}},
698 {{0, 17, 6}},
699 {{0, 20, 6}},
700 {{0, 23, 6}},
701 {{0, 26, 6}},
702 {{0, 29, 6}},
703 {{0, 52, 6}},
704 {{0, 51, 6}},
705 {{0, 50, 6}},
706 {{0, 49, 6}},
707 {{0, 48, 6}},
708 {{0, 47, 6}},
709 {{0, 46, 6}},
710}; /* ML_defaultDTable */
711
712static const FSE_decode_t4 OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = {
713 {{OF_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
714 {{0, 0, 5}}, /* 0 : base, symbol, bits */
715 {{0, 6, 4}},
716 {{0, 9, 5}},
717 {{0, 15, 5}},
718 {{0, 21, 5}},
719 {{0, 3, 5}},
720 {{0, 7, 4}},
721 {{0, 12, 5}},
722 {{0, 18, 5}},
723 {{0, 23, 5}},
724 {{0, 5, 5}},
725 {{0, 8, 4}},
726 {{0, 14, 5}},
727 {{0, 20, 5}},
728 {{0, 2, 5}},
729 {{16, 7, 4}},
730 {{0, 11, 5}},
731 {{0, 17, 5}},
732 {{0, 22, 5}},
733 {{0, 4, 5}},
734 {{16, 8, 4}},
735 {{0, 13, 5}},
736 {{0, 19, 5}},
737 {{0, 1, 5}},
738 {{16, 6, 4}},
739 {{0, 10, 5}},
740 {{0, 16, 5}},
741 {{0, 28, 5}},
742 {{0, 27, 5}},
743 {{0, 26, 5}},
744 {{0, 25, 5}},
745 {{0, 24, 5}},
746}; /* OF_defaultDTable */
747
748/*! ZSTD_buildSeqTable() :
749 @return : nb bytes read from src,
750 or an error code if it fails, testable with ZSTD_isError()
751*/
752static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src,
753 size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable, void *workspace, size_t workspaceSize)
754{
755 const void *const tmpPtr = defaultTable; /* bypass strict aliasing */
756 switch (type) {
757 case set_rle:
758 if (!srcSize)
759 return ERROR(srcSize_wrong);
760 if ((*(const BYTE *)src) > max)
761 return ERROR(corruption_detected);
762 FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src);
763 *DTablePtr = DTableSpace;
764 return 1;
765 case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0;
766 case set_repeat:
767 if (!flagRepeatTable)
768 return ERROR(corruption_detected);
769 return 0;
770 default: /* impossible */
771 case set_compressed: {
772 U32 tableLog;
773 S16 *norm = (S16 *)workspace;
774 size_t const spaceUsed32 = ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2;
775
776 if ((spaceUsed32 << 2) > workspaceSize)
777 return ERROR(GENERIC);
778 workspace = (U32 *)workspace + spaceUsed32;
779 workspaceSize -= (spaceUsed32 << 2);
780 {
781 size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
782 if (FSE_isError(headerSize))
783 return ERROR(corruption_detected);
784 if (tableLog > maxLog)
785 return ERROR(corruption_detected);
786 FSE_buildDTable_wksp(DTableSpace, norm, max, tableLog, workspace, workspaceSize);
787 *DTablePtr = DTableSpace;
788 return headerSize;
789 }
790 }
791 }
792}
793
794size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize)
795{
796 const BYTE *const istart = (const BYTE *const)src;
797 const BYTE *const iend = istart + srcSize;
798 const BYTE *ip = istart;
799
800 /* check */
801 if (srcSize < MIN_SEQUENCES_SIZE)
802 return ERROR(srcSize_wrong);
803
804 /* SeqHead */
805 {
806 int nbSeq = *ip++;
807 if (!nbSeq) {
808 *nbSeqPtr = 0;
809 return 1;
810 }
811 if (nbSeq > 0x7F) {
812 if (nbSeq == 0xFF) {
813 if (ip + 2 > iend)
814 return ERROR(srcSize_wrong);
815 nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2;
816 } else {
817 if (ip >= iend)
818 return ERROR(srcSize_wrong);
819 nbSeq = ((nbSeq - 0x80) << 8) + *ip++;
820 }
821 }
822 *nbSeqPtr = nbSeq;
823 }
824
825 /* FSE table descriptors */
826 if (ip + 4 > iend)
827 return ERROR(srcSize_wrong); /* minimum possible size */
828 {
829 symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
830 symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
831 symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
832 ip++;
833
834 /* Build DTables */
835 {
836 size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip,
837 LL_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
838 if (ZSTD_isError(llhSize))
839 return ERROR(corruption_detected);
840 ip += llhSize;
841 }
842 {
843 size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip,
844 OF_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
845 if (ZSTD_isError(ofhSize))
846 return ERROR(corruption_detected);
847 ip += ofhSize;
848 }
849 {
850 size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip,
851 ML_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
852 if (ZSTD_isError(mlhSize))
853 return ERROR(corruption_detected);
854 ip += mlhSize;
855 }
856 }
857
858 return ip - istart;
859}
860
861typedef struct {
862 size_t litLength;
863 size_t matchLength;
864 size_t offset;
865 const BYTE *match;
866} seq_t;
867
868typedef struct {
869 BIT_DStream_t DStream;
870 FSE_DState_t stateLL;
871 FSE_DState_t stateOffb;
872 FSE_DState_t stateML;
873 size_t prevOffset[ZSTD_REP_NUM];
874 const BYTE *base;
875 size_t pos;
876 uPtrDiff gotoDict;
877} seqState_t;
878
879FORCE_NOINLINE
880size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
881 const BYTE *const vBase, const BYTE *const dictEnd)
882{
883 BYTE *const oLitEnd = op + sequence.litLength;
884 size_t const sequenceLength = sequence.litLength + sequence.matchLength;
885 BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
886 BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
887 const BYTE *const iLitEnd = *litPtr + sequence.litLength;
888 const BYTE *match = oLitEnd - sequence.offset;
889
890 /* check */
891 if (oMatchEnd > oend)
892 return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
893 if (iLitEnd > litLimit)
894 return ERROR(corruption_detected); /* over-read beyond lit buffer */
895 if (oLitEnd <= oend_w)
896 return ERROR(GENERIC); /* Precondition */
897
898 /* copy literals */
899 if (op < oend_w) {
900 ZSTD_wildcopy(op, *litPtr, oend_w - op);
901 *litPtr += oend_w - op;
902 op = oend_w;
903 }
904 while (op < oLitEnd)
905 *op++ = *(*litPtr)++;
906
907 /* copy Match */
908 if (sequence.offset > (size_t)(oLitEnd - base)) {
909 /* offset beyond prefix */
910 if (sequence.offset > (size_t)(oLitEnd - vBase))
911 return ERROR(corruption_detected);
912 match = dictEnd - (base - match);
913 if (match + sequence.matchLength <= dictEnd) {
914 memmove(oLitEnd, match, sequence.matchLength);
915 return sequenceLength;
916 }
917 /* span extDict & currPrefixSegment */
918 {
919 size_t const length1 = dictEnd - match;
920 memmove(oLitEnd, match, length1);
921 op = oLitEnd + length1;
922 sequence.matchLength -= length1;
923 match = base;
924 }
925 }
926 while (op < oMatchEnd)
927 *op++ = *match++;
928 return sequenceLength;
929}
930
931static seq_t ZSTD_decodeSequence(seqState_t *seqState)
932{
933 seq_t seq;
934
935 U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
936 U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
937 U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */
938
939 U32 const llBits = LL_bits[llCode];
940 U32 const mlBits = ML_bits[mlCode];
941 U32 const ofBits = ofCode;
942 U32 const totalBits = llBits + mlBits + ofBits;
943
944 static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
945 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
946
947 static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
948 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
949 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
950
951 static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
952 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
953 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
954
955 /* sequence */
956 {
957 size_t offset;
958 if (!ofCode)
959 offset = 0;
960 else {
961 offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
962 if (ZSTD_32bits())
963 BIT_reloadDStream(&seqState->DStream);
964 }
965
966 if (ofCode <= 1) {
967 offset += (llCode == 0);
968 if (offset) {
969 size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
970 temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
971 if (offset != 1)
972 seqState->prevOffset[2] = seqState->prevOffset[1];
973 seqState->prevOffset[1] = seqState->prevOffset[0];
974 seqState->prevOffset[0] = offset = temp;
975 } else {
976 offset = seqState->prevOffset[0];
977 }
978 } else {
979 seqState->prevOffset[2] = seqState->prevOffset[1];
980 seqState->prevOffset[1] = seqState->prevOffset[0];
981 seqState->prevOffset[0] = offset;
982 }
983 seq.offset = offset;
984 }
985
986 seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
987 if (ZSTD_32bits() && (mlBits + llBits > 24))
988 BIT_reloadDStream(&seqState->DStream);
989
990 seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
991 if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
992 BIT_reloadDStream(&seqState->DStream);
993
994 /* ANS state update */
995 FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
996 FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
997 if (ZSTD_32bits())
998 BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
999 FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
1000
1001 seq.match = NULL;
1002
1003 return seq;
1004}
1005
1006FORCE_INLINE
1007size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
1008 const BYTE *const vBase, const BYTE *const dictEnd)
1009{
1010 BYTE *const oLitEnd = op + sequence.litLength;
1011 size_t const sequenceLength = sequence.litLength + sequence.matchLength;
1012 BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
1013 BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
1014 const BYTE *const iLitEnd = *litPtr + sequence.litLength;
1015 const BYTE *match = oLitEnd - sequence.offset;
1016
1017 /* check */
1018 if (oMatchEnd > oend)
1019 return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
1020 if (iLitEnd > litLimit)
1021 return ERROR(corruption_detected); /* over-read beyond lit buffer */
1022 if (oLitEnd > oend_w)
1023 return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
1024
1025 /* copy Literals */
1026 ZSTD_copy8(op, *litPtr);
1027 if (sequence.litLength > 8)
1028 ZSTD_wildcopy(op + 8, (*litPtr) + 8,
1029 sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
1030 op = oLitEnd;
1031 *litPtr = iLitEnd; /* update for next sequence */
1032
1033 /* copy Match */
1034 if (sequence.offset > (size_t)(oLitEnd - base)) {
1035 /* offset beyond prefix */
1036 if (sequence.offset > (size_t)(oLitEnd - vBase))
1037 return ERROR(corruption_detected);
1038 match = dictEnd + (match - base);
1039 if (match + sequence.matchLength <= dictEnd) {
1040 memmove(oLitEnd, match, sequence.matchLength);
1041 return sequenceLength;
1042 }
1043 /* span extDict & currPrefixSegment */
1044 {
1045 size_t const length1 = dictEnd - match;
1046 memmove(oLitEnd, match, length1);
1047 op = oLitEnd + length1;
1048 sequence.matchLength -= length1;
1049 match = base;
1050 if (op > oend_w || sequence.matchLength < MINMATCH) {
1051 U32 i;
1052 for (i = 0; i < sequence.matchLength; ++i)
1053 op[i] = match[i];
1054 return sequenceLength;
1055 }
1056 }
1057 }
1058 /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
1059
1060 /* match within prefix */
1061 if (sequence.offset < 8) {
1062 /* close range match, overlap */
1063 static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
1064 static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
1065 int const sub2 = dec64table[sequence.offset];
1066 op[0] = match[0];
1067 op[1] = match[1];
1068 op[2] = match[2];
1069 op[3] = match[3];
1070 match += dec32table[sequence.offset];
1071 ZSTD_copy4(op + 4, match);
1072 match -= sub2;
1073 } else {
1074 ZSTD_copy8(op, match);
1075 }
1076 op += 8;
1077 match += 8;
1078
1079 if (oMatchEnd > oend - (16 - MINMATCH)) {
1080 if (op < oend_w) {
1081 ZSTD_wildcopy(op, match, oend_w - op);
1082 match += oend_w - op;
1083 op = oend_w;
1084 }
1085 while (op < oMatchEnd)
1086 *op++ = *match++;
1087 } else {
1088 ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
1089 }
1090 return sequenceLength;
1091}
1092
1093static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
1094{
1095 const BYTE *ip = (const BYTE *)seqStart;
1096 const BYTE *const iend = ip + seqSize;
1097 BYTE *const ostart = (BYTE * const)dst;
1098 BYTE *const oend = ostart + maxDstSize;
1099 BYTE *op = ostart;
1100 const BYTE *litPtr = dctx->litPtr;
1101 const BYTE *const litEnd = litPtr + dctx->litSize;
1102 const BYTE *const base = (const BYTE *)(dctx->base);
1103 const BYTE *const vBase = (const BYTE *)(dctx->vBase);
1104 const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
1105 int nbSeq;
1106
1107 /* Build Decoding Tables */
1108 {
1109 size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
1110 if (ZSTD_isError(seqHSize))
1111 return seqHSize;
1112 ip += seqHSize;
1113 }
1114
1115 /* Regen sequences */
1116 if (nbSeq) {
1117 seqState_t seqState;
1118 dctx->fseEntropy = 1;
1119 {
1120 U32 i;
1121 for (i = 0; i < ZSTD_REP_NUM; i++)
1122 seqState.prevOffset[i] = dctx->entropy.rep[i];
1123 }
1124 CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
1125 FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
1126 FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
1127 FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
1128
1129 for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) {
1130 nbSeq--;
1131 {
1132 seq_t const sequence = ZSTD_decodeSequence(&seqState);
1133 size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
1134 if (ZSTD_isError(oneSeqSize))
1135 return oneSeqSize;
1136 op += oneSeqSize;
1137 }
1138 }
1139
1140 /* check if reached exact end */
1141 if (nbSeq)
1142 return ERROR(corruption_detected);
1143 /* save reps for next block */
1144 {
1145 U32 i;
1146 for (i = 0; i < ZSTD_REP_NUM; i++)
1147 dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
1148 }
1149 }
1150
1151 /* last literal segment */
1152 {
1153 size_t const lastLLSize = litEnd - litPtr;
1154 if (lastLLSize > (size_t)(oend - op))
1155 return ERROR(dstSize_tooSmall);
1156 memcpy(op, litPtr, lastLLSize);
1157 op += lastLLSize;
1158 }
1159
1160 return op - ostart;
1161}
1162
1163FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets)
1164{
1165 seq_t seq;
1166
1167 U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
1168 U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
1169 U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */
1170
1171 U32 const llBits = LL_bits[llCode];
1172 U32 const mlBits = ML_bits[mlCode];
1173 U32 const ofBits = ofCode;
1174 U32 const totalBits = llBits + mlBits + ofBits;
1175
1176 static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
1177 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
1178
1179 static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
1180 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
1181 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
1182
1183 static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
1184 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
1185 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
1186
1187 /* sequence */
1188 {
1189 size_t offset;
1190 if (!ofCode)
1191 offset = 0;
1192 else {
1193 if (longOffsets) {
1194 int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
1195 offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
1196 if (ZSTD_32bits() || extraBits)
1197 BIT_reloadDStream(&seqState->DStream);
1198 if (extraBits)
1199 offset += BIT_readBitsFast(&seqState->DStream, extraBits);
1200 } else {
1201 offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
1202 if (ZSTD_32bits())
1203 BIT_reloadDStream(&seqState->DStream);
1204 }
1205 }
1206
1207 if (ofCode <= 1) {
1208 offset += (llCode == 0);
1209 if (offset) {
1210 size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
1211 temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
1212 if (offset != 1)
1213 seqState->prevOffset[2] = seqState->prevOffset[1];
1214 seqState->prevOffset[1] = seqState->prevOffset[0];
1215 seqState->prevOffset[0] = offset = temp;
1216 } else {
1217 offset = seqState->prevOffset[0];
1218 }
1219 } else {
1220 seqState->prevOffset[2] = seqState->prevOffset[1];
1221 seqState->prevOffset[1] = seqState->prevOffset[0];
1222 seqState->prevOffset[0] = offset;
1223 }
1224 seq.offset = offset;
1225 }
1226
1227 seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
1228 if (ZSTD_32bits() && (mlBits + llBits > 24))
1229 BIT_reloadDStream(&seqState->DStream);
1230
1231 seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
1232 if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
1233 BIT_reloadDStream(&seqState->DStream);
1234
1235 {
1236 size_t const pos = seqState->pos + seq.litLength;
1237 seq.match = seqState->base + pos - seq.offset; /* single memory segment */
1238 if (seq.offset > pos)
1239 seq.match += seqState->gotoDict; /* separate memory segment */
1240 seqState->pos = pos + seq.matchLength;
1241 }
1242
1243 /* ANS state update */
1244 FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
1245 FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
1246 if (ZSTD_32bits())
1247 BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
1248 FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
1249
1250 return seq;
1251}
1252
1253static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize)
1254{
1255 if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
1256 return ZSTD_decodeSequenceLong_generic(seqState, 1);
1257 } else {
1258 return ZSTD_decodeSequenceLong_generic(seqState, 0);
1259 }
1260}
1261
1262FORCE_INLINE
1263size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
1264 const BYTE *const vBase, const BYTE *const dictEnd)
1265{
1266 BYTE *const oLitEnd = op + sequence.litLength;
1267 size_t const sequenceLength = sequence.litLength + sequence.matchLength;
1268 BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
1269 BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
1270 const BYTE *const iLitEnd = *litPtr + sequence.litLength;
1271 const BYTE *match = sequence.match;
1272
1273 /* check */
1274 if (oMatchEnd > oend)
1275 return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
1276 if (iLitEnd > litLimit)
1277 return ERROR(corruption_detected); /* over-read beyond lit buffer */
1278 if (oLitEnd > oend_w)
1279 return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
1280
1281 /* copy Literals */
1282 ZSTD_copy8(op, *litPtr);
1283 if (sequence.litLength > 8)
1284 ZSTD_wildcopy(op + 8, (*litPtr) + 8,
1285 sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
1286 op = oLitEnd;
1287 *litPtr = iLitEnd; /* update for next sequence */
1288
1289 /* copy Match */
1290 if (sequence.offset > (size_t)(oLitEnd - base)) {
1291 /* offset beyond prefix */
1292 if (sequence.offset > (size_t)(oLitEnd - vBase))
1293 return ERROR(corruption_detected);
1294 if (match + sequence.matchLength <= dictEnd) {
1295 memmove(oLitEnd, match, sequence.matchLength);
1296 return sequenceLength;
1297 }
1298 /* span extDict & currPrefixSegment */
1299 {
1300 size_t const length1 = dictEnd - match;
1301 memmove(oLitEnd, match, length1);
1302 op = oLitEnd + length1;
1303 sequence.matchLength -= length1;
1304 match = base;
1305 if (op > oend_w || sequence.matchLength < MINMATCH) {
1306 U32 i;
1307 for (i = 0; i < sequence.matchLength; ++i)
1308 op[i] = match[i];
1309 return sequenceLength;
1310 }
1311 }
1312 }
1313 /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
1314
1315 /* match within prefix */
1316 if (sequence.offset < 8) {
1317 /* close range match, overlap */
1318 static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
1319 static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
1320 int const sub2 = dec64table[sequence.offset];
1321 op[0] = match[0];
1322 op[1] = match[1];
1323 op[2] = match[2];
1324 op[3] = match[3];
1325 match += dec32table[sequence.offset];
1326 ZSTD_copy4(op + 4, match);
1327 match -= sub2;
1328 } else {
1329 ZSTD_copy8(op, match);
1330 }
1331 op += 8;
1332 match += 8;
1333
1334 if (oMatchEnd > oend - (16 - MINMATCH)) {
1335 if (op < oend_w) {
1336 ZSTD_wildcopy(op, match, oend_w - op);
1337 match += oend_w - op;
1338 op = oend_w;
1339 }
1340 while (op < oMatchEnd)
1341 *op++ = *match++;
1342 } else {
1343 ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
1344 }
1345 return sequenceLength;
1346}
1347
1348static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
1349{
1350 const BYTE *ip = (const BYTE *)seqStart;
1351 const BYTE *const iend = ip + seqSize;
1352 BYTE *const ostart = (BYTE * const)dst;
1353 BYTE *const oend = ostart + maxDstSize;
1354 BYTE *op = ostart;
1355 const BYTE *litPtr = dctx->litPtr;
1356 const BYTE *const litEnd = litPtr + dctx->litSize;
1357 const BYTE *const base = (const BYTE *)(dctx->base);
1358 const BYTE *const vBase = (const BYTE *)(dctx->vBase);
1359 const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
1360 unsigned const windowSize = dctx->fParams.windowSize;
1361 int nbSeq;
1362
1363 /* Build Decoding Tables */
1364 {
1365 size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
1366 if (ZSTD_isError(seqHSize))
1367 return seqHSize;
1368 ip += seqHSize;
1369 }
1370
1371 /* Regen sequences */
1372 if (nbSeq) {
1373#define STORED_SEQS 4
1374#define STOSEQ_MASK (STORED_SEQS - 1)
1375#define ADVANCED_SEQS 4
1376 seq_t *sequences = (seq_t *)dctx->entropy.workspace;
1377 int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
1378 seqState_t seqState;
1379 int seqNb;
1380 ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.workspace) >= sizeof(seq_t) * STORED_SEQS);
1381 dctx->fseEntropy = 1;
1382 {
1383 U32 i;
1384 for (i = 0; i < ZSTD_REP_NUM; i++)
1385 seqState.prevOffset[i] = dctx->entropy.rep[i];
1386 }
1387 seqState.base = base;
1388 seqState.pos = (size_t)(op - base);
1389 seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
1390 CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
1391 FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
1392 FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
1393 FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
1394
1395 /* prepare in advance */
1396 for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) {
1397 sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
1398 }
1399 if (seqNb < seqAdvance)
1400 return ERROR(corruption_detected);
1401
1402 /* decode and decompress */
1403 for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) {
1404 seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
1405 size_t const oneSeqSize =
1406 ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
1407 if (ZSTD_isError(oneSeqSize))
1408 return oneSeqSize;
1409 ZSTD_PREFETCH(sequence.match);
1410 sequences[seqNb & STOSEQ_MASK] = sequence;
1411 op += oneSeqSize;
1412 }
1413 if (seqNb < nbSeq)
1414 return ERROR(corruption_detected);
1415
1416 /* finish queue */
1417 seqNb -= seqAdvance;
1418 for (; seqNb < nbSeq; seqNb++) {
1419 size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
1420 if (ZSTD_isError(oneSeqSize))
1421 return oneSeqSize;
1422 op += oneSeqSize;
1423 }
1424
1425 /* save reps for next block */
1426 {
1427 U32 i;
1428 for (i = 0; i < ZSTD_REP_NUM; i++)
1429 dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
1430 }
1431 }
1432
1433 /* last literal segment */
1434 {
1435 size_t const lastLLSize = litEnd - litPtr;
1436 if (lastLLSize > (size_t)(oend - op))
1437 return ERROR(dstSize_tooSmall);
1438 memcpy(op, litPtr, lastLLSize);
1439 op += lastLLSize;
1440 }
1441
1442 return op - ostart;
1443}
1444
1445static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
1446{ /* blockType == blockCompressed */
1447 const BYTE *ip = (const BYTE *)src;
1448
1449 if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX)
1450 return ERROR(srcSize_wrong);
1451
1452 /* Decode literals section */
1453 {
1454 size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
1455 if (ZSTD_isError(litCSize))
1456 return litCSize;
1457 ip += litCSize;
1458 srcSize -= litCSize;
1459 }
1460 if (sizeof(size_t) > 4) /* do not enable prefetching on 32-bits x86, as it's performance detrimental */
1461 /* likely because of register pressure */
1462 /* if that's the correct cause, then 32-bits ARM should be affected differently */
1463 /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */
1464 if (dctx->fParams.windowSize > (1 << 23))
1465 return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
1466 return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
1467}
1468
1469static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst)
1470{
1471 if (dst != dctx->previousDstEnd) { /* not contiguous */
1472 dctx->dictEnd = dctx->previousDstEnd;
1473 dctx->vBase = (const char *)dst - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
1474 dctx->base = dst;
1475 dctx->previousDstEnd = dst;
1476 }
1477}
1478
1479size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
1480{
1481 size_t dSize;
1482 ZSTD_checkContinuity(dctx, dst);
1483 dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
1484 dctx->previousDstEnd = (char *)dst + dSize;
1485 return dSize;
1486}
1487
1488/** ZSTD_insertBlock() :
1489 insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
1490size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize)
1491{
1492 ZSTD_checkContinuity(dctx, blockStart);
1493 dctx->previousDstEnd = (const char *)blockStart + blockSize;
1494 return blockSize;
1495}
1496
1497size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length)
1498{
1499 if (length > dstCapacity)
1500 return ERROR(dstSize_tooSmall);
1501 memset(dst, byte, length);
1502 return length;
1503}
1504
1505/** ZSTD_findFrameCompressedSize() :
1506 * compatible with legacy mode
1507 * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
1508 * `srcSize` must be at least as large as the frame contained
1509 * @return : the compressed size of the frame starting at `src` */
1510size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
1511{
1512 if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
1513 return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4);
1514 } else {
1515 const BYTE *ip = (const BYTE *)src;
1516 const BYTE *const ipstart = ip;
1517 size_t remainingSize = srcSize;
1518 ZSTD_frameParams fParams;
1519
1520 size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
1521 if (ZSTD_isError(headerSize))
1522 return headerSize;
1523
1524 /* Frame Header */
1525 {
1526 size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
1527 if (ZSTD_isError(ret))
1528 return ret;
1529 if (ret > 0)
1530 return ERROR(srcSize_wrong);
1531 }
1532
1533 ip += headerSize;
1534 remainingSize -= headerSize;
1535
1536 /* Loop on each block */
1537 while (1) {
1538 blockProperties_t blockProperties;
1539 size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
1540 if (ZSTD_isError(cBlockSize))
1541 return cBlockSize;
1542
1543 if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
1544 return ERROR(srcSize_wrong);
1545
1546 ip += ZSTD_blockHeaderSize + cBlockSize;
1547 remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
1548
1549 if (blockProperties.lastBlock)
1550 break;
1551 }
1552
1553 if (fParams.checksumFlag) { /* Frame content checksum */
1554 if (remainingSize < 4)
1555 return ERROR(srcSize_wrong);
1556 ip += 4;
1557 remainingSize -= 4;
1558 }
1559
1560 return ip - ipstart;
1561 }
1562}
1563
1564/*! ZSTD_decompressFrame() :
1565* @dctx must be properly initialized */
1566static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr)
1567{
1568 const BYTE *ip = (const BYTE *)(*srcPtr);
1569 BYTE *const ostart = (BYTE * const)dst;
1570 BYTE *const oend = ostart + dstCapacity;
1571 BYTE *op = ostart;
1572 size_t remainingSize = *srcSizePtr;
1573
1574 /* check */
1575 if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize)
1576 return ERROR(srcSize_wrong);
1577
1578 /* Frame Header */
1579 {
1580 size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
1581 if (ZSTD_isError(frameHeaderSize))
1582 return frameHeaderSize;
1583 if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize)
1584 return ERROR(srcSize_wrong);
1585 CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize));
1586 ip += frameHeaderSize;
1587 remainingSize -= frameHeaderSize;
1588 }
1589
1590 /* Loop on each block */
1591 while (1) {
1592 size_t decodedSize;
1593 blockProperties_t blockProperties;
1594 size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
1595 if (ZSTD_isError(cBlockSize))
1596 return cBlockSize;
1597
1598 ip += ZSTD_blockHeaderSize;
1599 remainingSize -= ZSTD_blockHeaderSize;
1600 if (cBlockSize > remainingSize)
1601 return ERROR(srcSize_wrong);
1602
1603 switch (blockProperties.blockType) {
1604 case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break;
1605 case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break;
1606 case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break;
1607 case bt_reserved:
1608 default: return ERROR(corruption_detected);
1609 }
1610
1611 if (ZSTD_isError(decodedSize))
1612 return decodedSize;
1613 if (dctx->fParams.checksumFlag)
1614 xxh64_update(&dctx->xxhState, op, decodedSize);
1615 op += decodedSize;
1616 ip += cBlockSize;
1617 remainingSize -= cBlockSize;
1618 if (blockProperties.lastBlock)
1619 break;
1620 }
1621
1622 if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
1623 U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState);
1624 U32 checkRead;
1625 if (remainingSize < 4)
1626 return ERROR(checksum_wrong);
1627 checkRead = ZSTD_readLE32(ip);
1628 if (checkRead != checkCalc)
1629 return ERROR(checksum_wrong);
1630 ip += 4;
1631 remainingSize -= 4;
1632 }
1633
1634 /* Allow caller to get size read */
1635 *srcPtr = ip;
1636 *srcSizePtr = remainingSize;
1637 return op - ostart;
1638}
1639
1640static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict);
1641static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict);
1642
1643static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
1644 const ZSTD_DDict *ddict)
1645{
1646 void *const dststart = dst;
1647
1648 if (ddict) {
1649 if (dict) {
1650 /* programmer error, these two cases should be mutually exclusive */
1651 return ERROR(GENERIC);
1652 }
1653
1654 dict = ZSTD_DDictDictContent(ddict);
1655 dictSize = ZSTD_DDictDictSize(ddict);
1656 }
1657
1658 while (srcSize >= ZSTD_frameHeaderSize_prefix) {
1659 U32 magicNumber;
1660
1661 magicNumber = ZSTD_readLE32(src);
1662 if (magicNumber != ZSTD_MAGICNUMBER) {
1663 if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
1664 size_t skippableSize;
1665 if (srcSize < ZSTD_skippableHeaderSize)
1666 return ERROR(srcSize_wrong);
1667 skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
1668 if (srcSize < skippableSize) {
1669 return ERROR(srcSize_wrong);
1670 }
1671
1672 src = (const BYTE *)src + skippableSize;
1673 srcSize -= skippableSize;
1674 continue;
1675 } else {
1676 return ERROR(prefix_unknown);
1677 }
1678 }
1679
1680 if (ddict) {
1681 /* we were called from ZSTD_decompress_usingDDict */
1682 ZSTD_refDDict(dctx, ddict);
1683 } else {
1684 /* this will initialize correctly with no dict if dict == NULL, so
1685 * use this in all cases but ddict */
1686 CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
1687 }
1688 ZSTD_checkContinuity(dctx, dst);
1689
1690 {
1691 const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize);
1692 if (ZSTD_isError(res))
1693 return res;
1694 /* don't need to bounds check this, ZSTD_decompressFrame will have
1695 * already */
1696 dst = (BYTE *)dst + res;
1697 dstCapacity -= res;
1698 }
1699 }
1700
1701 if (srcSize)
1702 return ERROR(srcSize_wrong); /* input not entirely consumed */
1703
1704 return (BYTE *)dst - (BYTE *)dststart;
1705}
1706
1707size_t ZSTD_decompress_usingDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize)
1708{
1709 return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
1710}
1711
1712size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
1713{
1714 return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
1715}
1716
1717/*-**************************************
1718* Advanced Streaming Decompression API
1719* Bufferless and synchronous
1720****************************************/
1721size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; }
1722
1723ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx)
1724{
1725 switch (dctx->stage) {
1726 default: /* should not happen */
1727 case ZSTDds_getFrameHeaderSize:
1728 case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader;
1729 case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader;
1730 case ZSTDds_decompressBlock: return ZSTDnit_block;
1731 case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock;
1732 case ZSTDds_checkChecksum: return ZSTDnit_checksum;
1733 case ZSTDds_decodeSkippableHeader:
1734 case ZSTDds_skipFrame: return ZSTDnit_skippableFrame;
1735 }
1736}
1737
1738int ZSTD_isSkipFrame(ZSTD_DCtx *dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */
1739
1740/** ZSTD_decompressContinue() :
1741* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
1742* or an error code, which can be tested using ZSTD_isError() */
1743size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
1744{
1745 /* Sanity check */
1746 if (srcSize != dctx->expected)
1747 return ERROR(srcSize_wrong);
1748 if (dstCapacity)
1749 ZSTD_checkContinuity(dctx, dst);
1750
1751 switch (dctx->stage) {
1752 case ZSTDds_getFrameHeaderSize:
1753 if (srcSize != ZSTD_frameHeaderSize_prefix)
1754 return ERROR(srcSize_wrong); /* impossible */
1755 if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
1756 memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
1757 dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
1758 dctx->stage = ZSTDds_decodeSkippableHeader;
1759 return 0;
1760 }
1761 dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
1762 if (ZSTD_isError(dctx->headerSize))
1763 return dctx->headerSize;
1764 memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
1765 if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
1766 dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
1767 dctx->stage = ZSTDds_decodeFrameHeader;
1768 return 0;
1769 }
1770 dctx->expected = 0; /* not necessary to copy more */
1771
1772 case ZSTDds_decodeFrameHeader:
1773 memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
1774 CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
1775 dctx->expected = ZSTD_blockHeaderSize;
1776 dctx->stage = ZSTDds_decodeBlockHeader;
1777 return 0;
1778
1779 case ZSTDds_decodeBlockHeader: {
1780 blockProperties_t bp;
1781 size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
1782 if (ZSTD_isError(cBlockSize))
1783 return cBlockSize;
1784 dctx->expected = cBlockSize;
1785 dctx->bType = bp.blockType;
1786 dctx->rleSize = bp.origSize;
1787 if (cBlockSize) {
1788 dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
1789 return 0;
1790 }
1791 /* empty block */
1792 if (bp.lastBlock) {
1793 if (dctx->fParams.checksumFlag) {
1794 dctx->expected = 4;
1795 dctx->stage = ZSTDds_checkChecksum;
1796 } else {
1797 dctx->expected = 0; /* end of frame */
1798 dctx->stage = ZSTDds_getFrameHeaderSize;
1799 }
1800 } else {
1801 dctx->expected = 3; /* go directly to next header */
1802 dctx->stage = ZSTDds_decodeBlockHeader;
1803 }
1804 return 0;
1805 }
1806 case ZSTDds_decompressLastBlock:
1807 case ZSTDds_decompressBlock: {
1808 size_t rSize;
1809 switch (dctx->bType) {
1810 case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break;
1811 case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break;
1812 case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break;
1813 case bt_reserved: /* should never happen */
1814 default: return ERROR(corruption_detected);
1815 }
1816 if (ZSTD_isError(rSize))
1817 return rSize;
1818 if (dctx->fParams.checksumFlag)
1819 xxh64_update(&dctx->xxhState, dst, rSize);
1820
1821 if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
1822 if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
1823 dctx->expected = 4;
1824 dctx->stage = ZSTDds_checkChecksum;
1825 } else {
1826 dctx->expected = 0; /* ends here */
1827 dctx->stage = ZSTDds_getFrameHeaderSize;
1828 }
1829 } else {
1830 dctx->stage = ZSTDds_decodeBlockHeader;
1831 dctx->expected = ZSTD_blockHeaderSize;
1832 dctx->previousDstEnd = (char *)dst + rSize;
1833 }
1834 return rSize;
1835 }
1836 case ZSTDds_checkChecksum: {
1837 U32 const h32 = (U32)xxh64_digest(&dctx->xxhState);
1838 U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */
1839 if (check32 != h32)
1840 return ERROR(checksum_wrong);
1841 dctx->expected = 0;
1842 dctx->stage = ZSTDds_getFrameHeaderSize;
1843 return 0;
1844 }
1845 case ZSTDds_decodeSkippableHeader: {
1846 memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
1847 dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4);
1848 dctx->stage = ZSTDds_skipFrame;
1849 return 0;
1850 }
1851 case ZSTDds_skipFrame: {
1852 dctx->expected = 0;
1853 dctx->stage = ZSTDds_getFrameHeaderSize;
1854 return 0;
1855 }
1856 default:
1857 return ERROR(GENERIC); /* impossible */
1858 }
1859}
1860
1861static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
1862{
1863 dctx->dictEnd = dctx->previousDstEnd;
1864 dctx->vBase = (const char *)dict - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
1865 dctx->base = dict;
1866 dctx->previousDstEnd = (const char *)dict + dictSize;
1867 return 0;
1868}
1869
1870/* ZSTD_loadEntropy() :
1871 * dict : must point at beginning of a valid zstd dictionary
1872 * @return : size of entropy tables read */
1873static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t *entropy, const void *const dict, size_t const dictSize)
1874{
1875 const BYTE *dictPtr = (const BYTE *)dict;
1876 const BYTE *const dictEnd = dictPtr + dictSize;
1877
1878 if (dictSize <= 8)
1879 return ERROR(dictionary_corrupted);
1880 dictPtr += 8; /* skip header = magic + dictID */
1881
1882 {
1883 size_t const hSize = HUF_readDTableX4_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, entropy->workspace, sizeof(entropy->workspace));
1884 if (HUF_isError(hSize))
1885 return ERROR(dictionary_corrupted);
1886 dictPtr += hSize;
1887 }
1888
1889 {
1890 short offcodeNCount[MaxOff + 1];
1891 U32 offcodeMaxValue = MaxOff, offcodeLog;
1892 size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
1893 if (FSE_isError(offcodeHeaderSize))
1894 return ERROR(dictionary_corrupted);
1895 if (offcodeLog > OffFSELog)
1896 return ERROR(dictionary_corrupted);
1897 CHECK_E(FSE_buildDTable_wksp(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
1898 dictPtr += offcodeHeaderSize;
1899 }
1900
1901 {
1902 short matchlengthNCount[MaxML + 1];
1903 unsigned matchlengthMaxValue = MaxML, matchlengthLog;
1904 size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
1905 if (FSE_isError(matchlengthHeaderSize))
1906 return ERROR(dictionary_corrupted);
1907 if (matchlengthLog > MLFSELog)
1908 return ERROR(dictionary_corrupted);
1909 CHECK_E(FSE_buildDTable_wksp(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
1910 dictPtr += matchlengthHeaderSize;
1911 }
1912
1913 {
1914 short litlengthNCount[MaxLL + 1];
1915 unsigned litlengthMaxValue = MaxLL, litlengthLog;
1916 size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
1917 if (FSE_isError(litlengthHeaderSize))
1918 return ERROR(dictionary_corrupted);
1919 if (litlengthLog > LLFSELog)
1920 return ERROR(dictionary_corrupted);
1921 CHECK_E(FSE_buildDTable_wksp(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
1922 dictPtr += litlengthHeaderSize;
1923 }
1924
1925 if (dictPtr + 12 > dictEnd)
1926 return ERROR(dictionary_corrupted);
1927 {
1928 int i;
1929 size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12));
1930 for (i = 0; i < 3; i++) {
1931 U32 const rep = ZSTD_readLE32(dictPtr);
1932 dictPtr += 4;
1933 if (rep == 0 || rep >= dictContentSize)
1934 return ERROR(dictionary_corrupted);
1935 entropy->rep[i] = rep;
1936 }
1937 }
1938
1939 return dictPtr - (const BYTE *)dict;
1940}
1941
1942static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
1943{
1944 if (dictSize < 8)
1945 return ZSTD_refDictContent(dctx, dict, dictSize);
1946 {
1947 U32 const magic = ZSTD_readLE32(dict);
1948 if (magic != ZSTD_DICT_MAGIC) {
1949 return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
1950 }
1951 }
1952 dctx->dictID = ZSTD_readLE32((const char *)dict + 4);
1953
1954 /* load entropy tables */
1955 {
1956 size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
1957 if (ZSTD_isError(eSize))
1958 return ERROR(dictionary_corrupted);
1959 dict = (const char *)dict + eSize;
1960 dictSize -= eSize;
1961 }
1962 dctx->litEntropy = dctx->fseEntropy = 1;
1963
1964 /* reference dictionary content */
1965 return ZSTD_refDictContent(dctx, dict, dictSize);
1966}
1967
1968size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
1969{
1970 CHECK_F(ZSTD_decompressBegin(dctx));
1971 if (dict && dictSize)
1972 CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
1973 return 0;
1974}
1975
1976/* ====== ZSTD_DDict ====== */
1977
1978struct ZSTD_DDict_s {
1979 void *dictBuffer;
1980 const void *dictContent;
1981 size_t dictSize;
1982 ZSTD_entropyTables_t entropy;
1983 U32 dictID;
1984 U32 entropyPresent;
1985 ZSTD_customMem cMem;
1986}; /* typedef'd to ZSTD_DDict within "zstd.h" */
1987
1988size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); }
1989
1990static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; }
1991
1992static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; }
1993
1994static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict)
1995{
1996 ZSTD_decompressBegin(dstDCtx); /* init */
1997 if (ddict) { /* support refDDict on NULL */
1998 dstDCtx->dictID = ddict->dictID;
1999 dstDCtx->base = ddict->dictContent;
2000 dstDCtx->vBase = ddict->dictContent;
2001 dstDCtx->dictEnd = (const BYTE *)ddict->dictContent + ddict->dictSize;
2002 dstDCtx->previousDstEnd = dstDCtx->dictEnd;
2003 if (ddict->entropyPresent) {
2004 dstDCtx->litEntropy = 1;
2005 dstDCtx->fseEntropy = 1;
2006 dstDCtx->LLTptr = ddict->entropy.LLTable;
2007 dstDCtx->MLTptr = ddict->entropy.MLTable;
2008 dstDCtx->OFTptr = ddict->entropy.OFTable;
2009 dstDCtx->HUFptr = ddict->entropy.hufTable;
2010 dstDCtx->entropy.rep[0] = ddict->entropy.rep[0];
2011 dstDCtx->entropy.rep[1] = ddict->entropy.rep[1];
2012 dstDCtx->entropy.rep[2] = ddict->entropy.rep[2];
2013 } else {
2014 dstDCtx->litEntropy = 0;
2015 dstDCtx->fseEntropy = 0;
2016 }
2017 }
2018}
2019
2020static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict *ddict)
2021{
2022 ddict->dictID = 0;
2023 ddict->entropyPresent = 0;
2024 if (ddict->dictSize < 8)
2025 return 0;
2026 {
2027 U32 const magic = ZSTD_readLE32(ddict->dictContent);
2028 if (magic != ZSTD_DICT_MAGIC)
2029 return 0; /* pure content mode */
2030 }
2031 ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4);
2032
2033 /* load entropy tables */
2034 CHECK_E(ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted);
2035 ddict->entropyPresent = 1;
2036 return 0;
2037}
2038
2039static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
2040{
2041 if (!customMem.customAlloc || !customMem.customFree)
2042 return NULL;
2043
2044 {
2045 ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
2046 if (!ddict)
2047 return NULL;
2048 ddict->cMem = customMem;
2049
2050 if ((byReference) || (!dict) || (!dictSize)) {
2051 ddict->dictBuffer = NULL;
2052 ddict->dictContent = dict;
2053 } else {
2054 void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
2055 if (!internalBuffer) {
2056 ZSTD_freeDDict(ddict);
2057 return NULL;
2058 }
2059 memcpy(internalBuffer, dict, dictSize);
2060 ddict->dictBuffer = internalBuffer;
2061 ddict->dictContent = internalBuffer;
2062 }
2063 ddict->dictSize = dictSize;
2064 ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
2065 /* parse dictionary content */
2066 {
2067 size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
2068 if (ZSTD_isError(errorCode)) {
2069 ZSTD_freeDDict(ddict);
2070 return NULL;
2071 }
2072 }
2073
2074 return ddict;
2075 }
2076}
2077
2078/*! ZSTD_initDDict() :
2079* Create a digested dictionary, to start decompression without startup delay.
2080* `dict` content is copied inside DDict.
2081* Consequently, `dict` can be released after `ZSTD_DDict` creation */
2082ZSTD_DDict *ZSTD_initDDict(const void *dict, size_t dictSize, void *workspace, size_t workspaceSize)
2083{
2084 ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
2085 return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem);
2086}
2087
2088size_t ZSTD_freeDDict(ZSTD_DDict *ddict)
2089{
2090 if (ddict == NULL)
2091 return 0; /* support free on NULL */
2092 {
2093 ZSTD_customMem const cMem = ddict->cMem;
2094 ZSTD_free(ddict->dictBuffer, cMem);
2095 ZSTD_free(ddict, cMem);
2096 return 0;
2097 }
2098}
2099
2100/*! ZSTD_getDictID_fromDict() :
2101 * Provides the dictID stored within dictionary.
2102 * if @return == 0, the dictionary is not conformant with Zstandard specification.
2103 * It can still be loaded, but as a content-only dictionary. */
2104unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize)
2105{
2106 if (dictSize < 8)
2107 return 0;
2108 if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC)
2109 return 0;
2110 return ZSTD_readLE32((const char *)dict + 4);
2111}
2112
2113/*! ZSTD_getDictID_fromDDict() :
2114 * Provides the dictID of the dictionary loaded into `ddict`.
2115 * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
2116 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
2117unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict)
2118{
2119 if (ddict == NULL)
2120 return 0;
2121 return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
2122}
2123
2124/*! ZSTD_getDictID_fromFrame() :
2125 * Provides the dictID required to decompressed the frame stored within `src`.
2126 * If @return == 0, the dictID could not be decoded.
2127 * This could for one of the following reasons :
2128 * - The frame does not require a dictionary to be decoded (most common case).
2129 * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
2130 * Note : this use case also happens when using a non-conformant dictionary.
2131 * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
2132 * - This is not a Zstandard frame.
2133 * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
2134unsigned ZSTD_getDictID_fromFrame(const void *src, size_t srcSize)
2135{
2136 ZSTD_frameParams zfp = {0, 0, 0, 0};
2137 size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
2138 if (ZSTD_isError(hError))
2139 return 0;
2140 return zfp.dictID;
2141}
2142
2143/*! ZSTD_decompress_usingDDict() :
2144* Decompression using a pre-digested Dictionary
2145* Use dictionary without significant overhead. */
2146size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict)
2147{
2148 /* pass content and size in case legacy frames are encountered */
2149 return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict);
2150}
2151
2152/*=====================================
2153* Streaming decompression
2154*====================================*/
2155
2156typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
2157
2158/* *** Resource management *** */
2159struct ZSTD_DStream_s {
2160 ZSTD_DCtx *dctx;
2161 ZSTD_DDict *ddictLocal;
2162 const ZSTD_DDict *ddict;
2163 ZSTD_frameParams fParams;
2164 ZSTD_dStreamStage stage;
2165 char *inBuff;
2166 size_t inBuffSize;
2167 size_t inPos;
2168 size_t maxWindowSize;
2169 char *outBuff;
2170 size_t outBuffSize;
2171 size_t outStart;
2172 size_t outEnd;
2173 size_t blockSize;
2174 BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
2175 size_t lhSize;
2176 ZSTD_customMem customMem;
2177 void *legacyContext;
2178 U32 previousLegacyVersion;
2179 U32 legacyVersion;
2180 U32 hostageByte;
2181}; /* typedef'd to ZSTD_DStream within "zstd.h" */
2182
2183size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize)
2184{
2185 size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
2186 size_t const inBuffSize = blockSize;
2187 size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
2188 return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
2189}
2190
2191static ZSTD_DStream *ZSTD_createDStream_advanced(ZSTD_customMem customMem)
2192{
2193 ZSTD_DStream *zds;
2194
2195 if (!customMem.customAlloc || !customMem.customFree)
2196 return NULL;
2197
2198 zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
2199 if (zds == NULL)
2200 return NULL;
2201 memset(zds, 0, sizeof(ZSTD_DStream));
2202 memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
2203 zds->dctx = ZSTD_createDCtx_advanced(customMem);
2204 if (zds->dctx == NULL) {
2205 ZSTD_freeDStream(zds);
2206 return NULL;
2207 }
2208 zds->stage = zdss_init;
2209 zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
2210 return zds;
2211}
2212
2213ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t workspaceSize)
2214{
2215 ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
2216 ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem);
2217 if (!zds) {
2218 return NULL;
2219 }
2220
2221 zds->maxWindowSize = maxWindowSize;
2222 zds->stage = zdss_loadHeader;
2223 zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
2224 ZSTD_freeDDict(zds->ddictLocal);
2225 zds->ddictLocal = NULL;
2226 zds->ddict = zds->ddictLocal;
2227 zds->legacyVersion = 0;
2228 zds->hostageByte = 0;
2229
2230 {
2231 size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
2232 size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
2233
2234 zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem);
2235 zds->inBuffSize = blockSize;
2236 zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem);
2237 zds->outBuffSize = neededOutSize;
2238 if (zds->inBuff == NULL || zds->outBuff == NULL) {
2239 ZSTD_freeDStream(zds);
2240 return NULL;
2241 }
2242 }
2243 return zds;
2244}
2245
2246ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize)
2247{
2248 ZSTD_DStream *zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize);
2249 if (zds) {
2250 zds->ddict = ddict;
2251 }
2252 return zds;
2253}
2254
2255size_t ZSTD_freeDStream(ZSTD_DStream *zds)
2256{
2257 if (zds == NULL)
2258 return 0; /* support free on null */
2259 {
2260 ZSTD_customMem const cMem = zds->customMem;
2261 ZSTD_freeDCtx(zds->dctx);
2262 zds->dctx = NULL;
2263 ZSTD_freeDDict(zds->ddictLocal);
2264 zds->ddictLocal = NULL;
2265 ZSTD_free(zds->inBuff, cMem);
2266 zds->inBuff = NULL;
2267 ZSTD_free(zds->outBuff, cMem);
2268 zds->outBuff = NULL;
2269 ZSTD_free(zds, cMem);
2270 return 0;
2271 }
2272}
2273
2274/* *** Initialization *** */
2275
2276size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
2277size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
2278
2279size_t ZSTD_resetDStream(ZSTD_DStream *zds)
2280{
2281 zds->stage = zdss_loadHeader;
2282 zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
2283 zds->legacyVersion = 0;
2284 zds->hostageByte = 0;
2285 return ZSTD_frameHeaderSize_prefix;
2286}
2287
2288/* ***** Decompression ***** */
2289
2290ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
2291{
2292 size_t const length = MIN(dstCapacity, srcSize);
2293 memcpy(dst, src, length);
2294 return length;
2295}
2296
2297size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
2298{
2299 const char *const istart = (const char *)(input->src) + input->pos;
2300 const char *const iend = (const char *)(input->src) + input->size;
2301 const char *ip = istart;
2302 char *const ostart = (char *)(output->dst) + output->pos;
2303 char *const oend = (char *)(output->dst) + output->size;
2304 char *op = ostart;
2305 U32 someMoreWork = 1;
2306
2307 while (someMoreWork) {
2308 switch (zds->stage) {
2309 case zdss_init:
2310 ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
2311 /* fall-through */
2312
2313 case zdss_loadHeader: {
2314 size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
2315 if (ZSTD_isError(hSize))
2316 return hSize;
2317 if (hSize != 0) { /* need more input */
2318 size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
2319 if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */
2320 memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip);
2321 zds->lhSize += iend - ip;
2322 input->pos = input->size;
2323 return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) +
2324 ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
2325 }
2326 memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad);
2327 zds->lhSize = hSize;
2328 ip += toLoad;
2329 break;
2330 }
2331
2332 /* check for single-pass mode opportunity */
2333 if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
2334 && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) {
2335 size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart);
2336 if (cSize <= (size_t)(iend - istart)) {
2337 size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict);
2338 if (ZSTD_isError(decompressedSize))
2339 return decompressedSize;
2340 ip = istart + cSize;
2341 op += decompressedSize;
2342 zds->dctx->expected = 0;
2343 zds->stage = zdss_init;
2344 someMoreWork = 0;
2345 break;
2346 }
2347 }
2348
2349 /* Consume header */
2350 ZSTD_refDDict(zds->dctx, zds->ddict);
2351 {
2352 size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
2353 CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
2354 {
2355 size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
2356 CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size));
2357 }
2358 }
2359
2360 zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
2361 if (zds->fParams.windowSize > zds->maxWindowSize)
2362 return ERROR(frameParameter_windowTooLarge);
2363
2364 /* Buffers are preallocated, but double check */
2365 {
2366 size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
2367 size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
2368 if (zds->inBuffSize < blockSize) {
2369 return ERROR(GENERIC);
2370 }
2371 if (zds->outBuffSize < neededOutSize) {
2372 return ERROR(GENERIC);
2373 }
2374 zds->blockSize = blockSize;
2375 }
2376 zds->stage = zdss_read;
2377 }
2378 /* pass-through */
2379
2380 case zdss_read: {
2381 size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
2382 if (neededInSize == 0) { /* end of frame */
2383 zds->stage = zdss_init;
2384 someMoreWork = 0;
2385 break;
2386 }
2387 if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */
2388 const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
2389 size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart,
2390 (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize);
2391 if (ZSTD_isError(decodedSize))
2392 return decodedSize;
2393 ip += neededInSize;
2394 if (!decodedSize && !isSkipFrame)
2395 break; /* this was just a header */
2396 zds->outEnd = zds->outStart + decodedSize;
2397 zds->stage = zdss_flush;
2398 break;
2399 }
2400 if (ip == iend) {
2401 someMoreWork = 0;
2402 break;
2403 } /* no more input */
2404 zds->stage = zdss_load;
2405 /* pass-through */
2406 }
2407
2408 case zdss_load: {
2409 size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
2410 size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
2411 size_t loadedSize;
2412 if (toLoad > zds->inBuffSize - zds->inPos)
2413 return ERROR(corruption_detected); /* should never happen */
2414 loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip);
2415 ip += loadedSize;
2416 zds->inPos += loadedSize;
2417 if (loadedSize < toLoad) {
2418 someMoreWork = 0;
2419 break;
2420 } /* not enough input, wait for more */
2421
2422 /* decode loaded input */
2423 {
2424 const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
2425 size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
2426 zds->inBuff, neededInSize);
2427 if (ZSTD_isError(decodedSize))
2428 return decodedSize;
2429 zds->inPos = 0; /* input is consumed */
2430 if (!decodedSize && !isSkipFrame) {
2431 zds->stage = zdss_read;
2432 break;
2433 } /* this was just a header */
2434 zds->outEnd = zds->outStart + decodedSize;
2435 zds->stage = zdss_flush;
2436 /* pass-through */
2437 }
2438 }
2439
2440 case zdss_flush: {
2441 size_t const toFlushSize = zds->outEnd - zds->outStart;
2442 size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize);
2443 op += flushedSize;
2444 zds->outStart += flushedSize;
2445 if (flushedSize == toFlushSize) { /* flush completed */
2446 zds->stage = zdss_read;
2447 if (zds->outStart + zds->blockSize > zds->outBuffSize)
2448 zds->outStart = zds->outEnd = 0;
2449 break;
2450 }
2451 /* cannot complete flush */
2452 someMoreWork = 0;
2453 break;
2454 }
2455 default:
2456 return ERROR(GENERIC); /* impossible */
2457 }
2458 }
2459
2460 /* result */
2461 input->pos += (size_t)(ip - istart);
2462 output->pos += (size_t)(op - ostart);
2463 {
2464 size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
2465 if (!nextSrcSizeHint) { /* frame fully decoded */
2466 if (zds->outEnd == zds->outStart) { /* output fully flushed */
2467 if (zds->hostageByte) {
2468 if (input->pos >= input->size) {
2469 zds->stage = zdss_read;
2470 return 1;
2471 } /* can't release hostage (not present) */
2472 input->pos++; /* release hostage */
2473 }
2474 return 0;
2475 }
2476 if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
2477 input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
2478 zds->hostageByte = 1;
2479 }
2480 return 1;
2481 }
2482 nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
2483 if (zds->inPos > nextSrcSizeHint)
2484 return ERROR(GENERIC); /* should never happen */
2485 nextSrcSizeHint -= zds->inPos; /* already loaded*/
2486 return nextSrcSizeHint;
2487 }
2488}
2489
2490EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound);
2491EXPORT_SYMBOL(ZSTD_initDCtx);
2492EXPORT_SYMBOL(ZSTD_decompressDCtx);
2493EXPORT_SYMBOL(ZSTD_decompress_usingDict);
2494
2495EXPORT_SYMBOL(ZSTD_DDictWorkspaceBound);
2496EXPORT_SYMBOL(ZSTD_initDDict);
2497EXPORT_SYMBOL(ZSTD_decompress_usingDDict);
2498
2499EXPORT_SYMBOL(ZSTD_DStreamWorkspaceBound);
2500EXPORT_SYMBOL(ZSTD_initDStream);
2501EXPORT_SYMBOL(ZSTD_initDStream_usingDDict);
2502EXPORT_SYMBOL(ZSTD_resetDStream);
2503EXPORT_SYMBOL(ZSTD_decompressStream);
2504EXPORT_SYMBOL(ZSTD_DStreamInSize);
2505EXPORT_SYMBOL(ZSTD_DStreamOutSize);
2506
2507EXPORT_SYMBOL(ZSTD_findFrameCompressedSize);
2508EXPORT_SYMBOL(ZSTD_getFrameContentSize);
2509EXPORT_SYMBOL(ZSTD_findDecompressedSize);
2510
2511EXPORT_SYMBOL(ZSTD_isFrame);
2512EXPORT_SYMBOL(ZSTD_getDictID_fromDict);
2513EXPORT_SYMBOL(ZSTD_getDictID_fromDDict);
2514EXPORT_SYMBOL(ZSTD_getDictID_fromFrame);
2515
2516EXPORT_SYMBOL(ZSTD_getFrameParams);
2517EXPORT_SYMBOL(ZSTD_decompressBegin);
2518EXPORT_SYMBOL(ZSTD_decompressBegin_usingDict);
2519EXPORT_SYMBOL(ZSTD_copyDCtx);
2520EXPORT_SYMBOL(ZSTD_nextSrcSizeToDecompress);
2521EXPORT_SYMBOL(ZSTD_decompressContinue);
2522EXPORT_SYMBOL(ZSTD_nextInputType);
2523
2524EXPORT_SYMBOL(ZSTD_decompressBlock);
2525EXPORT_SYMBOL(ZSTD_insertBlock);
2526
2527MODULE_LICENSE("Dual BSD/GPL");
2528MODULE_DESCRIPTION("Zstd Decompressor");
diff --git a/lib/zstd/entropy_common.c b/lib/zstd/entropy_common.c
new file mode 100644
index 000000000000..2b0a643c32c4
--- /dev/null
+++ b/lib/zstd/entropy_common.c
@@ -0,0 +1,243 @@
1/*
2 * Common functions of New Generation Entropy library
3 * Copyright (C) 2016, Yann Collet.
4 *
5 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * This program is free software; you can redistribute it and/or modify it under
31 * the terms of the GNU General Public License version 2 as published by the
32 * Free Software Foundation. This program is dual-licensed; you may select
33 * either version 2 of the GNU General Public License ("GPL") or BSD license
34 * ("BSD").
35 *
36 * You can contact the author at :
37 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
38 */
39
40/* *************************************
41* Dependencies
42***************************************/
43#include "error_private.h" /* ERR_*, ERROR */
44#include "fse.h"
45#include "huf.h"
46#include "mem.h"
47
48/*=== Version ===*/
49unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
50
51/*=== Error Management ===*/
52unsigned FSE_isError(size_t code) { return ERR_isError(code); }
53
54unsigned HUF_isError(size_t code) { return ERR_isError(code); }
55
56/*-**************************************************************
57* FSE NCount encoding-decoding
58****************************************************************/
59size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize)
60{
61 const BYTE *const istart = (const BYTE *)headerBuffer;
62 const BYTE *const iend = istart + hbSize;
63 const BYTE *ip = istart;
64 int nbBits;
65 int remaining;
66 int threshold;
67 U32 bitStream;
68 int bitCount;
69 unsigned charnum = 0;
70 int previous0 = 0;
71
72 if (hbSize < 4)
73 return ERROR(srcSize_wrong);
74 bitStream = ZSTD_readLE32(ip);
75 nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
76 if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX)
77 return ERROR(tableLog_tooLarge);
78 bitStream >>= 4;
79 bitCount = 4;
80 *tableLogPtr = nbBits;
81 remaining = (1 << nbBits) + 1;
82 threshold = 1 << nbBits;
83 nbBits++;
84
85 while ((remaining > 1) & (charnum <= *maxSVPtr)) {
86 if (previous0) {
87 unsigned n0 = charnum;
88 while ((bitStream & 0xFFFF) == 0xFFFF) {
89 n0 += 24;
90 if (ip < iend - 5) {
91 ip += 2;
92 bitStream = ZSTD_readLE32(ip) >> bitCount;
93 } else {
94 bitStream >>= 16;
95 bitCount += 16;
96 }
97 }
98 while ((bitStream & 3) == 3) {
99 n0 += 3;
100 bitStream >>= 2;
101 bitCount += 2;
102 }
103 n0 += bitStream & 3;
104 bitCount += 2;
105 if (n0 > *maxSVPtr)
106 return ERROR(maxSymbolValue_tooSmall);
107 while (charnum < n0)
108 normalizedCounter[charnum++] = 0;
109 if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
110 ip += bitCount >> 3;
111 bitCount &= 7;
112 bitStream = ZSTD_readLE32(ip) >> bitCount;
113 } else {
114 bitStream >>= 2;
115 }
116 }
117 {
118 int const max = (2 * threshold - 1) - remaining;
119 int count;
120
121 if ((bitStream & (threshold - 1)) < (U32)max) {
122 count = bitStream & (threshold - 1);
123 bitCount += nbBits - 1;
124 } else {
125 count = bitStream & (2 * threshold - 1);
126 if (count >= threshold)
127 count -= max;
128 bitCount += nbBits;
129 }
130
131 count--; /* extra accuracy */
132 remaining -= count < 0 ? -count : count; /* -1 means +1 */
133 normalizedCounter[charnum++] = (short)count;
134 previous0 = !count;
135 while (remaining < threshold) {
136 nbBits--;
137 threshold >>= 1;
138 }
139
140 if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
141 ip += bitCount >> 3;
142 bitCount &= 7;
143 } else {
144 bitCount -= (int)(8 * (iend - 4 - ip));
145 ip = iend - 4;
146 }
147 bitStream = ZSTD_readLE32(ip) >> (bitCount & 31);
148 }
149 } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
150 if (remaining != 1)
151 return ERROR(corruption_detected);
152 if (bitCount > 32)
153 return ERROR(corruption_detected);
154 *maxSVPtr = charnum - 1;
155
156 ip += (bitCount + 7) >> 3;
157 return ip - istart;
158}
159
160/*! HUF_readStats() :
161 Read compact Huffman tree, saved by HUF_writeCTable().
162 `huffWeight` is destination buffer.
163 `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
164 @return : size read from `src` , or an error Code .
165 Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
166*/
167size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
168{
169 U32 weightTotal;
170 const BYTE *ip = (const BYTE *)src;
171 size_t iSize;
172 size_t oSize;
173
174 if (!srcSize)
175 return ERROR(srcSize_wrong);
176 iSize = ip[0];
177 /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
178
179 if (iSize >= 128) { /* special header */
180 oSize = iSize - 127;
181 iSize = ((oSize + 1) / 2);
182 if (iSize + 1 > srcSize)
183 return ERROR(srcSize_wrong);
184 if (oSize >= hwSize)
185 return ERROR(corruption_detected);
186 ip += 1;
187 {
188 U32 n;
189 for (n = 0; n < oSize; n += 2) {
190 huffWeight[n] = ip[n / 2] >> 4;
191 huffWeight[n + 1] = ip[n / 2] & 15;
192 }
193 }
194 } else { /* header compressed with FSE (normal case) */
195 if (iSize + 1 > srcSize)
196 return ERROR(srcSize_wrong);
197 oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, 6, workspace, workspaceSize); /* max (hwSize-1) values decoded, as last one is implied */
198 if (FSE_isError(oSize))
199 return oSize;
200 }
201
202 /* collect weight stats */
203 memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
204 weightTotal = 0;
205 {
206 U32 n;
207 for (n = 0; n < oSize; n++) {
208 if (huffWeight[n] >= HUF_TABLELOG_MAX)
209 return ERROR(corruption_detected);
210 rankStats[huffWeight[n]]++;
211 weightTotal += (1 << huffWeight[n]) >> 1;
212 }
213 }
214 if (weightTotal == 0)
215 return ERROR(corruption_detected);
216
217 /* get last non-null symbol weight (implied, total must be 2^n) */
218 {
219 U32 const tableLog = BIT_highbit32(weightTotal) + 1;
220 if (tableLog > HUF_TABLELOG_MAX)
221 return ERROR(corruption_detected);
222 *tableLogPtr = tableLog;
223 /* determine last weight */
224 {
225 U32 const total = 1 << tableLog;
226 U32 const rest = total - weightTotal;
227 U32 const verif = 1 << BIT_highbit32(rest);
228 U32 const lastWeight = BIT_highbit32(rest) + 1;
229 if (verif != rest)
230 return ERROR(corruption_detected); /* last value must be a clean power of 2 */
231 huffWeight[oSize] = (BYTE)lastWeight;
232 rankStats[lastWeight]++;
233 }
234 }
235
236 /* check tree construction validity */
237 if ((rankStats[1] < 2) || (rankStats[1] & 1))
238 return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
239
240 /* results */
241 *nbSymbolsPtr = (U32)(oSize + 1);
242 return iSize + 1;
243}
diff --git a/lib/zstd/error_private.h b/lib/zstd/error_private.h
new file mode 100644
index 000000000000..1a60b31f706c
--- /dev/null
+++ b/lib/zstd/error_private.h
@@ -0,0 +1,53 @@
1/**
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of https://github.com/facebook/zstd.
7 * An additional grant of patent rights can be found in the PATENTS file in the
8 * same directory.
9 *
10 * This program is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU General Public License version 2 as published by the
12 * Free Software Foundation. This program is dual-licensed; you may select
13 * either version 2 of the GNU General Public License ("GPL") or BSD license
14 * ("BSD").
15 */
16
17/* Note : this module is expected to remain private, do not expose it */
18
19#ifndef ERROR_H_MODULE
20#define ERROR_H_MODULE
21
22/* ****************************************
23* Dependencies
24******************************************/
25#include <linux/types.h> /* size_t */
26#include <linux/zstd.h> /* enum list */
27
28/* ****************************************
29* Compiler-specific
30******************************************/
31#define ERR_STATIC static __attribute__((unused))
32
33/*-****************************************
34* Customization (error_public.h)
35******************************************/
36typedef ZSTD_ErrorCode ERR_enum;
37#define PREFIX(name) ZSTD_error_##name
38
39/*-****************************************
40* Error codes handling
41******************************************/
42#define ERROR(name) ((size_t)-PREFIX(name))
43
44ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
45
46ERR_STATIC ERR_enum ERR_getErrorCode(size_t code)
47{
48 if (!ERR_isError(code))
49 return (ERR_enum)0;
50 return (ERR_enum)(0 - code);
51}
52
53#endif /* ERROR_H_MODULE */
diff --git a/lib/zstd/fse.h b/lib/zstd/fse.h
new file mode 100644
index 000000000000..7460ab04b191
--- /dev/null
+++ b/lib/zstd/fse.h
@@ -0,0 +1,575 @@
1/*
2 * FSE : Finite State Entropy codec
3 * Public Prototypes declaration
4 * Copyright (C) 2013-2016, Yann Collet.
5 *
6 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above
15 * copyright notice, this list of conditions and the following disclaimer
16 * in the documentation and/or other materials provided with the
17 * distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * This program is free software; you can redistribute it and/or modify it under
32 * the terms of the GNU General Public License version 2 as published by the
33 * Free Software Foundation. This program is dual-licensed; you may select
34 * either version 2 of the GNU General Public License ("GPL") or BSD license
35 * ("BSD").
36 *
37 * You can contact the author at :
38 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
39 */
40#ifndef FSE_H
41#define FSE_H
42
43/*-*****************************************
44* Dependencies
45******************************************/
46#include <linux/types.h> /* size_t, ptrdiff_t */
47
48/*-*****************************************
49* FSE_PUBLIC_API : control library symbols visibility
50******************************************/
51#define FSE_PUBLIC_API
52
53/*------ Version ------*/
54#define FSE_VERSION_MAJOR 0
55#define FSE_VERSION_MINOR 9
56#define FSE_VERSION_RELEASE 0
57
58#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
59#define FSE_QUOTE(str) #str
60#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
61#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
62
63#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE)
64FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
65
66/*-*****************************************
67* Tool functions
68******************************************/
69FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
70
71/* Error Management */
72FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
73
74/*-*****************************************
75* FSE detailed API
76******************************************/
77/*!
78FSE_compress() does the following:
791. count symbol occurrence from source[] into table count[]
802. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
813. save normalized counters to memory buffer using writeNCount()
824. build encoding table 'CTable' from normalized counters
835. encode the data stream using encoding table 'CTable'
84
85FSE_decompress() does the following:
861. read normalized counters with readNCount()
872. build decoding table 'DTable' from normalized counters
883. decode the data stream using decoding table 'DTable'
89
90The following API allows targeting specific sub-functions for advanced tasks.
91For example, it's possible to compress several blocks using the same 'CTable',
92or to save and provide normalized distribution using external method.
93*/
94
95/* *** COMPRESSION *** */
96/*! FSE_optimalTableLog():
97 dynamically downsize 'tableLog' when conditions are met.
98 It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
99 @return : recommended tableLog (necessarily <= 'maxTableLog') */
100FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
101
102/*! FSE_normalizeCount():
103 normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
104 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
105 @return : tableLog,
106 or an errorCode, which can be tested using FSE_isError() */
107FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue);
108
109/*! FSE_NCountWriteBound():
110 Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
111 Typically useful for allocation purpose. */
112FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
113
114/*! FSE_writeNCount():
115 Compactly save 'normalizedCounter' into 'buffer'.
116 @return : size of the compressed table,
117 or an errorCode, which can be tested using FSE_isError(). */
118FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
119
120/*! Constructor and Destructor of FSE_CTable.
121 Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
122typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
123
124/*! FSE_compress_usingCTable():
125 Compress `src` using `ct` into `dst` which must be already allocated.
126 @return : size of compressed data (<= `dstCapacity`),
127 or 0 if compressed data could not fit into `dst`,
128 or an errorCode, which can be tested using FSE_isError() */
129FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct);
130
131/*!
132Tutorial :
133----------
134The first step is to count all symbols. FSE_count() does this job very fast.
135Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
136'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
137maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
138FSE_count() will return the number of occurrence of the most frequent symbol.
139This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
140If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
141
142The next step is to normalize the frequencies.
143FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
144It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
145You can use 'tableLog'==0 to mean "use default tableLog value".
146If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
147which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
148
149The result of FSE_normalizeCount() will be saved into a table,
150called 'normalizedCounter', which is a table of signed short.
151'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
152The return value is tableLog if everything proceeded as expected.
153It is 0 if there is a single symbol within distribution.
154If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
155
156'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
157'buffer' must be already allocated.
158For guaranteed success, buffer size must be at least FSE_headerBound().
159The result of the function is the number of bytes written into 'buffer'.
160If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
161
162'normalizedCounter' can then be used to create the compression table 'CTable'.
163The space required by 'CTable' must be already allocated, using FSE_createCTable().
164You can then use FSE_buildCTable() to fill 'CTable'.
165If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
166
167'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
168Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
169The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
170If it returns '0', compressed data could not fit into 'dst'.
171If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
172*/
173
174/* *** DECOMPRESSION *** */
175
176/*! FSE_readNCount():
177 Read compactly saved 'normalizedCounter' from 'rBuffer'.
178 @return : size read from 'rBuffer',
179 or an errorCode, which can be tested using FSE_isError().
180 maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
181FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize);
182
183/*! Constructor and Destructor of FSE_DTable.
184 Note that its size depends on 'tableLog' */
185typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
186
187/*! FSE_buildDTable():
188 Builds 'dt', which must be already allocated, using FSE_createDTable().
189 return : 0, or an errorCode, which can be tested using FSE_isError() */
190FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize);
191
192/*! FSE_decompress_usingDTable():
193 Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
194 into `dst` which must be already allocated.
195 @return : size of regenerated data (necessarily <= `dstCapacity`),
196 or an errorCode, which can be tested using FSE_isError() */
197FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt);
198
199/*!
200Tutorial :
201----------
202(Note : these functions only decompress FSE-compressed blocks.
203 If block is uncompressed, use memcpy() instead
204 If block is a single repeated byte, use memset() instead )
205
206The first step is to obtain the normalized frequencies of symbols.
207This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
208'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
209In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
210or size the table to handle worst case situations (typically 256).
211FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
212The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
213Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
214If there is an error, the function will return an error code, which can be tested using FSE_isError().
215
216The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
217This is performed by the function FSE_buildDTable().
218The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
219If there is an error, the function will return an error code, which can be tested using FSE_isError().
220
221`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
222`cSrcSize` must be strictly correct, otherwise decompression will fail.
223FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
224If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
225*/
226
227/* *** Dependency *** */
228#include "bitstream.h"
229
230/* *****************************************
231* Static allocation
232*******************************************/
233/* FSE buffer bounds */
234#define FSE_NCOUNTBOUND 512
235#define FSE_BLOCKBOUND(size) (size + (size >> 7))
236#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
237
238/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
239#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2))
240#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog))
241
242/* *****************************************
243* FSE advanced API
244*******************************************/
245/* FSE_count_wksp() :
246 * Same as FSE_count(), but using an externally provided scratch buffer.
247 * `workSpace` size must be table of >= `1024` unsigned
248 */
249size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace);
250
251/* FSE_countFast_wksp() :
252 * Same as FSE_countFast(), but using an externally provided scratch buffer.
253 * `workSpace` must be a table of minimum `1024` unsigned
254 */
255size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace);
256
257/*! FSE_count_simple
258 * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
259 * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
260*/
261size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize);
262
263unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
264/**< same as FSE_optimalTableLog(), which used `minus==2` */
265
266size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits);
267/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
268
269size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue);
270/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
271
272/* FSE_buildCTable_wksp() :
273 * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
274 * `wkspSize` must be >= `(1<<tableLog)`.
275 */
276size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize);
277
278size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits);
279/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
280
281size_t FSE_buildDTable_rle(FSE_DTable *dt, unsigned char symbolValue);
282/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
283
284size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize);
285/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
286
287/* *****************************************
288* FSE symbol compression API
289*******************************************/
290/*!
291 This API consists of small unitary functions, which highly benefit from being inlined.
292 Hence their body are included in next section.
293*/
294typedef struct {
295 ptrdiff_t value;
296 const void *stateTable;
297 const void *symbolTT;
298 unsigned stateLog;
299} FSE_CState_t;
300
301static void FSE_initCState(FSE_CState_t *CStatePtr, const FSE_CTable *ct);
302
303static void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *CStatePtr, unsigned symbol);
304
305static void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *CStatePtr);
306
307/**<
308These functions are inner components of FSE_compress_usingCTable().
309They allow the creation of custom streams, mixing multiple tables and bit sources.
310
311A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
312So the first symbol you will encode is the last you will decode, like a LIFO stack.
313
314You will need a few variables to track your CStream. They are :
315
316FSE_CTable ct; // Provided by FSE_buildCTable()
317BIT_CStream_t bitStream; // bitStream tracking structure
318FSE_CState_t state; // State tracking structure (can have several)
319
320
321The first thing to do is to init bitStream and state.
322 size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
323 FSE_initCState(&state, ct);
324
325Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
326You can then encode your input data, byte after byte.
327FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
328Remember decoding will be done in reverse direction.
329 FSE_encodeByte(&bitStream, &state, symbol);
330
331At any time, you can also add any bit sequence.
332Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
333 BIT_addBits(&bitStream, bitField, nbBits);
334
335The above methods don't commit data to memory, they just store it into local register, for speed.
336Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
337Writing data to memory is a manual operation, performed by the flushBits function.
338 BIT_flushBits(&bitStream);
339
340Your last FSE encoding operation shall be to flush your last state value(s).
341 FSE_flushState(&bitStream, &state);
342
343Finally, you must close the bitStream.
344The function returns the size of CStream in bytes.
345If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
346If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
347 size_t size = BIT_closeCStream(&bitStream);
348*/
349
350/* *****************************************
351* FSE symbol decompression API
352*******************************************/
353typedef struct {
354 size_t state;
355 const void *table; /* precise table may vary, depending on U16 */
356} FSE_DState_t;
357
358static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt);
359
360static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
361
362static unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr);
363
364/**<
365Let's now decompose FSE_decompress_usingDTable() into its unitary components.
366You will decode FSE-encoded symbols from the bitStream,
367and also any other bitFields you put in, **in reverse order**.
368
369You will need a few variables to track your bitStream. They are :
370
371BIT_DStream_t DStream; // Stream context
372FSE_DState_t DState; // State context. Multiple ones are possible
373FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
374
375The first thing to do is to init the bitStream.
376 errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
377
378You should then retrieve your initial state(s)
379(in reverse flushing order if you have several ones) :
380 errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
381
382You can then decode your data, symbol after symbol.
383For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
384Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
385 unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
386
387You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
388Note : maximum allowed nbBits is 25, for 32-bits compatibility
389 size_t bitField = BIT_readBits(&DStream, nbBits);
390
391All above operations only read from local register (which size depends on size_t).
392Refueling the register from memory is manually performed by the reload method.
393 endSignal = FSE_reloadDStream(&DStream);
394
395BIT_reloadDStream() result tells if there is still some more data to read from DStream.
396BIT_DStream_unfinished : there is still some data left into the DStream.
397BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
398BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
399BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
400
401When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
402to properly detect the exact end of stream.
403After each decoded symbol, check if DStream is fully consumed using this simple test :
404 BIT_reloadDStream(&DStream) >= BIT_DStream_completed
405
406When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
407Checking if DStream has reached its end is performed by :
408 BIT_endOfDStream(&DStream);
409Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
410 FSE_endOfDState(&DState);
411*/
412
413/* *****************************************
414* FSE unsafe API
415*******************************************/
416static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
417/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
418
419/* *****************************************
420* Implementation of inlined functions
421*******************************************/
422typedef struct {
423 int deltaFindState;
424 U32 deltaNbBits;
425} FSE_symbolCompressionTransform; /* total 8 bytes */
426
427ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct)
428{
429 const void *ptr = ct;
430 const U16 *u16ptr = (const U16 *)ptr;
431 const U32 tableLog = ZSTD_read16(ptr);
432 statePtr->value = (ptrdiff_t)1 << tableLog;
433 statePtr->stateTable = u16ptr + 2;
434 statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1));
435 statePtr->stateLog = tableLog;
436}
437
438/*! FSE_initCState2() :
439* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
440* uses the smallest state value possible, saving the cost of this symbol */
441ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol)
442{
443 FSE_initCState(statePtr, ct);
444 {
445 const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
446 const U16 *stateTable = (const U16 *)(statePtr->stateTable);
447 U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16);
448 statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
449 statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
450 }
451}
452
453ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol)
454{
455 const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
456 const U16 *const stateTable = (const U16 *)(statePtr->stateTable);
457 U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
458 BIT_addBits(bitC, statePtr->value, nbBitsOut);
459 statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
460}
461
462ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr)
463{
464 BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
465 BIT_flushBits(bitC);
466}
467
468/* ====== Decompression ====== */
469
470typedef struct {
471 U16 tableLog;
472 U16 fastMode;
473} FSE_DTableHeader; /* sizeof U32 */
474
475typedef struct {
476 unsigned short newState;
477 unsigned char symbol;
478 unsigned char nbBits;
479} FSE_decode_t; /* size == U32 */
480
481ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt)
482{
483 const void *ptr = dt;
484 const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr;
485 DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
486 BIT_reloadDStream(bitD);
487 DStatePtr->table = dt + 1;
488}
489
490ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr)
491{
492 FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
493 return DInfo.symbol;
494}
495
496ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
497{
498 FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
499 U32 const nbBits = DInfo.nbBits;
500 size_t const lowBits = BIT_readBits(bitD, nbBits);
501 DStatePtr->state = DInfo.newState + lowBits;
502}
503
504ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
505{
506 FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
507 U32 const nbBits = DInfo.nbBits;
508 BYTE const symbol = DInfo.symbol;
509 size_t const lowBits = BIT_readBits(bitD, nbBits);
510
511 DStatePtr->state = DInfo.newState + lowBits;
512 return symbol;
513}
514
515/*! FSE_decodeSymbolFast() :
516 unsafe, only works if no symbol has a probability > 50% */
517ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
518{
519 FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
520 U32 const nbBits = DInfo.nbBits;
521 BYTE const symbol = DInfo.symbol;
522 size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
523
524 DStatePtr->state = DInfo.newState + lowBits;
525 return symbol;
526}
527
528ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; }
529
530/* **************************************************************
531* Tuning parameters
532****************************************************************/
533/*!MEMORY_USAGE :
534* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
535* Increasing memory usage improves compression ratio
536* Reduced memory usage can improve speed, due to cache effect
537* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
538#ifndef FSE_MAX_MEMORY_USAGE
539#define FSE_MAX_MEMORY_USAGE 14
540#endif
541#ifndef FSE_DEFAULT_MEMORY_USAGE
542#define FSE_DEFAULT_MEMORY_USAGE 13
543#endif
544
545/*!FSE_MAX_SYMBOL_VALUE :
546* Maximum symbol value authorized.
547* Required for proper stack allocation */
548#ifndef FSE_MAX_SYMBOL_VALUE
549#define FSE_MAX_SYMBOL_VALUE 255
550#endif
551
552/* **************************************************************
553* template functions type & suffix
554****************************************************************/
555#define FSE_FUNCTION_TYPE BYTE
556#define FSE_FUNCTION_EXTENSION
557#define FSE_DECODE_TYPE FSE_decode_t
558
559/* ***************************************************************
560* Constants
561*****************************************************************/
562#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2)
563#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG)
564#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1)
565#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2)
566#define FSE_MIN_TABLELOG 5
567
568#define FSE_TABLELOG_ABSOLUTE_MAX 15
569#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
570#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
571#endif
572
573#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3)
574
575#endif /* FSE_H */
diff --git a/lib/zstd/fse_compress.c b/lib/zstd/fse_compress.c
new file mode 100644
index 000000000000..ef3d1741d532
--- /dev/null
+++ b/lib/zstd/fse_compress.c
@@ -0,0 +1,795 @@
1/*
2 * FSE : Finite State Entropy encoder
3 * Copyright (C) 2013-2015, Yann Collet.
4 *
5 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * This program is free software; you can redistribute it and/or modify it under
31 * the terms of the GNU General Public License version 2 as published by the
32 * Free Software Foundation. This program is dual-licensed; you may select
33 * either version 2 of the GNU General Public License ("GPL") or BSD license
34 * ("BSD").
35 *
36 * You can contact the author at :
37 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
38 */
39
40/* **************************************************************
41* Compiler specifics
42****************************************************************/
43#define FORCE_INLINE static __always_inline
44
45/* **************************************************************
46* Includes
47****************************************************************/
48#include "bitstream.h"
49#include "fse.h"
50#include <linux/compiler.h>
51#include <linux/kernel.h>
52#include <linux/math64.h>
53#include <linux/string.h> /* memcpy, memset */
54
55/* **************************************************************
56* Error Management
57****************************************************************/
58#define FSE_STATIC_ASSERT(c) \
59 { \
60 enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
61 } /* use only *after* variable declarations */
62
63/* **************************************************************
64* Templates
65****************************************************************/
66/*
67 designed to be included
68 for type-specific functions (template emulation in C)
69 Objective is to write these functions only once, for improved maintenance
70*/
71
72/* safety checks */
73#ifndef FSE_FUNCTION_EXTENSION
74#error "FSE_FUNCTION_EXTENSION must be defined"
75#endif
76#ifndef FSE_FUNCTION_TYPE
77#error "FSE_FUNCTION_TYPE must be defined"
78#endif
79
80/* Function names */
81#define FSE_CAT(X, Y) X##Y
82#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
83#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
84
85/* Function templates */
86
87/* FSE_buildCTable_wksp() :
88 * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
89 * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
90 * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
91 */
92size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize)
93{
94 U32 const tableSize = 1 << tableLog;
95 U32 const tableMask = tableSize - 1;
96 void *const ptr = ct;
97 U16 *const tableU16 = ((U16 *)ptr) + 2;
98 void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableLog ? tableSize >> 1 : 1);
99 FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
100 U32 const step = FSE_TABLESTEP(tableSize);
101 U32 highThreshold = tableSize - 1;
102
103 U32 *cumul;
104 FSE_FUNCTION_TYPE *tableSymbol;
105 size_t spaceUsed32 = 0;
106
107 cumul = (U32 *)workspace + spaceUsed32;
108 spaceUsed32 += FSE_MAX_SYMBOL_VALUE + 2;
109 tableSymbol = (FSE_FUNCTION_TYPE *)((U32 *)workspace + spaceUsed32);
110 spaceUsed32 += ALIGN(sizeof(FSE_FUNCTION_TYPE) * ((size_t)1 << tableLog), sizeof(U32)) >> 2;
111
112 if ((spaceUsed32 << 2) > workspaceSize)
113 return ERROR(tableLog_tooLarge);
114 workspace = (U32 *)workspace + spaceUsed32;
115 workspaceSize -= (spaceUsed32 << 2);
116
117 /* CTable header */
118 tableU16[-2] = (U16)tableLog;
119 tableU16[-1] = (U16)maxSymbolValue;
120
121 /* For explanations on how to distribute symbol values over the table :
122 * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
123
124 /* symbol start positions */
125 {
126 U32 u;
127 cumul[0] = 0;
128 for (u = 1; u <= maxSymbolValue + 1; u++) {
129 if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */
130 cumul[u] = cumul[u - 1] + 1;
131 tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1);
132 } else {
133 cumul[u] = cumul[u - 1] + normalizedCounter[u - 1];
134 }
135 }
136 cumul[maxSymbolValue + 1] = tableSize + 1;
137 }
138
139 /* Spread symbols */
140 {
141 U32 position = 0;
142 U32 symbol;
143 for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
144 int nbOccurences;
145 for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
146 tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
147 position = (position + step) & tableMask;
148 while (position > highThreshold)
149 position = (position + step) & tableMask; /* Low proba area */
150 }
151 }
152
153 if (position != 0)
154 return ERROR(GENERIC); /* Must have gone through all positions */
155 }
156
157 /* Build table */
158 {
159 U32 u;
160 for (u = 0; u < tableSize; u++) {
161 FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
162 tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */
163 }
164 }
165
166 /* Build Symbol Transformation Table */
167 {
168 unsigned total = 0;
169 unsigned s;
170 for (s = 0; s <= maxSymbolValue; s++) {
171 switch (normalizedCounter[s]) {
172 case 0: break;
173
174 case -1:
175 case 1:
176 symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog);
177 symbolTT[s].deltaFindState = total - 1;
178 total++;
179 break;
180 default: {
181 U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1);
182 U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
183 symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
184 symbolTT[s].deltaFindState = total - normalizedCounter[s];
185 total += normalizedCounter[s];
186 }
187 }
188 }
189 }
190
191 return 0;
192}
193
194/*-**************************************************************
195* FSE NCount encoding-decoding
196****************************************************************/
197size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
198{
199 size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3;
200 return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
201}
202
203static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
204 unsigned writeIsSafe)
205{
206 BYTE *const ostart = (BYTE *)header;
207 BYTE *out = ostart;
208 BYTE *const oend = ostart + headerBufferSize;
209 int nbBits;
210 const int tableSize = 1 << tableLog;
211 int remaining;
212 int threshold;
213 U32 bitStream;
214 int bitCount;
215 unsigned charnum = 0;
216 int previous0 = 0;
217
218 bitStream = 0;
219 bitCount = 0;
220 /* Table Size */
221 bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount;
222 bitCount += 4;
223
224 /* Init */
225 remaining = tableSize + 1; /* +1 for extra accuracy */
226 threshold = tableSize;
227 nbBits = tableLog + 1;
228
229 while (remaining > 1) { /* stops at 1 */
230 if (previous0) {
231 unsigned start = charnum;
232 while (!normalizedCounter[charnum])
233 charnum++;
234 while (charnum >= start + 24) {
235 start += 24;
236 bitStream += 0xFFFFU << bitCount;
237 if ((!writeIsSafe) && (out > oend - 2))
238 return ERROR(dstSize_tooSmall); /* Buffer overflow */
239 out[0] = (BYTE)bitStream;
240 out[1] = (BYTE)(bitStream >> 8);
241 out += 2;
242 bitStream >>= 16;
243 }
244 while (charnum >= start + 3) {
245 start += 3;
246 bitStream += 3 << bitCount;
247 bitCount += 2;
248 }
249 bitStream += (charnum - start) << bitCount;
250 bitCount += 2;
251 if (bitCount > 16) {
252 if ((!writeIsSafe) && (out > oend - 2))
253 return ERROR(dstSize_tooSmall); /* Buffer overflow */
254 out[0] = (BYTE)bitStream;
255 out[1] = (BYTE)(bitStream >> 8);
256 out += 2;
257 bitStream >>= 16;
258 bitCount -= 16;
259 }
260 }
261 {
262 int count = normalizedCounter[charnum++];
263 int const max = (2 * threshold - 1) - remaining;
264 remaining -= count < 0 ? -count : count;
265 count++; /* +1 for extra accuracy */
266 if (count >= threshold)
267 count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
268 bitStream += count << bitCount;
269 bitCount += nbBits;
270 bitCount -= (count < max);
271 previous0 = (count == 1);
272 if (remaining < 1)
273 return ERROR(GENERIC);
274 while (remaining < threshold)
275 nbBits--, threshold >>= 1;
276 }
277 if (bitCount > 16) {
278 if ((!writeIsSafe) && (out > oend - 2))
279 return ERROR(dstSize_tooSmall); /* Buffer overflow */
280 out[0] = (BYTE)bitStream;
281 out[1] = (BYTE)(bitStream >> 8);
282 out += 2;
283 bitStream >>= 16;
284 bitCount -= 16;
285 }
286 }
287
288 /* flush remaining bitStream */
289 if ((!writeIsSafe) && (out > oend - 2))
290 return ERROR(dstSize_tooSmall); /* Buffer overflow */
291 out[0] = (BYTE)bitStream;
292 out[1] = (BYTE)(bitStream >> 8);
293 out += (bitCount + 7) / 8;
294
295 if (charnum > maxSymbolValue + 1)
296 return ERROR(GENERIC);
297
298 return (out - ostart);
299}
300
301size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
302{
303 if (tableLog > FSE_MAX_TABLELOG)
304 return ERROR(tableLog_tooLarge); /* Unsupported */
305 if (tableLog < FSE_MIN_TABLELOG)
306 return ERROR(GENERIC); /* Unsupported */
307
308 if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
309 return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
310
311 return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
312}
313
314/*-**************************************************************
315* Counting histogram
316****************************************************************/
317/*! FSE_count_simple
318 This function counts byte values within `src`, and store the histogram into table `count`.
319 It doesn't use any additional memory.
320 But this function is unsafe : it doesn't check that all values within `src` can fit into `count`.
321 For this reason, prefer using a table `count` with 256 elements.
322 @return : count of most numerous element
323*/
324size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize)
325{
326 const BYTE *ip = (const BYTE *)src;
327 const BYTE *const end = ip + srcSize;
328 unsigned maxSymbolValue = *maxSymbolValuePtr;
329 unsigned max = 0;
330
331 memset(count, 0, (maxSymbolValue + 1) * sizeof(*count));
332 if (srcSize == 0) {
333 *maxSymbolValuePtr = 0;
334 return 0;
335 }
336
337 while (ip < end)
338 count[*ip++]++;
339
340 while (!count[maxSymbolValue])
341 maxSymbolValue--;
342 *maxSymbolValuePtr = maxSymbolValue;
343
344 {
345 U32 s;
346 for (s = 0; s <= maxSymbolValue; s++)
347 if (count[s] > max)
348 max = count[s];
349 }
350
351 return (size_t)max;
352}
353
354/* FSE_count_parallel_wksp() :
355 * Same as FSE_count_parallel(), but using an externally provided scratch buffer.
356 * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */
357static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax,
358 unsigned *const workSpace)
359{
360 const BYTE *ip = (const BYTE *)source;
361 const BYTE *const iend = ip + sourceSize;
362 unsigned maxSymbolValue = *maxSymbolValuePtr;
363 unsigned max = 0;
364 U32 *const Counting1 = workSpace;
365 U32 *const Counting2 = Counting1 + 256;
366 U32 *const Counting3 = Counting2 + 256;
367 U32 *const Counting4 = Counting3 + 256;
368
369 memset(Counting1, 0, 4 * 256 * sizeof(unsigned));
370
371 /* safety checks */
372 if (!sourceSize) {
373 memset(count, 0, maxSymbolValue + 1);
374 *maxSymbolValuePtr = 0;
375 return 0;
376 }
377 if (!maxSymbolValue)
378 maxSymbolValue = 255; /* 0 == default */
379
380 /* by stripes of 16 bytes */
381 {
382 U32 cached = ZSTD_read32(ip);
383 ip += 4;
384 while (ip < iend - 15) {
385 U32 c = cached;
386 cached = ZSTD_read32(ip);
387 ip += 4;
388 Counting1[(BYTE)c]++;
389 Counting2[(BYTE)(c >> 8)]++;
390 Counting3[(BYTE)(c >> 16)]++;
391 Counting4[c >> 24]++;
392 c = cached;
393 cached = ZSTD_read32(ip);
394 ip += 4;
395 Counting1[(BYTE)c]++;
396 Counting2[(BYTE)(c >> 8)]++;
397 Counting3[(BYTE)(c >> 16)]++;
398 Counting4[c >> 24]++;
399 c = cached;
400 cached = ZSTD_read32(ip);
401 ip += 4;
402 Counting1[(BYTE)c]++;
403 Counting2[(BYTE)(c >> 8)]++;
404 Counting3[(BYTE)(c >> 16)]++;
405 Counting4[c >> 24]++;
406 c = cached;
407 cached = ZSTD_read32(ip);
408 ip += 4;
409 Counting1[(BYTE)c]++;
410 Counting2[(BYTE)(c >> 8)]++;
411 Counting3[(BYTE)(c >> 16)]++;
412 Counting4[c >> 24]++;
413 }
414 ip -= 4;
415 }
416
417 /* finish last symbols */
418 while (ip < iend)
419 Counting1[*ip++]++;
420
421 if (checkMax) { /* verify stats will fit into destination table */
422 U32 s;
423 for (s = 255; s > maxSymbolValue; s--) {
424 Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
425 if (Counting1[s])
426 return ERROR(maxSymbolValue_tooSmall);
427 }
428 }
429
430 {
431 U32 s;
432 for (s = 0; s <= maxSymbolValue; s++) {
433 count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
434 if (count[s] > max)
435 max = count[s];
436 }
437 }
438
439 while (!count[maxSymbolValue])
440 maxSymbolValue--;
441 *maxSymbolValuePtr = maxSymbolValue;
442 return (size_t)max;
443}
444
445/* FSE_countFast_wksp() :
446 * Same as FSE_countFast(), but using an externally provided scratch buffer.
447 * `workSpace` size must be table of >= `1024` unsigned */
448size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
449{
450 if (sourceSize < 1500)
451 return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
452 return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
453}
454
455/* FSE_count_wksp() :
456 * Same as FSE_count(), but using an externally provided scratch buffer.
457 * `workSpace` size must be table of >= `1024` unsigned */
458size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
459{
460 if (*maxSymbolValuePtr < 255)
461 return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
462 *maxSymbolValuePtr = 255;
463 return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
464}
465
466/*-**************************************************************
467* FSE Compression Code
468****************************************************************/
469/*! FSE_sizeof_CTable() :
470 FSE_CTable is a variable size structure which contains :
471 `U16 tableLog;`
472 `U16 maxSymbolValue;`
473 `U16 nextStateNumber[1 << tableLog];` // This size is variable
474 `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
475Allocation is manual (C standard does not support variable-size structures).
476*/
477size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog)
478{
479 if (tableLog > FSE_MAX_TABLELOG)
480 return ERROR(tableLog_tooLarge);
481 return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32);
482}
483
484/* provides the minimum logSize to safely represent a distribution */
485static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
486{
487 U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
488 U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
489 U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
490 return minBits;
491}
492
493unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
494{
495 U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
496 U32 tableLog = maxTableLog;
497 U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
498 if (tableLog == 0)
499 tableLog = FSE_DEFAULT_TABLELOG;
500 if (maxBitsSrc < tableLog)
501 tableLog = maxBitsSrc; /* Accuracy can be reduced */
502 if (minBits > tableLog)
503 tableLog = minBits; /* Need a minimum to safely represent all symbol values */
504 if (tableLog < FSE_MIN_TABLELOG)
505 tableLog = FSE_MIN_TABLELOG;
506 if (tableLog > FSE_MAX_TABLELOG)
507 tableLog = FSE_MAX_TABLELOG;
508 return tableLog;
509}
510
511unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
512{
513 return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
514}
515
516/* Secondary normalization method.
517 To be used when primary method fails. */
518
519static size_t FSE_normalizeM2(short *norm, U32 tableLog, const unsigned *count, size_t total, U32 maxSymbolValue)
520{
521 short const NOT_YET_ASSIGNED = -2;
522 U32 s;
523 U32 distributed = 0;
524 U32 ToDistribute;
525
526 /* Init */
527 U32 const lowThreshold = (U32)(total >> tableLog);
528 U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
529
530 for (s = 0; s <= maxSymbolValue; s++) {
531 if (count[s] == 0) {
532 norm[s] = 0;
533 continue;
534 }
535 if (count[s] <= lowThreshold) {
536 norm[s] = -1;
537 distributed++;
538 total -= count[s];
539 continue;
540 }
541 if (count[s] <= lowOne) {
542 norm[s] = 1;
543 distributed++;
544 total -= count[s];
545 continue;
546 }
547
548 norm[s] = NOT_YET_ASSIGNED;
549 }
550 ToDistribute = (1 << tableLog) - distributed;
551
552 if ((total / ToDistribute) > lowOne) {
553 /* risk of rounding to zero */
554 lowOne = (U32)((total * 3) / (ToDistribute * 2));
555 for (s = 0; s <= maxSymbolValue; s++) {
556 if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
557 norm[s] = 1;
558 distributed++;
559 total -= count[s];
560 continue;
561 }
562 }
563 ToDistribute = (1 << tableLog) - distributed;
564 }
565
566 if (distributed == maxSymbolValue + 1) {
567 /* all values are pretty poor;
568 probably incompressible data (should have already been detected);
569 find max, then give all remaining points to max */
570 U32 maxV = 0, maxC = 0;
571 for (s = 0; s <= maxSymbolValue; s++)
572 if (count[s] > maxC)
573 maxV = s, maxC = count[s];
574 norm[maxV] += (short)ToDistribute;
575 return 0;
576 }
577
578 if (total == 0) {
579 /* all of the symbols were low enough for the lowOne or lowThreshold */
580 for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1))
581 if (norm[s] > 0)
582 ToDistribute--, norm[s]++;
583 return 0;
584 }
585
586 {
587 U64 const vStepLog = 62 - tableLog;
588 U64 const mid = (1ULL << (vStepLog - 1)) - 1;
589 U64 const rStep = div_u64((((U64)1 << vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
590 U64 tmpTotal = mid;
591 for (s = 0; s <= maxSymbolValue; s++) {
592 if (norm[s] == NOT_YET_ASSIGNED) {
593 U64 const end = tmpTotal + (count[s] * rStep);
594 U32 const sStart = (U32)(tmpTotal >> vStepLog);
595 U32 const sEnd = (U32)(end >> vStepLog);
596 U32 const weight = sEnd - sStart;
597 if (weight < 1)
598 return ERROR(GENERIC);
599 norm[s] = (short)weight;
600 tmpTotal = end;
601 }
602 }
603 }
604
605 return 0;
606}
607
608size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue)
609{
610 /* Sanity checks */
611 if (tableLog == 0)
612 tableLog = FSE_DEFAULT_TABLELOG;
613 if (tableLog < FSE_MIN_TABLELOG)
614 return ERROR(GENERIC); /* Unsupported size */
615 if (tableLog > FSE_MAX_TABLELOG)
616 return ERROR(tableLog_tooLarge); /* Unsupported size */
617 if (tableLog < FSE_minTableLog(total, maxSymbolValue))
618 return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
619
620 {
621 U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000};
622 U64 const scale = 62 - tableLog;
623 U64 const step = div_u64((U64)1 << 62, (U32)total); /* <== here, one division ! */
624 U64 const vStep = 1ULL << (scale - 20);
625 int stillToDistribute = 1 << tableLog;
626 unsigned s;
627 unsigned largest = 0;
628 short largestP = 0;
629 U32 lowThreshold = (U32)(total >> tableLog);
630
631 for (s = 0; s <= maxSymbolValue; s++) {
632 if (count[s] == total)
633 return 0; /* rle special case */
634 if (count[s] == 0) {
635 normalizedCounter[s] = 0;
636 continue;
637 }
638 if (count[s] <= lowThreshold) {
639 normalizedCounter[s] = -1;
640 stillToDistribute--;
641 } else {
642 short proba = (short)((count[s] * step) >> scale);
643 if (proba < 8) {
644 U64 restToBeat = vStep * rtbTable[proba];
645 proba += (count[s] * step) - ((U64)proba << scale) > restToBeat;
646 }
647 if (proba > largestP)
648 largestP = proba, largest = s;
649 normalizedCounter[s] = proba;
650 stillToDistribute -= proba;
651 }
652 }
653 if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
654 /* corner case, need another normalization method */
655 size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
656 if (FSE_isError(errorCode))
657 return errorCode;
658 } else
659 normalizedCounter[largest] += (short)stillToDistribute;
660 }
661
662 return tableLog;
663}
664
665/* fake FSE_CTable, for raw (uncompressed) input */
666size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits)
667{
668 const unsigned tableSize = 1 << nbBits;
669 const unsigned tableMask = tableSize - 1;
670 const unsigned maxSymbolValue = tableMask;
671 void *const ptr = ct;
672 U16 *const tableU16 = ((U16 *)ptr) + 2;
673 void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableSize >> 1); /* assumption : tableLog >= 1 */
674 FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
675 unsigned s;
676
677 /* Sanity checks */
678 if (nbBits < 1)
679 return ERROR(GENERIC); /* min size */
680
681 /* header */
682 tableU16[-2] = (U16)nbBits;
683 tableU16[-1] = (U16)maxSymbolValue;
684
685 /* Build table */
686 for (s = 0; s < tableSize; s++)
687 tableU16[s] = (U16)(tableSize + s);
688
689 /* Build Symbol Transformation Table */
690 {
691 const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
692 for (s = 0; s <= maxSymbolValue; s++) {
693 symbolTT[s].deltaNbBits = deltaNbBits;
694 symbolTT[s].deltaFindState = s - 1;
695 }
696 }
697
698 return 0;
699}
700
701/* fake FSE_CTable, for rle input (always same symbol) */
702size_t FSE_buildCTable_rle(FSE_CTable *ct, BYTE symbolValue)
703{
704 void *ptr = ct;
705 U16 *tableU16 = ((U16 *)ptr) + 2;
706 void *FSCTptr = (U32 *)ptr + 2;
707 FSE_symbolCompressionTransform *symbolTT = (FSE_symbolCompressionTransform *)FSCTptr;
708
709 /* header */
710 tableU16[-2] = (U16)0;
711 tableU16[-1] = (U16)symbolValue;
712
713 /* Build table */
714 tableU16[0] = 0;
715 tableU16[1] = 0; /* just in case */
716
717 /* Build Symbol Transformation Table */
718 symbolTT[symbolValue].deltaNbBits = 0;
719 symbolTT[symbolValue].deltaFindState = 0;
720
721 return 0;
722}
723
724static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast)
725{
726 const BYTE *const istart = (const BYTE *)src;
727 const BYTE *const iend = istart + srcSize;
728 const BYTE *ip = iend;
729
730 BIT_CStream_t bitC;
731 FSE_CState_t CState1, CState2;
732
733 /* init */
734 if (srcSize <= 2)
735 return 0;
736 {
737 size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
738 if (FSE_isError(initError))
739 return 0; /* not enough space available to write a bitstream */
740 }
741
742#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
743
744 if (srcSize & 1) {
745 FSE_initCState2(&CState1, ct, *--ip);
746 FSE_initCState2(&CState2, ct, *--ip);
747 FSE_encodeSymbol(&bitC, &CState1, *--ip);
748 FSE_FLUSHBITS(&bitC);
749 } else {
750 FSE_initCState2(&CState2, ct, *--ip);
751 FSE_initCState2(&CState1, ct, *--ip);
752 }
753
754 /* join to mod 4 */
755 srcSize -= 2;
756 if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */
757 FSE_encodeSymbol(&bitC, &CState2, *--ip);
758 FSE_encodeSymbol(&bitC, &CState1, *--ip);
759 FSE_FLUSHBITS(&bitC);
760 }
761
762 /* 2 or 4 encoding per loop */
763 while (ip > istart) {
764
765 FSE_encodeSymbol(&bitC, &CState2, *--ip);
766
767 if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */
768 FSE_FLUSHBITS(&bitC);
769
770 FSE_encodeSymbol(&bitC, &CState1, *--ip);
771
772 if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */
773 FSE_encodeSymbol(&bitC, &CState2, *--ip);
774 FSE_encodeSymbol(&bitC, &CState1, *--ip);
775 }
776
777 FSE_FLUSHBITS(&bitC);
778 }
779
780 FSE_flushCState(&bitC, &CState2);
781 FSE_flushCState(&bitC, &CState1);
782 return BIT_closeCStream(&bitC);
783}
784
785size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct)
786{
787 unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
788
789 if (fast)
790 return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
791 else
792 return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
793}
794
795size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
diff --git a/lib/zstd/fse_decompress.c b/lib/zstd/fse_decompress.c
new file mode 100644
index 000000000000..a84300e5a013
--- /dev/null
+++ b/lib/zstd/fse_decompress.c
@@ -0,0 +1,332 @@
1/*
2 * FSE : Finite State Entropy decoder
3 * Copyright (C) 2013-2015, Yann Collet.
4 *
5 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * This program is free software; you can redistribute it and/or modify it under
31 * the terms of the GNU General Public License version 2 as published by the
32 * Free Software Foundation. This program is dual-licensed; you may select
33 * either version 2 of the GNU General Public License ("GPL") or BSD license
34 * ("BSD").
35 *
36 * You can contact the author at :
37 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
38 */
39
40/* **************************************************************
41* Compiler specifics
42****************************************************************/
43#define FORCE_INLINE static __always_inline
44
45/* **************************************************************
46* Includes
47****************************************************************/
48#include "bitstream.h"
49#include "fse.h"
50#include <linux/compiler.h>
51#include <linux/kernel.h>
52#include <linux/string.h> /* memcpy, memset */
53
54/* **************************************************************
55* Error Management
56****************************************************************/
57#define FSE_isError ERR_isError
58#define FSE_STATIC_ASSERT(c) \
59 { \
60 enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
61 } /* use only *after* variable declarations */
62
63/* check and forward error code */
64#define CHECK_F(f) \
65 { \
66 size_t const e = f; \
67 if (FSE_isError(e)) \
68 return e; \
69 }
70
71/* **************************************************************
72* Templates
73****************************************************************/
74/*
75 designed to be included
76 for type-specific functions (template emulation in C)
77 Objective is to write these functions only once, for improved maintenance
78*/
79
80/* safety checks */
81#ifndef FSE_FUNCTION_EXTENSION
82#error "FSE_FUNCTION_EXTENSION must be defined"
83#endif
84#ifndef FSE_FUNCTION_TYPE
85#error "FSE_FUNCTION_TYPE must be defined"
86#endif
87
88/* Function names */
89#define FSE_CAT(X, Y) X##Y
90#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
91#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
92
93/* Function templates */
94
95size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize)
96{
97 void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
98 FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr);
99 U16 *symbolNext = (U16 *)workspace;
100
101 U32 const maxSV1 = maxSymbolValue + 1;
102 U32 const tableSize = 1 << tableLog;
103 U32 highThreshold = tableSize - 1;
104
105 /* Sanity Checks */
106 if (workspaceSize < sizeof(U16) * (FSE_MAX_SYMBOL_VALUE + 1))
107 return ERROR(tableLog_tooLarge);
108 if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE)
109 return ERROR(maxSymbolValue_tooLarge);
110 if (tableLog > FSE_MAX_TABLELOG)
111 return ERROR(tableLog_tooLarge);
112
113 /* Init, lay down lowprob symbols */
114 {
115 FSE_DTableHeader DTableH;
116 DTableH.tableLog = (U16)tableLog;
117 DTableH.fastMode = 1;
118 {
119 S16 const largeLimit = (S16)(1 << (tableLog - 1));
120 U32 s;
121 for (s = 0; s < maxSV1; s++) {
122 if (normalizedCounter[s] == -1) {
123 tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
124 symbolNext[s] = 1;
125 } else {
126 if (normalizedCounter[s] >= largeLimit)
127 DTableH.fastMode = 0;
128 symbolNext[s] = normalizedCounter[s];
129 }
130 }
131 }
132 memcpy(dt, &DTableH, sizeof(DTableH));
133 }
134
135 /* Spread symbols */
136 {
137 U32 const tableMask = tableSize - 1;
138 U32 const step = FSE_TABLESTEP(tableSize);
139 U32 s, position = 0;
140 for (s = 0; s < maxSV1; s++) {
141 int i;
142 for (i = 0; i < normalizedCounter[s]; i++) {
143 tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
144 position = (position + step) & tableMask;
145 while (position > highThreshold)
146 position = (position + step) & tableMask; /* lowprob area */
147 }
148 }
149 if (position != 0)
150 return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
151 }
152
153 /* Build Decoding table */
154 {
155 U32 u;
156 for (u = 0; u < tableSize; u++) {
157 FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
158 U16 nextState = symbolNext[symbol]++;
159 tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState));
160 tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize);
161 }
162 }
163
164 return 0;
165}
166
167/*-*******************************************************
168* Decompression (Byte symbols)
169*********************************************************/
170size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue)
171{
172 void *ptr = dt;
173 FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
174 void *dPtr = dt + 1;
175 FSE_decode_t *const cell = (FSE_decode_t *)dPtr;
176
177 DTableH->tableLog = 0;
178 DTableH->fastMode = 0;
179
180 cell->newState = 0;
181 cell->symbol = symbolValue;
182 cell->nbBits = 0;
183
184 return 0;
185}
186
187size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits)
188{
189 void *ptr = dt;
190 FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
191 void *dPtr = dt + 1;
192 FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr;
193 const unsigned tableSize = 1 << nbBits;
194 const unsigned tableMask = tableSize - 1;
195 const unsigned maxSV1 = tableMask + 1;
196 unsigned s;
197
198 /* Sanity checks */
199 if (nbBits < 1)
200 return ERROR(GENERIC); /* min size */
201
202 /* Build Decoding Table */
203 DTableH->tableLog = (U16)nbBits;
204 DTableH->fastMode = 1;
205 for (s = 0; s < maxSV1; s++) {
206 dinfo[s].newState = 0;
207 dinfo[s].symbol = (BYTE)s;
208 dinfo[s].nbBits = (BYTE)nbBits;
209 }
210
211 return 0;
212}
213
214FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt,
215 const unsigned fast)
216{
217 BYTE *const ostart = (BYTE *)dst;
218 BYTE *op = ostart;
219 BYTE *const omax = op + maxDstSize;
220 BYTE *const olimit = omax - 3;
221
222 BIT_DStream_t bitD;
223 FSE_DState_t state1;
224 FSE_DState_t state2;
225
226 /* Init */
227 CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
228
229 FSE_initDState(&state1, &bitD, dt);
230 FSE_initDState(&state2, &bitD, dt);
231
232#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
233
234 /* 4 symbols per loop */
235 for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) {
236 op[0] = FSE_GETSYMBOL(&state1);
237
238 if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
239 BIT_reloadDStream(&bitD);
240
241 op[1] = FSE_GETSYMBOL(&state2);
242
243 if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
244 {
245 if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) {
246 op += 2;
247 break;
248 }
249 }
250
251 op[2] = FSE_GETSYMBOL(&state1);
252
253 if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
254 BIT_reloadDStream(&bitD);
255
256 op[3] = FSE_GETSYMBOL(&state2);
257 }
258
259 /* tail */
260 /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
261 while (1) {
262 if (op > (omax - 2))
263 return ERROR(dstSize_tooSmall);
264 *op++ = FSE_GETSYMBOL(&state1);
265 if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
266 *op++ = FSE_GETSYMBOL(&state2);
267 break;
268 }
269
270 if (op > (omax - 2))
271 return ERROR(dstSize_tooSmall);
272 *op++ = FSE_GETSYMBOL(&state2);
273 if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
274 *op++ = FSE_GETSYMBOL(&state1);
275 break;
276 }
277 }
278
279 return op - ostart;
280}
281
282size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)
283{
284 const void *ptr = dt;
285 const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr;
286 const U32 fastMode = DTableH->fastMode;
287
288 /* select fast mode (static) */
289 if (fastMode)
290 return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
291 return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
292}
293
294size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize)
295{
296 const BYTE *const istart = (const BYTE *)cSrc;
297 const BYTE *ip = istart;
298 unsigned tableLog;
299 unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
300 size_t NCountLength;
301
302 FSE_DTable *dt;
303 short *counting;
304 size_t spaceUsed32 = 0;
305
306 FSE_STATIC_ASSERT(sizeof(FSE_DTable) == sizeof(U32));
307
308 dt = (FSE_DTable *)((U32 *)workspace + spaceUsed32);
309 spaceUsed32 += FSE_DTABLE_SIZE_U32(maxLog);
310 counting = (short *)((U32 *)workspace + spaceUsed32);
311 spaceUsed32 += ALIGN(sizeof(short) * (FSE_MAX_SYMBOL_VALUE + 1), sizeof(U32)) >> 2;
312
313 if ((spaceUsed32 << 2) > workspaceSize)
314 return ERROR(tableLog_tooLarge);
315 workspace = (U32 *)workspace + spaceUsed32;
316 workspaceSize -= (spaceUsed32 << 2);
317
318 /* normal FSE decoding mode */
319 NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
320 if (FSE_isError(NCountLength))
321 return NCountLength;
322 // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining
323 // case : NCountLength==cSrcSize */
324 if (tableLog > maxLog)
325 return ERROR(tableLog_tooLarge);
326 ip += NCountLength;
327 cSrcSize -= NCountLength;
328
329 CHECK_F(FSE_buildDTable_wksp(dt, counting, maxSymbolValue, tableLog, workspace, workspaceSize));
330
331 return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, dt); /* always return, even if it is an error code */
332}
diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h
new file mode 100644
index 000000000000..2143da28d952
--- /dev/null
+++ b/lib/zstd/huf.h
@@ -0,0 +1,212 @@
1/*
2 * Huffman coder, part of New Generation Entropy library
3 * header file
4 * Copyright (C) 2013-2016, Yann Collet.
5 *
6 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above
15 * copyright notice, this list of conditions and the following disclaimer
16 * in the documentation and/or other materials provided with the
17 * distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * This program is free software; you can redistribute it and/or modify it under
32 * the terms of the GNU General Public License version 2 as published by the
33 * Free Software Foundation. This program is dual-licensed; you may select
34 * either version 2 of the GNU General Public License ("GPL") or BSD license
35 * ("BSD").
36 *
37 * You can contact the author at :
38 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
39 */
40#ifndef HUF_H_298734234
41#define HUF_H_298734234
42
43/* *** Dependencies *** */
44#include <linux/types.h> /* size_t */
45
46/* *** Tool functions *** */
47#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
48size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
49
50/* Error Management */
51unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
52
53/* *** Advanced function *** */
54
55/** HUF_compress4X_wksp() :
56* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
57size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
58 size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
59
60/* *** Dependencies *** */
61#include "mem.h" /* U32 */
62
63/* *** Constants *** */
64#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
65#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
66#define HUF_SYMBOLVALUE_MAX 255
67
68#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
69#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
70#error "HUF_TABLELOG_MAX is too large !"
71#endif
72
73/* ****************************************
74* Static allocation
75******************************************/
76/* HUF buffer bounds */
77#define HUF_CTABLEBOUND 129
78#define HUF_BLOCKBOUND(size) (size + (size >> 8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
79#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
80
81/* static allocation of HUF's Compression Table */
82#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
83 U32 name##hb[maxSymbolValue + 1]; \
84 void *name##hv = &(name##hb); \
85 HUF_CElt *name = (HUF_CElt *)(name##hv) /* no final ; */
86
87/* static allocation of HUF's DTable */
88typedef U32 HUF_DTable;
89#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog)))
90#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)}
91#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)}
92
93/* The workspace must have alignment at least 4 and be at least this large */
94#define HUF_COMPRESS_WORKSPACE_SIZE (6 << 10)
95#define HUF_COMPRESS_WORKSPACE_SIZE_U32 (HUF_COMPRESS_WORKSPACE_SIZE / sizeof(U32))
96
97/* The workspace must have alignment at least 4 and be at least this large */
98#define HUF_DECOMPRESS_WORKSPACE_SIZE (3 << 10)
99#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
100
101/* ****************************************
102* Advanced decompression functions
103******************************************/
104size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); /**< decodes RLE and uncompressed */
105size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
106 size_t workspaceSize); /**< considers RLE and uncompressed as errors */
107size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
108 size_t workspaceSize); /**< single-symbol decoder */
109size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
110 size_t workspaceSize); /**< double-symbols decoder */
111
112/* ****************************************
113* HUF detailed API
114******************************************/
115/*!
116HUF_compress() does the following:
1171. count symbol occurrence from source[] into table count[] using FSE_count()
1182. (optional) refine tableLog using HUF_optimalTableLog()
1193. build Huffman table from count using HUF_buildCTable()
1204. save Huffman table to memory buffer using HUF_writeCTable_wksp()
1215. encode the data stream using HUF_compress4X_usingCTable()
122
123The following API allows targeting specific sub-functions for advanced tasks.
124For example, it's possible to compress several blocks using the same 'CTable',
125or to save and regenerate 'CTable' using external methods.
126*/
127/* FSE_count() : find it within "fse.h" */
128unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
129typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
130size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, unsigned maxSymbolValue, unsigned huffLog, void *workspace, size_t workspaceSize);
131size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
132
133typedef enum {
134 HUF_repeat_none, /**< Cannot use the previous table */
135 HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
136 4}X_repeat */
137 HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
138} HUF_repeat;
139/** HUF_compress4X_repeat() :
140* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
141* If it uses hufTable it does not modify hufTable or repeat.
142* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
143* If preferRepeat then the old table will always be used if valid. */
144size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
145 size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
146 int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
147
148/** HUF_buildCTable_wksp() :
149 * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
150 * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
151 */
152size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize);
153
154/*! HUF_readStats() :
155 Read compact Huffman tree, saved by HUF_writeCTable().
156 `huffWeight` is destination buffer.
157 @return : size read from `src` , or an error Code .
158 Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
159size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize,
160 void *workspace, size_t workspaceSize);
161
162/** HUF_readCTable() :
163* Loading a CTable saved with HUF_writeCTable() */
164size_t HUF_readCTable_wksp(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
165
166/*
167HUF_decompress() does the following:
1681. select the decompression algorithm (X2, X4) based on pre-computed heuristics
1692. build Huffman table from save, using HUF_readDTableXn()
1703. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
171*/
172
173/** HUF_selectDecoder() :
174* Tells which decoder is likely to decode faster,
175* based on a set of pre-determined metrics.
176* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
177* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
178U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize);
179
180size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
181size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
182
183size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
184size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
185size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
186
187/* single stream variants */
188
189size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
190 size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
191size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
192/** HUF_compress1X_repeat() :
193* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
194* If it uses hufTable it does not modify hufTable or repeat.
195* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
196* If preferRepeat then the old table will always be used if valid. */
197size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
198 size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
199 int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
200
201size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize);
202size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
203 size_t workspaceSize); /**< single-symbol decoder */
204size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
205 size_t workspaceSize); /**< double-symbols decoder */
206
207size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize,
208 const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
209size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
210size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
211
212#endif /* HUF_H_298734234 */
diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c
new file mode 100644
index 000000000000..40055a7016e6
--- /dev/null
+++ b/lib/zstd/huf_compress.c
@@ -0,0 +1,770 @@
1/*
2 * Huffman encoder, part of New Generation Entropy library
3 * Copyright (C) 2013-2016, Yann Collet.
4 *
5 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * This program is free software; you can redistribute it and/or modify it under
31 * the terms of the GNU General Public License version 2 as published by the
32 * Free Software Foundation. This program is dual-licensed; you may select
33 * either version 2 of the GNU General Public License ("GPL") or BSD license
34 * ("BSD").
35 *
36 * You can contact the author at :
37 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
38 */
39
40/* **************************************************************
41* Includes
42****************************************************************/
43#include "bitstream.h"
44#include "fse.h" /* header compression */
45#include "huf.h"
46#include <linux/kernel.h>
47#include <linux/string.h> /* memcpy, memset */
48
49/* **************************************************************
50* Error Management
51****************************************************************/
52#define HUF_STATIC_ASSERT(c) \
53 { \
54 enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
55 } /* use only *after* variable declarations */
56#define CHECK_V_F(e, f) \
57 size_t const e = f; \
58 if (ERR_isError(e)) \
59 return f
60#define CHECK_F(f) \
61 { \
62 CHECK_V_F(_var_err__, f); \
63 }
64
65/* **************************************************************
66* Utils
67****************************************************************/
68unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
69{
70 return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
71}
72
73/* *******************************************************
74* HUF : Huffman block compression
75*********************************************************/
76/* HUF_compressWeights() :
77 * Same as FSE_compress(), but dedicated to huff0's weights compression.
78 * The use case needs much less stack memory.
79 * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
80 */
81#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
82size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize)
83{
84 BYTE *const ostart = (BYTE *)dst;
85 BYTE *op = ostart;
86 BYTE *const oend = ostart + dstSize;
87
88 U32 maxSymbolValue = HUF_TABLELOG_MAX;
89 U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
90
91 FSE_CTable *CTable;
92 U32 *count;
93 S16 *norm;
94 size_t spaceUsed32 = 0;
95
96 HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32));
97
98 CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32);
99 spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX);
100 count = (U32 *)workspace + spaceUsed32;
101 spaceUsed32 += HUF_TABLELOG_MAX + 1;
102 norm = (S16 *)((U32 *)workspace + spaceUsed32);
103 spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2;
104
105 if ((spaceUsed32 << 2) > workspaceSize)
106 return ERROR(tableLog_tooLarge);
107 workspace = (U32 *)workspace + spaceUsed32;
108 workspaceSize -= (spaceUsed32 << 2);
109
110 /* init conditions */
111 if (wtSize <= 1)
112 return 0; /* Not compressible */
113
114 /* Scan input and build symbol stats */
115 {
116 CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize));
117 if (maxCount == wtSize)
118 return 1; /* only a single symbol in src : rle */
119 if (maxCount == 1)
120 return 0; /* each symbol present maximum once => not compressible */
121 }
122
123 tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
124 CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue));
125
126 /* Write table description header */
127 {
128 CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
129 op += hSize;
130 }
131
132 /* Compress */
133 CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize));
134 {
135 CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable));
136 if (cSize == 0)
137 return 0; /* not enough space for compressed data */
138 op += cSize;
139 }
140
141 return op - ostart;
142}
143
144struct HUF_CElt_s {
145 U16 val;
146 BYTE nbBits;
147}; /* typedef'd to HUF_CElt within "huf.h" */
148
149/*! HUF_writeCTable_wksp() :
150 `CTable` : Huffman tree to save, using huf representation.
151 @return : size of saved CTable */
152size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog, void *workspace, size_t workspaceSize)
153{
154 BYTE *op = (BYTE *)dst;
155 U32 n;
156
157 BYTE *bitsToWeight;
158 BYTE *huffWeight;
159 size_t spaceUsed32 = 0;
160
161 bitsToWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
162 spaceUsed32 += ALIGN(HUF_TABLELOG_MAX + 1, sizeof(U32)) >> 2;
163 huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
164 spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX, sizeof(U32)) >> 2;
165
166 if ((spaceUsed32 << 2) > workspaceSize)
167 return ERROR(tableLog_tooLarge);
168 workspace = (U32 *)workspace + spaceUsed32;
169 workspaceSize -= (spaceUsed32 << 2);
170
171 /* check conditions */
172 if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
173 return ERROR(maxSymbolValue_tooLarge);
174
175 /* convert to weight */
176 bitsToWeight[0] = 0;
177 for (n = 1; n < huffLog + 1; n++)
178 bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
179 for (n = 0; n < maxSymbolValue; n++)
180 huffWeight[n] = bitsToWeight[CTable[n].nbBits];
181
182 /* attempt weights compression by FSE */
183 {
184 CHECK_V_F(hSize, HUF_compressWeights_wksp(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue, workspace, workspaceSize));
185 if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */
186 op[0] = (BYTE)hSize;
187 return hSize + 1;
188 }
189 }
190
191 /* write raw values as 4-bits (max : 15) */
192 if (maxSymbolValue > (256 - 128))
193 return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
194 if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize)
195 return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
196 op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1));
197 huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
198 for (n = 0; n < maxSymbolValue; n += 2)
199 op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]);
200 return ((maxSymbolValue + 1) / 2) + 1;
201}
202
203size_t HUF_readCTable_wksp(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
204{
205 U32 *rankVal;
206 BYTE *huffWeight;
207 U32 tableLog = 0;
208 U32 nbSymbols = 0;
209 size_t readSize;
210 size_t spaceUsed32 = 0;
211
212 rankVal = (U32 *)workspace + spaceUsed32;
213 spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
214 huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
215 spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
216
217 if ((spaceUsed32 << 2) > workspaceSize)
218 return ERROR(tableLog_tooLarge);
219 workspace = (U32 *)workspace + spaceUsed32;
220 workspaceSize -= (spaceUsed32 << 2);
221
222 /* get symbol weights */
223 readSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
224 if (ERR_isError(readSize))
225 return readSize;
226
227 /* check result */
228 if (tableLog > HUF_TABLELOG_MAX)
229 return ERROR(tableLog_tooLarge);
230 if (nbSymbols > maxSymbolValue + 1)
231 return ERROR(maxSymbolValue_tooSmall);
232
233 /* Prepare base value per rank */
234 {
235 U32 n, nextRankStart = 0;
236 for (n = 1; n <= tableLog; n++) {
237 U32 curr = nextRankStart;
238 nextRankStart += (rankVal[n] << (n - 1));
239 rankVal[n] = curr;
240 }
241 }
242
243 /* fill nbBits */
244 {
245 U32 n;
246 for (n = 0; n < nbSymbols; n++) {
247 const U32 w = huffWeight[n];
248 CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
249 }
250 }
251
252 /* fill val */
253 {
254 U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */
255 U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0};
256 {
257 U32 n;
258 for (n = 0; n < nbSymbols; n++)
259 nbPerRank[CTable[n].nbBits]++;
260 }
261 /* determine stating value per rank */
262 valPerRank[tableLog + 1] = 0; /* for w==0 */
263 {
264 U16 min = 0;
265 U32 n;
266 for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */
267 valPerRank[n] = min; /* get starting value within each rank */
268 min += nbPerRank[n];
269 min >>= 1;
270 }
271 }
272 /* assign value within rank, symbol order */
273 {
274 U32 n;
275 for (n = 0; n <= maxSymbolValue; n++)
276 CTable[n].val = valPerRank[CTable[n].nbBits]++;
277 }
278 }
279
280 return readSize;
281}
282
283typedef struct nodeElt_s {
284 U32 count;
285 U16 parent;
286 BYTE byte;
287 BYTE nbBits;
288} nodeElt;
289
290static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits)
291{
292 const U32 largestBits = huffNode[lastNonNull].nbBits;
293 if (largestBits <= maxNbBits)
294 return largestBits; /* early exit : no elt > maxNbBits */
295
296 /* there are several too large elements (at least >= 2) */
297 {
298 int totalCost = 0;
299 const U32 baseCost = 1 << (largestBits - maxNbBits);
300 U32 n = lastNonNull;
301
302 while (huffNode[n].nbBits > maxNbBits) {
303 totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
304 huffNode[n].nbBits = (BYTE)maxNbBits;
305 n--;
306 } /* n stops at huffNode[n].nbBits <= maxNbBits */
307 while (huffNode[n].nbBits == maxNbBits)
308 n--; /* n end at index of smallest symbol using < maxNbBits */
309
310 /* renorm totalCost */
311 totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
312
313 /* repay normalized cost */
314 {
315 U32 const noSymbol = 0xF0F0F0F0;
316 U32 rankLast[HUF_TABLELOG_MAX + 2];
317 int pos;
318
319 /* Get pos of last (smallest) symbol per rank */
320 memset(rankLast, 0xF0, sizeof(rankLast));
321 {
322 U32 currNbBits = maxNbBits;
323 for (pos = n; pos >= 0; pos--) {
324 if (huffNode[pos].nbBits >= currNbBits)
325 continue;
326 currNbBits = huffNode[pos].nbBits; /* < maxNbBits */
327 rankLast[maxNbBits - currNbBits] = pos;
328 }
329 }
330
331 while (totalCost > 0) {
332 U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
333 for (; nBitsToDecrease > 1; nBitsToDecrease--) {
334 U32 highPos = rankLast[nBitsToDecrease];
335 U32 lowPos = rankLast[nBitsToDecrease - 1];
336 if (highPos == noSymbol)
337 continue;
338 if (lowPos == noSymbol)
339 break;
340 {
341 U32 const highTotal = huffNode[highPos].count;
342 U32 const lowTotal = 2 * huffNode[lowPos].count;
343 if (highTotal <= lowTotal)
344 break;
345 }
346 }
347 /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
348 /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
349 while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))
350 nBitsToDecrease++;
351 totalCost -= 1 << (nBitsToDecrease - 1);
352 if (rankLast[nBitsToDecrease - 1] == noSymbol)
353 rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
354 huffNode[rankLast[nBitsToDecrease]].nbBits++;
355 if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
356 rankLast[nBitsToDecrease] = noSymbol;
357 else {
358 rankLast[nBitsToDecrease]--;
359 if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease)
360 rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
361 }
362 } /* while (totalCost > 0) */
363
364 while (totalCost < 0) { /* Sometimes, cost correction overshoot */
365 if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0
366 (using maxNbBits) */
367 while (huffNode[n].nbBits == maxNbBits)
368 n--;
369 huffNode[n + 1].nbBits--;
370 rankLast[1] = n + 1;
371 totalCost++;
372 continue;
373 }
374 huffNode[rankLast[1] + 1].nbBits--;
375 rankLast[1]++;
376 totalCost++;
377 }
378 }
379 } /* there are several too large elements (at least >= 2) */
380
381 return maxNbBits;
382}
383
384typedef struct {
385 U32 base;
386 U32 curr;
387} rankPos;
388
389static void HUF_sort(nodeElt *huffNode, const U32 *count, U32 maxSymbolValue)
390{
391 rankPos rank[32];
392 U32 n;
393
394 memset(rank, 0, sizeof(rank));
395 for (n = 0; n <= maxSymbolValue; n++) {
396 U32 r = BIT_highbit32(count[n] + 1);
397 rank[r].base++;
398 }
399 for (n = 30; n > 0; n--)
400 rank[n - 1].base += rank[n].base;
401 for (n = 0; n < 32; n++)
402 rank[n].curr = rank[n].base;
403 for (n = 0; n <= maxSymbolValue; n++) {
404 U32 const c = count[n];
405 U32 const r = BIT_highbit32(c + 1) + 1;
406 U32 pos = rank[r].curr++;
407 while ((pos > rank[r].base) && (c > huffNode[pos - 1].count))
408 huffNode[pos] = huffNode[pos - 1], pos--;
409 huffNode[pos].count = c;
410 huffNode[pos].byte = (BYTE)n;
411 }
412}
413
414/** HUF_buildCTable_wksp() :
415 * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
416 * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
417 */
418#define STARTNODE (HUF_SYMBOLVALUE_MAX + 1)
419typedef nodeElt huffNodeTable[2 * HUF_SYMBOLVALUE_MAX + 1 + 1];
420size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize)
421{
422 nodeElt *const huffNode0 = (nodeElt *)workSpace;
423 nodeElt *const huffNode = huffNode0 + 1;
424 U32 n, nonNullRank;
425 int lowS, lowN;
426 U16 nodeNb = STARTNODE;
427 U32 nodeRoot;
428
429 /* safety checks */
430 if (wkspSize < sizeof(huffNodeTable))
431 return ERROR(GENERIC); /* workSpace is not large enough */
432 if (maxNbBits == 0)
433 maxNbBits = HUF_TABLELOG_DEFAULT;
434 if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
435 return ERROR(GENERIC);
436 memset(huffNode0, 0, sizeof(huffNodeTable));
437
438 /* sort, decreasing order */
439 HUF_sort(huffNode, count, maxSymbolValue);
440
441 /* init for parents */
442 nonNullRank = maxSymbolValue;
443 while (huffNode[nonNullRank].count == 0)
444 nonNullRank--;
445 lowS = nonNullRank;
446 nodeRoot = nodeNb + lowS - 1;
447 lowN = nodeNb;
448 huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count;
449 huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb;
450 nodeNb++;
451 lowS -= 2;
452 for (n = nodeNb; n <= nodeRoot; n++)
453 huffNode[n].count = (U32)(1U << 30);
454 huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */
455
456 /* create parents */
457 while (nodeNb <= nodeRoot) {
458 U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
459 U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
460 huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
461 huffNode[n1].parent = huffNode[n2].parent = nodeNb;
462 nodeNb++;
463 }
464
465 /* distribute weights (unlimited tree height) */
466 huffNode[nodeRoot].nbBits = 0;
467 for (n = nodeRoot - 1; n >= STARTNODE; n--)
468 huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
469 for (n = 0; n <= nonNullRank; n++)
470 huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
471
472 /* enforce maxTableLog */
473 maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
474
475 /* fill result into tree (val, nbBits) */
476 {
477 U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0};
478 U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0};
479 if (maxNbBits > HUF_TABLELOG_MAX)
480 return ERROR(GENERIC); /* check fit into table */
481 for (n = 0; n <= nonNullRank; n++)
482 nbPerRank[huffNode[n].nbBits]++;
483 /* determine stating value per rank */
484 {
485 U16 min = 0;
486 for (n = maxNbBits; n > 0; n--) {
487 valPerRank[n] = min; /* get starting value within each rank */
488 min += nbPerRank[n];
489 min >>= 1;
490 }
491 }
492 for (n = 0; n <= maxSymbolValue; n++)
493 tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
494 for (n = 0; n <= maxSymbolValue; n++)
495 tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
496 }
497
498 return maxNbBits;
499}
500
501static size_t HUF_estimateCompressedSize(HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
502{
503 size_t nbBits = 0;
504 int s;
505 for (s = 0; s <= (int)maxSymbolValue; ++s) {
506 nbBits += CTable[s].nbBits * count[s];
507 }
508 return nbBits >> 3;
509}
510
511static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
512{
513 int bad = 0;
514 int s;
515 for (s = 0; s <= (int)maxSymbolValue; ++s) {
516 bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
517 }
518 return !bad;
519}
520
521static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt *CTable)
522{
523 BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
524}
525
526size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
527
528#define HUF_FLUSHBITS(s) BIT_flushBits(s)
529
530#define HUF_FLUSHBITS_1(stream) \
531 if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
532 HUF_FLUSHBITS(stream)
533
534#define HUF_FLUSHBITS_2(stream) \
535 if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \
536 HUF_FLUSHBITS(stream)
537
538size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
539{
540 const BYTE *ip = (const BYTE *)src;
541 BYTE *const ostart = (BYTE *)dst;
542 BYTE *const oend = ostart + dstSize;
543 BYTE *op = ostart;
544 size_t n;
545 BIT_CStream_t bitC;
546
547 /* init */
548 if (dstSize < 8)
549 return 0; /* not enough space to compress */
550 {
551 size_t const initErr = BIT_initCStream(&bitC, op, oend - op);
552 if (HUF_isError(initErr))
553 return 0;
554 }
555
556 n = srcSize & ~3; /* join to mod 4 */
557 switch (srcSize & 3) {
558 case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC);
559 case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC);
560 case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC);
561 case 0:
562 default:;
563 }
564
565 for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */
566 HUF_encodeSymbol(&bitC, ip[n - 1], CTable);
567 HUF_FLUSHBITS_1(&bitC);
568 HUF_encodeSymbol(&bitC, ip[n - 2], CTable);
569 HUF_FLUSHBITS_2(&bitC);
570 HUF_encodeSymbol(&bitC, ip[n - 3], CTable);
571 HUF_FLUSHBITS_1(&bitC);
572 HUF_encodeSymbol(&bitC, ip[n - 4], CTable);
573 HUF_FLUSHBITS(&bitC);
574 }
575
576 return BIT_closeCStream(&bitC);
577}
578
579size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
580{
581 size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */
582 const BYTE *ip = (const BYTE *)src;
583 const BYTE *const iend = ip + srcSize;
584 BYTE *const ostart = (BYTE *)dst;
585 BYTE *const oend = ostart + dstSize;
586 BYTE *op = ostart;
587
588 if (dstSize < 6 + 1 + 1 + 1 + 8)
589 return 0; /* minimum space to compress successfully */
590 if (srcSize < 12)
591 return 0; /* no saving possible : too small input */
592 op += 6; /* jumpTable */
593
594 {
595 CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
596 if (cSize == 0)
597 return 0;
598 ZSTD_writeLE16(ostart, (U16)cSize);
599 op += cSize;
600 }
601
602 ip += segmentSize;
603 {
604 CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
605 if (cSize == 0)
606 return 0;
607 ZSTD_writeLE16(ostart + 2, (U16)cSize);
608 op += cSize;
609 }
610
611 ip += segmentSize;
612 {
613 CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
614 if (cSize == 0)
615 return 0;
616 ZSTD_writeLE16(ostart + 4, (U16)cSize);
617 op += cSize;
618 }
619
620 ip += segmentSize;
621 {
622 CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable));
623 if (cSize == 0)
624 return 0;
625 op += cSize;
626 }
627
628 return op - ostart;
629}
630
631static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream,
632 const HUF_CElt *CTable)
633{
634 size_t const cSize =
635 singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
636 if (HUF_isError(cSize)) {
637 return cSize;
638 }
639 if (cSize == 0) {
640 return 0;
641 } /* uncompressible */
642 op += cSize;
643 /* check compressibility */
644 if ((size_t)(op - ostart) >= srcSize - 1) {
645 return 0;
646 }
647 return op - ostart;
648}
649
650/* `workSpace` must a table of at least 1024 unsigned */
651static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog,
652 unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat)
653{
654 BYTE *const ostart = (BYTE *)dst;
655 BYTE *const oend = ostart + dstSize;
656 BYTE *op = ostart;
657
658 U32 *count;
659 size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
660 HUF_CElt *CTable;
661 size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
662
663 /* checks & inits */
664 if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize)
665 return ERROR(GENERIC);
666 if (!srcSize)
667 return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
668 if (!dstSize)
669 return 0; /* cannot fit within dst budget */
670 if (srcSize > HUF_BLOCKSIZE_MAX)
671 return ERROR(srcSize_wrong); /* curr block size limit */
672 if (huffLog > HUF_TABLELOG_MAX)
673 return ERROR(tableLog_tooLarge);
674 if (!maxSymbolValue)
675 maxSymbolValue = HUF_SYMBOLVALUE_MAX;
676 if (!huffLog)
677 huffLog = HUF_TABLELOG_DEFAULT;
678
679 count = (U32 *)workSpace;
680 workSpace = (BYTE *)workSpace + countSize;
681 wkspSize -= countSize;
682 CTable = (HUF_CElt *)workSpace;
683 workSpace = (BYTE *)workSpace + CTableSize;
684 wkspSize -= CTableSize;
685
686 /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
687 if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
688 return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
689 }
690
691 /* Scan input and build symbol stats */
692 {
693 CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace));
694 if (largest == srcSize) {
695 *ostart = ((const BYTE *)src)[0];
696 return 1;
697 } /* single symbol, rle */
698 if (largest <= (srcSize >> 7) + 1)
699 return 0; /* Fast heuristic : not compressible enough */
700 }
701
702 /* Check validity of previous table */
703 if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
704 *repeat = HUF_repeat_none;
705 }
706 /* Heuristic : use existing table for small inputs */
707 if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
708 return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
709 }
710
711 /* Build Huffman Tree */
712 huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
713 {
714 CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize));
715 huffLog = (U32)maxBits;
716 /* Zero the unused symbols so we can check it for validity */
717 memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
718 }
719
720 /* Write table description header */
721 {
722 CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, CTable, maxSymbolValue, huffLog, workSpace, wkspSize));
723 /* Check if using the previous table will be beneficial */
724 if (repeat && *repeat != HUF_repeat_none) {
725 size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
726 size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
727 if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
728 return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
729 }
730 }
731 /* Use the new table */
732 if (hSize + 12ul >= srcSize) {
733 return 0;
734 }
735 op += hSize;
736 if (repeat) {
737 *repeat = HUF_repeat_none;
738 }
739 if (oldHufTable) {
740 memcpy(oldHufTable, CTable, CTableSize);
741 } /* Save the new table */
742 }
743 return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
744}
745
746size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
747 size_t wkspSize)
748{
749 return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
750}
751
752size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
753 size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
754{
755 return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat,
756 preferRepeat);
757}
758
759size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
760 size_t wkspSize)
761{
762 return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
763}
764
765size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
766 size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
767{
768 return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat,
769 preferRepeat);
770}
diff --git a/lib/zstd/huf_decompress.c b/lib/zstd/huf_decompress.c
new file mode 100644
index 000000000000..6526482047dc
--- /dev/null
+++ b/lib/zstd/huf_decompress.c
@@ -0,0 +1,960 @@
1/*
2 * Huffman decoder, part of New Generation Entropy library
3 * Copyright (C) 2013-2016, Yann Collet.
4 *
5 * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * This program is free software; you can redistribute it and/or modify it under
31 * the terms of the GNU General Public License version 2 as published by the
32 * Free Software Foundation. This program is dual-licensed; you may select
33 * either version 2 of the GNU General Public License ("GPL") or BSD license
34 * ("BSD").
35 *
36 * You can contact the author at :
37 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
38 */
39
40/* **************************************************************
41* Compiler specifics
42****************************************************************/
43#define FORCE_INLINE static __always_inline
44
45/* **************************************************************
46* Dependencies
47****************************************************************/
48#include "bitstream.h" /* BIT_* */
49#include "fse.h" /* header compression */
50#include "huf.h"
51#include <linux/compiler.h>
52#include <linux/kernel.h>
53#include <linux/string.h> /* memcpy, memset */
54
55/* **************************************************************
56* Error Management
57****************************************************************/
58#define HUF_STATIC_ASSERT(c) \
59 { \
60 enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
61 } /* use only *after* variable declarations */
62
63/*-***************************/
64/* generic DTableDesc */
65/*-***************************/
66
67typedef struct {
68 BYTE maxTableLog;
69 BYTE tableType;
70 BYTE tableLog;
71 BYTE reserved;
72} DTableDesc;
73
74static DTableDesc HUF_getDTableDesc(const HUF_DTable *table)
75{
76 DTableDesc dtd;
77 memcpy(&dtd, table, sizeof(dtd));
78 return dtd;
79}
80
81/*-***************************/
82/* single-symbol decoding */
83/*-***************************/
84
85typedef struct {
86 BYTE byte;
87 BYTE nbBits;
88} HUF_DEltX2; /* single-symbol decoding */
89
90size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
91{
92 U32 tableLog = 0;
93 U32 nbSymbols = 0;
94 size_t iSize;
95 void *const dtPtr = DTable + 1;
96 HUF_DEltX2 *const dt = (HUF_DEltX2 *)dtPtr;
97
98 U32 *rankVal;
99 BYTE *huffWeight;
100 size_t spaceUsed32 = 0;
101
102 rankVal = (U32 *)workspace + spaceUsed32;
103 spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
104 huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
105 spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
106
107 if ((spaceUsed32 << 2) > workspaceSize)
108 return ERROR(tableLog_tooLarge);
109 workspace = (U32 *)workspace + spaceUsed32;
110 workspaceSize -= (spaceUsed32 << 2);
111
112 HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
113 /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
114
115 iSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
116 if (HUF_isError(iSize))
117 return iSize;
118
119 /* Table header */
120 {
121 DTableDesc dtd = HUF_getDTableDesc(DTable);
122 if (tableLog > (U32)(dtd.maxTableLog + 1))
123 return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
124 dtd.tableType = 0;
125 dtd.tableLog = (BYTE)tableLog;
126 memcpy(DTable, &dtd, sizeof(dtd));
127 }
128
129 /* Calculate starting value for each rank */
130 {
131 U32 n, nextRankStart = 0;
132 for (n = 1; n < tableLog + 1; n++) {
133 U32 const curr = nextRankStart;
134 nextRankStart += (rankVal[n] << (n - 1));
135 rankVal[n] = curr;
136 }
137 }
138
139 /* fill DTable */
140 {
141 U32 n;
142 for (n = 0; n < nbSymbols; n++) {
143 U32 const w = huffWeight[n];
144 U32 const length = (1 << w) >> 1;
145 U32 u;
146 HUF_DEltX2 D;
147 D.byte = (BYTE)n;
148 D.nbBits = (BYTE)(tableLog + 1 - w);
149 for (u = rankVal[w]; u < rankVal[w] + length; u++)
150 dt[u] = D;
151 rankVal[w] += length;
152 }
153 }
154
155 return iSize;
156}
157
158static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog)
159{
160 size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
161 BYTE const c = dt[val].byte;
162 BIT_skipBits(Dstream, dt[val].nbBits);
163 return c;
164}
165
166#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
167
168#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
169 if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
170 HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
171
172#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
173 if (ZSTD_64bits()) \
174 HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
175
176FORCE_INLINE size_t HUF_decodeStreamX2(BYTE *p, BIT_DStream_t *const bitDPtr, BYTE *const pEnd, const HUF_DEltX2 *const dt, const U32 dtLog)
177{
178 BYTE *const pStart = p;
179
180 /* up to 4 symbols at a time */
181 while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd - 4)) {
182 HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
183 HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
184 HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
185 HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
186 }
187
188 /* closer to the end */
189 while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd))
190 HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
191
192 /* no more data to retrieve from bitstream, hence no need to reload */
193 while (p < pEnd)
194 HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
195
196 return pEnd - pStart;
197}
198
199static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
200{
201 BYTE *op = (BYTE *)dst;
202 BYTE *const oend = op + dstSize;
203 const void *dtPtr = DTable + 1;
204 const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
205 BIT_DStream_t bitD;
206 DTableDesc const dtd = HUF_getDTableDesc(DTable);
207 U32 const dtLog = dtd.tableLog;
208
209 {
210 size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
211 if (HUF_isError(errorCode))
212 return errorCode;
213 }
214
215 HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog);
216
217 /* check */
218 if (!BIT_endOfDStream(&bitD))
219 return ERROR(corruption_detected);
220
221 return dstSize;
222}
223
224size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
225{
226 DTableDesc dtd = HUF_getDTableDesc(DTable);
227 if (dtd.tableType != 0)
228 return ERROR(GENERIC);
229 return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
230}
231
232size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
233{
234 const BYTE *ip = (const BYTE *)cSrc;
235
236 size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize);
237 if (HUF_isError(hSize))
238 return hSize;
239 if (hSize >= cSrcSize)
240 return ERROR(srcSize_wrong);
241 ip += hSize;
242 cSrcSize -= hSize;
243
244 return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
245}
246
247static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
248{
249 /* Check */
250 if (cSrcSize < 10)
251 return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
252
253 {
254 const BYTE *const istart = (const BYTE *)cSrc;
255 BYTE *const ostart = (BYTE *)dst;
256 BYTE *const oend = ostart + dstSize;
257 const void *const dtPtr = DTable + 1;
258 const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
259
260 /* Init */
261 BIT_DStream_t bitD1;
262 BIT_DStream_t bitD2;
263 BIT_DStream_t bitD3;
264 BIT_DStream_t bitD4;
265 size_t const length1 = ZSTD_readLE16(istart);
266 size_t const length2 = ZSTD_readLE16(istart + 2);
267 size_t const length3 = ZSTD_readLE16(istart + 4);
268 size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
269 const BYTE *const istart1 = istart + 6; /* jumpTable */
270 const BYTE *const istart2 = istart1 + length1;
271 const BYTE *const istart3 = istart2 + length2;
272 const BYTE *const istart4 = istart3 + length3;
273 const size_t segmentSize = (dstSize + 3) / 4;
274 BYTE *const opStart2 = ostart + segmentSize;
275 BYTE *const opStart3 = opStart2 + segmentSize;
276 BYTE *const opStart4 = opStart3 + segmentSize;
277 BYTE *op1 = ostart;
278 BYTE *op2 = opStart2;
279 BYTE *op3 = opStart3;
280 BYTE *op4 = opStart4;
281 U32 endSignal;
282 DTableDesc const dtd = HUF_getDTableDesc(DTable);
283 U32 const dtLog = dtd.tableLog;
284
285 if (length4 > cSrcSize)
286 return ERROR(corruption_detected); /* overflow */
287 {
288 size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
289 if (HUF_isError(errorCode))
290 return errorCode;
291 }
292 {
293 size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
294 if (HUF_isError(errorCode))
295 return errorCode;
296 }
297 {
298 size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
299 if (HUF_isError(errorCode))
300 return errorCode;
301 }
302 {
303 size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
304 if (HUF_isError(errorCode))
305 return errorCode;
306 }
307
308 /* 16-32 symbols per loop (4-8 symbols per stream) */
309 endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
310 for (; (endSignal == BIT_DStream_unfinished) && (op4 < (oend - 7));) {
311 HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
312 HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
313 HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
314 HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
315 HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
316 HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
317 HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
318 HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
319 HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
320 HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
321 HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
322 HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
323 HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
324 HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
325 HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
326 HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
327 endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
328 }
329
330 /* check corruption */
331 if (op1 > opStart2)
332 return ERROR(corruption_detected);
333 if (op2 > opStart3)
334 return ERROR(corruption_detected);
335 if (op3 > opStart4)
336 return ERROR(corruption_detected);
337 /* note : op4 supposed already verified within main loop */
338
339 /* finish bitStreams one by one */
340 HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
341 HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
342 HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
343 HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
344
345 /* check */
346 endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
347 if (!endSignal)
348 return ERROR(corruption_detected);
349
350 /* decoded size */
351 return dstSize;
352 }
353}
354
355size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
356{
357 DTableDesc dtd = HUF_getDTableDesc(DTable);
358 if (dtd.tableType != 0)
359 return ERROR(GENERIC);
360 return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
361}
362
363size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
364{
365 const BYTE *ip = (const BYTE *)cSrc;
366
367 size_t const hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize);
368 if (HUF_isError(hSize))
369 return hSize;
370 if (hSize >= cSrcSize)
371 return ERROR(srcSize_wrong);
372 ip += hSize;
373 cSrcSize -= hSize;
374
375 return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
376}
377
378/* *************************/
379/* double-symbols decoding */
380/* *************************/
381typedef struct {
382 U16 sequence;
383 BYTE nbBits;
384 BYTE length;
385} HUF_DEltX4; /* double-symbols decoding */
386
387typedef struct {
388 BYTE symbol;
389 BYTE weight;
390} sortedSymbol_t;
391
392/* HUF_fillDTableX4Level2() :
393 * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
394static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight,
395 const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq)
396{
397 HUF_DEltX4 DElt;
398 U32 rankVal[HUF_TABLELOG_MAX + 1];
399
400 /* get pre-calculated rankVal */
401 memcpy(rankVal, rankValOrigin, sizeof(rankVal));
402
403 /* fill skipped values */
404 if (minWeight > 1) {
405 U32 i, skipSize = rankVal[minWeight];
406 ZSTD_writeLE16(&(DElt.sequence), baseSeq);
407 DElt.nbBits = (BYTE)(consumed);
408 DElt.length = 1;
409 for (i = 0; i < skipSize; i++)
410 DTable[i] = DElt;
411 }
412
413 /* fill DTable */
414 {
415 U32 s;
416 for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */
417 const U32 symbol = sortedSymbols[s].symbol;
418 const U32 weight = sortedSymbols[s].weight;
419 const U32 nbBits = nbBitsBaseline - weight;
420 const U32 length = 1 << (sizeLog - nbBits);
421 const U32 start = rankVal[weight];
422 U32 i = start;
423 const U32 end = start + length;
424
425 ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
426 DElt.nbBits = (BYTE)(nbBits + consumed);
427 DElt.length = 2;
428 do {
429 DTable[i++] = DElt;
430 } while (i < end); /* since length >= 1 */
431
432 rankVal[weight] += length;
433 }
434 }
435}
436
437typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
438typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
439
440static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart,
441 rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline)
442{
443 U32 rankVal[HUF_TABLELOG_MAX + 1];
444 const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
445 const U32 minBits = nbBitsBaseline - maxWeight;
446 U32 s;
447
448 memcpy(rankVal, rankValOrigin, sizeof(rankVal));
449
450 /* fill DTable */
451 for (s = 0; s < sortedListSize; s++) {
452 const U16 symbol = sortedList[s].symbol;
453 const U32 weight = sortedList[s].weight;
454 const U32 nbBits = nbBitsBaseline - weight;
455 const U32 start = rankVal[weight];
456 const U32 length = 1 << (targetLog - nbBits);
457
458 if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */
459 U32 sortedRank;
460 int minWeight = nbBits + scaleLog;
461 if (minWeight < 1)
462 minWeight = 1;
463 sortedRank = rankStart[minWeight];
464 HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank,
465 sortedListSize - sortedRank, nbBitsBaseline, symbol);
466 } else {
467 HUF_DEltX4 DElt;
468 ZSTD_writeLE16(&(DElt.sequence), symbol);
469 DElt.nbBits = (BYTE)(nbBits);
470 DElt.length = 1;
471 {
472 U32 const end = start + length;
473 U32 u;
474 for (u = start; u < end; u++)
475 DTable[u] = DElt;
476 }
477 }
478 rankVal[weight] += length;
479 }
480}
481
482size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
483{
484 U32 tableLog, maxW, sizeOfSort, nbSymbols;
485 DTableDesc dtd = HUF_getDTableDesc(DTable);
486 U32 const maxTableLog = dtd.maxTableLog;
487 size_t iSize;
488 void *dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */
489 HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr;
490 U32 *rankStart;
491
492 rankValCol_t *rankVal;
493 U32 *rankStats;
494 U32 *rankStart0;
495 sortedSymbol_t *sortedSymbol;
496 BYTE *weightList;
497 size_t spaceUsed32 = 0;
498
499 HUF_STATIC_ASSERT((sizeof(rankValCol_t) & 3) == 0);
500
501 rankVal = (rankValCol_t *)((U32 *)workspace + spaceUsed32);
502 spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2;
503 rankStats = (U32 *)workspace + spaceUsed32;
504 spaceUsed32 += HUF_TABLELOG_MAX + 1;
505 rankStart0 = (U32 *)workspace + spaceUsed32;
506 spaceUsed32 += HUF_TABLELOG_MAX + 2;
507 sortedSymbol = (sortedSymbol_t *)((U32 *)workspace + spaceUsed32);
508 spaceUsed32 += ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2;
509 weightList = (BYTE *)((U32 *)workspace + spaceUsed32);
510 spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
511
512 if ((spaceUsed32 << 2) > workspaceSize)
513 return ERROR(tableLog_tooLarge);
514 workspace = (U32 *)workspace + spaceUsed32;
515 workspaceSize -= (spaceUsed32 << 2);
516
517 rankStart = rankStart0 + 1;
518 memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1));
519
520 HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
521 if (maxTableLog > HUF_TABLELOG_MAX)
522 return ERROR(tableLog_tooLarge);
523 /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
524
525 iSize = HUF_readStats_wksp(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
526 if (HUF_isError(iSize))
527 return iSize;
528
529 /* check result */
530 if (tableLog > maxTableLog)
531 return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
532
533 /* find maxWeight */
534 for (maxW = tableLog; rankStats[maxW] == 0; maxW--) {
535 } /* necessarily finds a solution before 0 */
536
537 /* Get start index of each weight */
538 {
539 U32 w, nextRankStart = 0;
540 for (w = 1; w < maxW + 1; w++) {
541 U32 curr = nextRankStart;
542 nextRankStart += rankStats[w];
543 rankStart[w] = curr;
544 }
545 rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
546 sizeOfSort = nextRankStart;
547 }
548
549 /* sort symbols by weight */
550 {
551 U32 s;
552 for (s = 0; s < nbSymbols; s++) {
553 U32 const w = weightList[s];
554 U32 const r = rankStart[w]++;
555 sortedSymbol[r].symbol = (BYTE)s;
556 sortedSymbol[r].weight = (BYTE)w;
557 }
558 rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
559 }
560
561 /* Build rankVal */
562 {
563 U32 *const rankVal0 = rankVal[0];
564 {
565 int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */
566 U32 nextRankVal = 0;
567 U32 w;
568 for (w = 1; w < maxW + 1; w++) {
569 U32 curr = nextRankVal;
570 nextRankVal += rankStats[w] << (w + rescale);
571 rankVal0[w] = curr;
572 }
573 }
574 {
575 U32 const minBits = tableLog + 1 - maxW;
576 U32 consumed;
577 for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
578 U32 *const rankValPtr = rankVal[consumed];
579 U32 w;
580 for (w = 1; w < maxW + 1; w++) {
581 rankValPtr[w] = rankVal0[w] >> consumed;
582 }
583 }
584 }
585 }
586
587 HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1);
588
589 dtd.tableLog = (BYTE)maxTableLog;
590 dtd.tableType = 1;
591 memcpy(DTable, &dtd, sizeof(dtd));
592 return iSize;
593}
594
595static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
596{
597 size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
598 memcpy(op, dt + val, 2);
599 BIT_skipBits(DStream, dt[val].nbBits);
600 return dt[val].length;
601}
602
603static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
604{
605 size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
606 memcpy(op, dt + val, 1);
607 if (dt[val].length == 1)
608 BIT_skipBits(DStream, dt[val].nbBits);
609 else {
610 if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) {
611 BIT_skipBits(DStream, dt[val].nbBits);
612 if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8))
613 /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
614 DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8);
615 }
616 }
617 return 1;
618}
619
620#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
621
622#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
623 if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
624 ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
625
626#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
627 if (ZSTD_64bits()) \
628 ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
629
630FORCE_INLINE size_t HUF_decodeStreamX4(BYTE *p, BIT_DStream_t *bitDPtr, BYTE *const pEnd, const HUF_DEltX4 *const dt, const U32 dtLog)
631{
632 BYTE *const pStart = p;
633
634 /* up to 8 symbols at a time */
635 while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) {
636 HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
637 HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
638 HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
639 HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
640 }
641
642 /* closer to end : up to 2 symbols at a time */
643 while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2))
644 HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
645
646 while (p <= pEnd - 2)
647 HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
648
649 if (p < pEnd)
650 p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
651
652 return p - pStart;
653}
654
655static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
656{
657 BIT_DStream_t bitD;
658
659 /* Init */
660 {
661 size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
662 if (HUF_isError(errorCode))
663 return errorCode;
664 }
665
666 /* decode */
667 {
668 BYTE *const ostart = (BYTE *)dst;
669 BYTE *const oend = ostart + dstSize;
670 const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */
671 const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
672 DTableDesc const dtd = HUF_getDTableDesc(DTable);
673 HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
674 }
675
676 /* check */
677 if (!BIT_endOfDStream(&bitD))
678 return ERROR(corruption_detected);
679
680 /* decoded size */
681 return dstSize;
682}
683
684size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
685{
686 DTableDesc dtd = HUF_getDTableDesc(DTable);
687 if (dtd.tableType != 1)
688 return ERROR(GENERIC);
689 return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
690}
691
692size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
693{
694 const BYTE *ip = (const BYTE *)cSrc;
695
696 size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize);
697 if (HUF_isError(hSize))
698 return hSize;
699 if (hSize >= cSrcSize)
700 return ERROR(srcSize_wrong);
701 ip += hSize;
702 cSrcSize -= hSize;
703
704 return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
705}
706
707static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
708{
709 if (cSrcSize < 10)
710 return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
711
712 {
713 const BYTE *const istart = (const BYTE *)cSrc;
714 BYTE *const ostart = (BYTE *)dst;
715 BYTE *const oend = ostart + dstSize;
716 const void *const dtPtr = DTable + 1;
717 const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
718
719 /* Init */
720 BIT_DStream_t bitD1;
721 BIT_DStream_t bitD2;
722 BIT_DStream_t bitD3;
723 BIT_DStream_t bitD4;
724 size_t const length1 = ZSTD_readLE16(istart);
725 size_t const length2 = ZSTD_readLE16(istart + 2);
726 size_t const length3 = ZSTD_readLE16(istart + 4);
727 size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
728 const BYTE *const istart1 = istart + 6; /* jumpTable */
729 const BYTE *const istart2 = istart1 + length1;
730 const BYTE *const istart3 = istart2 + length2;
731 const BYTE *const istart4 = istart3 + length3;
732 size_t const segmentSize = (dstSize + 3) / 4;
733 BYTE *const opStart2 = ostart + segmentSize;
734 BYTE *const opStart3 = opStart2 + segmentSize;
735 BYTE *const opStart4 = opStart3 + segmentSize;
736 BYTE *op1 = ostart;
737 BYTE *op2 = opStart2;
738 BYTE *op3 = opStart3;
739 BYTE *op4 = opStart4;
740 U32 endSignal;
741 DTableDesc const dtd = HUF_getDTableDesc(DTable);
742 U32 const dtLog = dtd.tableLog;
743
744 if (length4 > cSrcSize)
745 return ERROR(corruption_detected); /* overflow */
746 {
747 size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
748 if (HUF_isError(errorCode))
749 return errorCode;
750 }
751 {
752 size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
753 if (HUF_isError(errorCode))
754 return errorCode;
755 }
756 {
757 size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
758 if (HUF_isError(errorCode))
759 return errorCode;
760 }
761 {
762 size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
763 if (HUF_isError(errorCode))
764 return errorCode;
765 }
766
767 /* 16-32 symbols per loop (4-8 symbols per stream) */
768 endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
769 for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) {
770 HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
771 HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
772 HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
773 HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
774 HUF_DECODE_SYMBOLX4_1(op1, &bitD1);
775 HUF_DECODE_SYMBOLX4_1(op2, &bitD2);
776 HUF_DECODE_SYMBOLX4_1(op3, &bitD3);
777 HUF_DECODE_SYMBOLX4_1(op4, &bitD4);
778 HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
779 HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
780 HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
781 HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
782 HUF_DECODE_SYMBOLX4_0(op1, &bitD1);
783 HUF_DECODE_SYMBOLX4_0(op2, &bitD2);
784 HUF_DECODE_SYMBOLX4_0(op3, &bitD3);
785 HUF_DECODE_SYMBOLX4_0(op4, &bitD4);
786
787 endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
788 }
789
790 /* check corruption */
791 if (op1 > opStart2)
792 return ERROR(corruption_detected);
793 if (op2 > opStart3)
794 return ERROR(corruption_detected);
795 if (op3 > opStart4)
796 return ERROR(corruption_detected);
797 /* note : op4 already verified within main loop */
798
799 /* finish bitStreams one by one */
800 HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
801 HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
802 HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
803 HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
804
805 /* check */
806 {
807 U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
808 if (!endCheck)
809 return ERROR(corruption_detected);
810 }
811
812 /* decoded size */
813 return dstSize;
814 }
815}
816
817size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
818{
819 DTableDesc dtd = HUF_getDTableDesc(DTable);
820 if (dtd.tableType != 1)
821 return ERROR(GENERIC);
822 return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
823}
824
825size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
826{
827 const BYTE *ip = (const BYTE *)cSrc;
828
829 size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize);
830 if (HUF_isError(hSize))
831 return hSize;
832 if (hSize >= cSrcSize)
833 return ERROR(srcSize_wrong);
834 ip += hSize;
835 cSrcSize -= hSize;
836
837 return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
838}
839
840/* ********************************/
841/* Generic decompression selector */
842/* ********************************/
843
844size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
845{
846 DTableDesc const dtd = HUF_getDTableDesc(DTable);
847 return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
848 : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
849}
850
851size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
852{
853 DTableDesc const dtd = HUF_getDTableDesc(DTable);
854 return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
855 : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
856}
857
858typedef struct {
859 U32 tableTime;
860 U32 decode256Time;
861} algo_time_t;
862static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = {
863 /* single, double, quad */
864 {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */
865 {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */
866 {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
867 {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
868 {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
869 {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
870 {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
871 {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
872 {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
873 {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
874 {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
875 {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
876 {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
877 {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */
878 {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */
879 {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */
880};
881
882/** HUF_selectDecoder() :
883* Tells which decoder is likely to decode faster,
884* based on a set of pre-determined metrics.
885* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
886* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
887U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize)
888{
889 /* decoder timing evaluation */
890 U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
891 U32 const D256 = (U32)(dstSize >> 8);
892 U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
893 U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
894 DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
895
896 return DTime1 < DTime0;
897}
898
899typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize);
900
901size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
902{
903 /* validation checks */
904 if (dstSize == 0)
905 return ERROR(dstSize_tooSmall);
906 if (cSrcSize > dstSize)
907 return ERROR(corruption_detected); /* invalid */
908 if (cSrcSize == dstSize) {
909 memcpy(dst, cSrc, dstSize);
910 return dstSize;
911 } /* not compressed */
912 if (cSrcSize == 1) {
913 memset(dst, *(const BYTE *)cSrc, dstSize);
914 return dstSize;
915 } /* RLE */
916
917 {
918 U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
919 return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
920 : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
921 }
922}
923
924size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
925{
926 /* validation checks */
927 if (dstSize == 0)
928 return ERROR(dstSize_tooSmall);
929 if ((cSrcSize >= dstSize) || (cSrcSize <= 1))
930 return ERROR(corruption_detected); /* invalid */
931
932 {
933 U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
934 return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
935 : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
936 }
937}
938
939size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
940{
941 /* validation checks */
942 if (dstSize == 0)
943 return ERROR(dstSize_tooSmall);
944 if (cSrcSize > dstSize)
945 return ERROR(corruption_detected); /* invalid */
946 if (cSrcSize == dstSize) {
947 memcpy(dst, cSrc, dstSize);
948 return dstSize;
949 } /* not compressed */
950 if (cSrcSize == 1) {
951 memset(dst, *(const BYTE *)cSrc, dstSize);
952 return dstSize;
953 } /* RLE */
954
955 {
956 U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
957 return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
958 : HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
959 }
960}
diff --git a/lib/zstd/mem.h b/lib/zstd/mem.h
new file mode 100644
index 000000000000..3a0f34c8706c
--- /dev/null
+++ b/lib/zstd/mem.h
@@ -0,0 +1,151 @@
1/**
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of https://github.com/facebook/zstd.
7 * An additional grant of patent rights can be found in the PATENTS file in the
8 * same directory.
9 *
10 * This program is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU General Public License version 2 as published by the
12 * Free Software Foundation. This program is dual-licensed; you may select
13 * either version 2 of the GNU General Public License ("GPL") or BSD license
14 * ("BSD").
15 */
16
17#ifndef MEM_H_MODULE
18#define MEM_H_MODULE
19
20/*-****************************************
21* Dependencies
22******************************************/
23#include <asm/unaligned.h>
24#include <linux/string.h> /* memcpy */
25#include <linux/types.h> /* size_t, ptrdiff_t */
26
27/*-****************************************
28* Compiler specifics
29******************************************/
30#define ZSTD_STATIC static __inline __attribute__((unused))
31
32/*-**************************************************************
33* Basic Types
34*****************************************************************/
35typedef uint8_t BYTE;
36typedef uint16_t U16;
37typedef int16_t S16;
38typedef uint32_t U32;
39typedef int32_t S32;
40typedef uint64_t U64;
41typedef int64_t S64;
42typedef ptrdiff_t iPtrDiff;
43typedef uintptr_t uPtrDiff;
44
45/*-**************************************************************
46* Memory I/O
47*****************************************************************/
48ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t) == 4; }
49ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t) == 8; }
50
51#if defined(__LITTLE_ENDIAN)
52#define ZSTD_LITTLE_ENDIAN 1
53#else
54#define ZSTD_LITTLE_ENDIAN 0
55#endif
56
57ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; }
58
59ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); }
60
61ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); }
62
63ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); }
64
65ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); }
66
67ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); }
68
69ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); }
70
71ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); }
72
73/*=== Little endian r/w ===*/
74
75ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); }
76
77ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); }
78
79ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); }
80
81ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val)
82{
83 ZSTD_writeLE16(memPtr, (U16)val);
84 ((BYTE *)memPtr)[2] = (BYTE)(val >> 16);
85}
86
87ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); }
88
89ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); }
90
91ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); }
92
93ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); }
94
95ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr)
96{
97 if (ZSTD_32bits())
98 return (size_t)ZSTD_readLE32(memPtr);
99 else
100 return (size_t)ZSTD_readLE64(memPtr);
101}
102
103ZSTD_STATIC void ZSTD_writeLEST(void *memPtr, size_t val)
104{
105 if (ZSTD_32bits())
106 ZSTD_writeLE32(memPtr, (U32)val);
107 else
108 ZSTD_writeLE64(memPtr, (U64)val);
109}
110
111/*=== Big endian r/w ===*/
112
113ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); }
114
115ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); }
116
117ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); }
118
119ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); }
120
121ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr)
122{
123 if (ZSTD_32bits())
124 return (size_t)ZSTD_readBE32(memPtr);
125 else
126 return (size_t)ZSTD_readBE64(memPtr);
127}
128
129ZSTD_STATIC void ZSTD_writeBEST(void *memPtr, size_t val)
130{
131 if (ZSTD_32bits())
132 ZSTD_writeBE32(memPtr, (U32)val);
133 else
134 ZSTD_writeBE64(memPtr, (U64)val);
135}
136
137/* function safe only for comparisons */
138ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length)
139{
140 switch (length) {
141 default:
142 case 4: return ZSTD_read32(memPtr);
143 case 3:
144 if (ZSTD_isLittleEndian())
145 return ZSTD_read32(memPtr) << 8;
146 else
147 return ZSTD_read32(memPtr) >> 8;
148 }
149}
150
151#endif /* MEM_H_MODULE */
diff --git a/lib/zstd/zstd_common.c b/lib/zstd/zstd_common.c
new file mode 100644
index 000000000000..a282624ee155
--- /dev/null
+++ b/lib/zstd/zstd_common.c
@@ -0,0 +1,75 @@
1/**
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of https://github.com/facebook/zstd.
7 * An additional grant of patent rights can be found in the PATENTS file in the
8 * same directory.
9 *
10 * This program is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU General Public License version 2 as published by the
12 * Free Software Foundation. This program is dual-licensed; you may select
13 * either version 2 of the GNU General Public License ("GPL") or BSD license
14 * ("BSD").
15 */
16
17/*-*************************************
18* Dependencies
19***************************************/
20#include "error_private.h"
21#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
22#include <linux/kernel.h>
23
24/*=**************************************************************
25* Custom allocator
26****************************************************************/
27
28#define stack_push(stack, size) \
29 ({ \
30 void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \
31 (stack)->ptr = (char *)ptr + (size); \
32 (stack)->ptr <= (stack)->end ? ptr : NULL; \
33 })
34
35ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize)
36{
37 ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace};
38 ZSTD_stack *stack = (ZSTD_stack *)workspace;
39 /* Verify preconditions */
40 if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) {
41 ZSTD_customMem error = {NULL, NULL, NULL};
42 return error;
43 }
44 /* Initialize the stack */
45 stack->ptr = workspace;
46 stack->end = (char *)workspace + workspaceSize;
47 stack_push(stack, sizeof(ZSTD_stack));
48 return stackMem;
49}
50
51void *ZSTD_stackAllocAll(void *opaque, size_t *size)
52{
53 ZSTD_stack *stack = (ZSTD_stack *)opaque;
54 *size = (BYTE const *)stack->end - (BYTE *)ZSTD_PTR_ALIGN(stack->ptr);
55 return stack_push(stack, *size);
56}
57
58void *ZSTD_stackAlloc(void *opaque, size_t size)
59{
60 ZSTD_stack *stack = (ZSTD_stack *)opaque;
61 return stack_push(stack, size);
62}
63void ZSTD_stackFree(void *opaque, void *address)
64{
65 (void)opaque;
66 (void)address;
67}
68
69void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); }
70
71void ZSTD_free(void *ptr, ZSTD_customMem customMem)
72{
73 if (ptr != NULL)
74 customMem.customFree(customMem.opaque, ptr);
75}
diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h
new file mode 100644
index 000000000000..1a79fab9e13a
--- /dev/null
+++ b/lib/zstd/zstd_internal.h
@@ -0,0 +1,263 @@
1/**
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of https://github.com/facebook/zstd.
7 * An additional grant of patent rights can be found in the PATENTS file in the
8 * same directory.
9 *
10 * This program is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU General Public License version 2 as published by the
12 * Free Software Foundation. This program is dual-licensed; you may select
13 * either version 2 of the GNU General Public License ("GPL") or BSD license
14 * ("BSD").
15 */
16
17#ifndef ZSTD_CCOMMON_H_MODULE
18#define ZSTD_CCOMMON_H_MODULE
19
20/*-*******************************************************
21* Compiler specifics
22*********************************************************/
23#define FORCE_INLINE static __always_inline
24#define FORCE_NOINLINE static noinline
25
26/*-*************************************
27* Dependencies
28***************************************/
29#include "error_private.h"
30#include "mem.h"
31#include <linux/compiler.h>
32#include <linux/kernel.h>
33#include <linux/xxhash.h>
34#include <linux/zstd.h>
35
36/*-*************************************
37* shared macros
38***************************************/
39#define MIN(a, b) ((a) < (b) ? (a) : (b))
40#define MAX(a, b) ((a) > (b) ? (a) : (b))
41#define CHECK_F(f) \
42 { \
43 size_t const errcod = f; \
44 if (ERR_isError(errcod)) \
45 return errcod; \
46 } /* check and Forward error code */
47#define CHECK_E(f, e) \
48 { \
49 size_t const errcod = f; \
50 if (ERR_isError(errcod)) \
51 return ERROR(e); \
52 } /* check and send Error code */
53#define ZSTD_STATIC_ASSERT(c) \
54 { \
55 enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \
56 }
57
58/*-*************************************
59* Common constants
60***************************************/
61#define ZSTD_OPT_NUM (1 << 12)
62#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
63
64#define ZSTD_REP_NUM 3 /* number of repcodes */
65#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */
66#define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1)
67#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
68static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8};
69
70#define KB *(1 << 10)
71#define MB *(1 << 20)
72#define GB *(1U << 30)
73
74#define BIT7 128
75#define BIT6 64
76#define BIT5 32
77#define BIT4 16
78#define BIT1 2
79#define BIT0 1
80
81#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
82static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8};
83static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4};
84
85#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
86static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
87typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
88
89#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
90#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
91
92#define HufLog 12
93typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
94
95#define LONGNBSEQ 0x7F00
96
97#define MINMATCH 3
98#define EQUAL_READ32 4
99
100#define Litbits 8
101#define MaxLit ((1 << Litbits) - 1)
102#define MaxML 52
103#define MaxLL 35
104#define MaxOff 28
105#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
106#define MLFSELog 9
107#define LLFSELog 9
108#define OffFSELog 8
109
110static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
111static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1};
112#define LL_DEFAULTNORMLOG 6 /* for static allocation */
113static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
114
115static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
116 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
117static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
118 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1};
119#define ML_DEFAULTNORMLOG 6 /* for static allocation */
120static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
121
122static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1};
123#define OF_DEFAULTNORMLOG 5 /* for static allocation */
124static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
125
126/*-*******************************************
127* Shared functions to include for inlining
128*********************************************/
129ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) {
130 memcpy(dst, src, 8);
131}
132/*! ZSTD_wildcopy() :
133* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
134#define WILDCOPY_OVERLENGTH 8
135ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
136{
137 const BYTE* ip = (const BYTE*)src;
138 BYTE* op = (BYTE*)dst;
139 BYTE* const oend = op + length;
140 /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
141 * Avoid the bad case where the loop only runs once by handling the
142 * special case separately. This doesn't trigger the bug because it
143 * doesn't involve pointer/integer overflow.
144 */
145 if (length <= 8)
146 return ZSTD_copy8(dst, src);
147 do {
148 ZSTD_copy8(op, ip);
149 op += 8;
150 ip += 8;
151 } while (op < oend);
152}
153
154/*-*******************************************
155* Private interfaces
156*********************************************/
157typedef struct ZSTD_stats_s ZSTD_stats_t;
158
159typedef struct {
160 U32 off;
161 U32 len;
162} ZSTD_match_t;
163
164typedef struct {
165 U32 price;
166 U32 off;
167 U32 mlen;
168 U32 litlen;
169 U32 rep[ZSTD_REP_NUM];
170} ZSTD_optimal_t;
171
172typedef struct seqDef_s {
173 U32 offset;
174 U16 litLength;
175 U16 matchLength;
176} seqDef;
177
178typedef struct {
179 seqDef *sequencesStart;
180 seqDef *sequences;
181 BYTE *litStart;
182 BYTE *lit;
183 BYTE *llCode;
184 BYTE *mlCode;
185 BYTE *ofCode;
186 U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
187 U32 longLengthPos;
188 /* opt */
189 ZSTD_optimal_t *priceTable;
190 ZSTD_match_t *matchTable;
191 U32 *matchLengthFreq;
192 U32 *litLengthFreq;
193 U32 *litFreq;
194 U32 *offCodeFreq;
195 U32 matchLengthSum;
196 U32 matchSum;
197 U32 litLengthSum;
198 U32 litSum;
199 U32 offCodeSum;
200 U32 log2matchLengthSum;
201 U32 log2matchSum;
202 U32 log2litLengthSum;
203 U32 log2litSum;
204 U32 log2offCodeSum;
205 U32 factor;
206 U32 staticPrices;
207 U32 cachedPrice;
208 U32 cachedLitLength;
209 const BYTE *cachedLiterals;
210} seqStore_t;
211
212const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx);
213void ZSTD_seqToCodes(const seqStore_t *seqStorePtr);
214int ZSTD_isSkipFrame(ZSTD_DCtx *dctx);
215
216/*= Custom memory allocation functions */
217typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size);
218typedef void (*ZSTD_freeFunction)(void *opaque, void *address);
219typedef struct {
220 ZSTD_allocFunction customAlloc;
221 ZSTD_freeFunction customFree;
222 void *opaque;
223} ZSTD_customMem;
224
225void *ZSTD_malloc(size_t size, ZSTD_customMem customMem);
226void ZSTD_free(void *ptr, ZSTD_customMem customMem);
227
228/*====== stack allocation ======*/
229
230typedef struct {
231 void *ptr;
232 const void *end;
233} ZSTD_stack;
234
235#define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t))
236#define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t))
237
238ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize);
239
240void *ZSTD_stackAllocAll(void *opaque, size_t *size);
241void *ZSTD_stackAlloc(void *opaque, size_t size);
242void ZSTD_stackFree(void *opaque, void *address);
243
244/*====== common function ======*/
245
246ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); }
247
248/* hidden functions */
249
250/* ZSTD_invalidateRepCodes() :
251 * ensures next compression will not use repcodes from previous block.
252 * Note : only works with regular variant;
253 * do not use with extDict variant ! */
254void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx);
255
256size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx);
257size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx);
258size_t ZSTD_freeCDict(ZSTD_CDict *cdict);
259size_t ZSTD_freeDDict(ZSTD_DDict *cdict);
260size_t ZSTD_freeCStream(ZSTD_CStream *zcs);
261size_t ZSTD_freeDStream(ZSTD_DStream *zds);
262
263#endif /* ZSTD_CCOMMON_H_MODULE */
diff --git a/lib/zstd/zstd_opt.h b/lib/zstd/zstd_opt.h
new file mode 100644
index 000000000000..55e1b4cba808
--- /dev/null
+++ b/lib/zstd/zstd_opt.h
@@ -0,0 +1,1014 @@
1/**
2 * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of https://github.com/facebook/zstd.
7 * An additional grant of patent rights can be found in the PATENTS file in the
8 * same directory.
9 *
10 * This program is free software; you can redistribute it and/or modify it under
11 * the terms of the GNU General Public License version 2 as published by the
12 * Free Software Foundation. This program is dual-licensed; you may select
13 * either version 2 of the GNU General Public License ("GPL") or BSD license
14 * ("BSD").
15 */
16
17/* Note : this file is intended to be included within zstd_compress.c */
18
19#ifndef ZSTD_OPT_H_91842398743
20#define ZSTD_OPT_H_91842398743
21
22#define ZSTD_LITFREQ_ADD 2
23#define ZSTD_FREQ_DIV 4
24#define ZSTD_MAX_PRICE (1 << 30)
25
26/*-*************************************
27* Price functions for optimal parser
28***************************************/
29FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t *ssPtr)
30{
31 ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum + 1);
32 ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum + 1);
33 ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum + 1);
34 ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum + 1);
35 ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum));
36}
37
38ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize)
39{
40 unsigned u;
41
42 ssPtr->cachedLiterals = NULL;
43 ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
44 ssPtr->staticPrices = 0;
45
46 if (ssPtr->litLengthSum == 0) {
47 if (srcSize <= 1024)
48 ssPtr->staticPrices = 1;
49
50 for (u = 0; u <= MaxLit; u++)
51 ssPtr->litFreq[u] = 0;
52 for (u = 0; u < srcSize; u++)
53 ssPtr->litFreq[src[u]]++;
54
55 ssPtr->litSum = 0;
56 ssPtr->litLengthSum = MaxLL + 1;
57 ssPtr->matchLengthSum = MaxML + 1;
58 ssPtr->offCodeSum = (MaxOff + 1);
59 ssPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits);
60
61 for (u = 0; u <= MaxLit; u++) {
62 ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> ZSTD_FREQ_DIV);
63 ssPtr->litSum += ssPtr->litFreq[u];
64 }
65 for (u = 0; u <= MaxLL; u++)
66 ssPtr->litLengthFreq[u] = 1;
67 for (u = 0; u <= MaxML; u++)
68 ssPtr->matchLengthFreq[u] = 1;
69 for (u = 0; u <= MaxOff; u++)
70 ssPtr->offCodeFreq[u] = 1;
71 } else {
72 ssPtr->matchLengthSum = 0;
73 ssPtr->litLengthSum = 0;
74 ssPtr->offCodeSum = 0;
75 ssPtr->matchSum = 0;
76 ssPtr->litSum = 0;
77
78 for (u = 0; u <= MaxLit; u++) {
79 ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> (ZSTD_FREQ_DIV + 1));
80 ssPtr->litSum += ssPtr->litFreq[u];
81 }
82 for (u = 0; u <= MaxLL; u++) {
83 ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u] >> (ZSTD_FREQ_DIV + 1));
84 ssPtr->litLengthSum += ssPtr->litLengthFreq[u];
85 }
86 for (u = 0; u <= MaxML; u++) {
87 ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV);
88 ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u];
89 ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3);
90 }
91 ssPtr->matchSum *= ZSTD_LITFREQ_ADD;
92 for (u = 0; u <= MaxOff; u++) {
93 ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV);
94 ssPtr->offCodeSum += ssPtr->offCodeFreq[u];
95 }
96 }
97
98 ZSTD_setLog2Prices(ssPtr);
99}
100
101FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals)
102{
103 U32 price, u;
104
105 if (ssPtr->staticPrices)
106 return ZSTD_highbit32((U32)litLength + 1) + (litLength * 6);
107
108 if (litLength == 0)
109 return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0] + 1);
110
111 /* literals */
112 if (ssPtr->cachedLiterals == literals) {
113 U32 const additional = litLength - ssPtr->cachedLitLength;
114 const BYTE *literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
115 price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
116 for (u = 0; u < additional; u++)
117 price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]] + 1);
118 ssPtr->cachedPrice = price;
119 ssPtr->cachedLitLength = litLength;
120 } else {
121 price = litLength * ssPtr->log2litSum;
122 for (u = 0; u < litLength; u++)
123 price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]] + 1);
124
125 if (litLength >= 12) {
126 ssPtr->cachedLiterals = literals;
127 ssPtr->cachedPrice = price;
128 ssPtr->cachedLitLength = litLength;
129 }
130 }
131
132 /* literal Length */
133 {
134 const BYTE LL_deltaCode = 19;
135 const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
136 price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1);
137 }
138
139 return price;
140}
141
142FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra)
143{
144 /* offset */
145 U32 price;
146 BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
147
148 if (seqStorePtr->staticPrices)
149 return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode;
150
151 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1);
152 if (!ultra && offCode >= 20)
153 price += (offCode - 19) * 2;
154
155 /* match Length */
156 {
157 const BYTE ML_deltaCode = 36;
158 const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
159 price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1);
160 }
161
162 return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
163}
164
165ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength)
166{
167 U32 u;
168
169 /* literals */
170 seqStorePtr->litSum += litLength * ZSTD_LITFREQ_ADD;
171 for (u = 0; u < litLength; u++)
172 seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
173
174 /* literal Length */
175 {
176 const BYTE LL_deltaCode = 19;
177 const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
178 seqStorePtr->litLengthFreq[llCode]++;
179 seqStorePtr->litLengthSum++;
180 }
181
182 /* match offset */
183 {
184 BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
185 seqStorePtr->offCodeSum++;
186 seqStorePtr->offCodeFreq[offCode]++;
187 }
188
189 /* match Length */
190 {
191 const BYTE ML_deltaCode = 36;
192 const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
193 seqStorePtr->matchLengthFreq[mlCode]++;
194 seqStorePtr->matchLengthSum++;
195 }
196
197 ZSTD_setLog2Prices(seqStorePtr);
198}
199
200#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
201 { \
202 while (last_pos < pos) { \
203 opt[last_pos + 1].price = ZSTD_MAX_PRICE; \
204 last_pos++; \
205 } \
206 opt[pos].mlen = mlen_; \
207 opt[pos].off = offset_; \
208 opt[pos].litlen = litlen_; \
209 opt[pos].price = price_; \
210 }
211
212/* Update hashTable3 up to ip (excluded)
213 Assumption : always within prefix (i.e. not within extDict) */
214FORCE_INLINE
215U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_CCtx *zc, const BYTE *ip)
216{
217 U32 *const hashTable3 = zc->hashTable3;
218 U32 const hashLog3 = zc->hashLog3;
219 const BYTE *const base = zc->base;
220 U32 idx = zc->nextToUpdate3;
221 const U32 target = zc->nextToUpdate3 = (U32)(ip - base);
222 const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3);
223
224 while (idx < target) {
225 hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx;
226 idx++;
227 }
228
229 return hashTable3[hash3];
230}
231
232/*-*************************************
233* Binary Tree search
234***************************************/
235static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict,
236 ZSTD_match_t *matches, const U32 minMatchLen)
237{
238 const BYTE *const base = zc->base;
239 const U32 curr = (U32)(ip - base);
240 const U32 hashLog = zc->params.cParams.hashLog;
241 const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
242 U32 *const hashTable = zc->hashTable;
243 U32 matchIndex = hashTable[h];
244 U32 *const bt = zc->chainTable;
245 const U32 btLog = zc->params.cParams.chainLog - 1;
246 const U32 btMask = (1U << btLog) - 1;
247 size_t commonLengthSmaller = 0, commonLengthLarger = 0;
248 const BYTE *const dictBase = zc->dictBase;
249 const U32 dictLimit = zc->dictLimit;
250 const BYTE *const dictEnd = dictBase + dictLimit;
251 const BYTE *const prefixStart = base + dictLimit;
252 const U32 btLow = btMask >= curr ? 0 : curr - btMask;
253 const U32 windowLow = zc->lowLimit;
254 U32 *smallerPtr = bt + 2 * (curr & btMask);
255 U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
256 U32 matchEndIdx = curr + 8;
257 U32 dummy32; /* to be nullified at the end */
258 U32 mnum = 0;
259
260 const U32 minMatch = (mls == 3) ? 3 : 4;
261 size_t bestLength = minMatchLen - 1;
262
263 if (minMatch == 3) { /* HC3 match finder */
264 U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(zc, ip);
265 if (matchIndex3 > windowLow && (curr - matchIndex3 < (1 << 18))) {
266 const BYTE *match;
267 size_t currMl = 0;
268 if ((!extDict) || matchIndex3 >= dictLimit) {
269 match = base + matchIndex3;
270 if (match[bestLength] == ip[bestLength])
271 currMl = ZSTD_count(ip, match, iLimit);
272 } else {
273 match = dictBase + matchIndex3;
274 if (ZSTD_readMINMATCH(match, MINMATCH) ==
275 ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
276 currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
277 }
278
279 /* save best solution */
280 if (currMl > bestLength) {
281 bestLength = currMl;
282 matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3;
283 matches[mnum].len = (U32)currMl;
284 mnum++;
285 if (currMl > ZSTD_OPT_NUM)
286 goto update;
287 if (ip + currMl == iLimit)
288 goto update; /* best possible, and avoid read overflow*/
289 }
290 }
291 }
292
293 hashTable[h] = curr; /* Update Hash Table */
294
295 while (nbCompares-- && (matchIndex > windowLow)) {
296 U32 *nextPtr = bt + 2 * (matchIndex & btMask);
297 size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
298 const BYTE *match;
299
300 if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
301 match = base + matchIndex;
302 if (match[matchLength] == ip[matchLength]) {
303 matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iLimit) + 1;
304 }
305 } else {
306 match = dictBase + matchIndex;
307 matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart);
308 if (matchIndex + matchLength >= dictLimit)
309 match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
310 }
311
312 if (matchLength > bestLength) {
313 if (matchLength > matchEndIdx - matchIndex)
314 matchEndIdx = matchIndex + (U32)matchLength;
315 bestLength = matchLength;
316 matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex;
317 matches[mnum].len = (U32)matchLength;
318 mnum++;
319 if (matchLength > ZSTD_OPT_NUM)
320 break;
321 if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */
322 break; /* drop, to guarantee consistency (miss a little bit of compression) */
323 }
324
325 if (match[matchLength] < ip[matchLength]) {
326 /* match is smaller than curr */
327 *smallerPtr = matchIndex; /* update smaller idx */
328 commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
329 if (matchIndex <= btLow) {
330 smallerPtr = &dummy32;
331 break;
332 } /* beyond tree size, stop the search */
333 smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
334 matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
335 } else {
336 /* match is larger than curr */
337 *largerPtr = matchIndex;
338 commonLengthLarger = matchLength;
339 if (matchIndex <= btLow) {
340 largerPtr = &dummy32;
341 break;
342 } /* beyond tree size, stop the search */
343 largerPtr = nextPtr;
344 matchIndex = nextPtr[0];
345 }
346 }
347
348 *smallerPtr = *largerPtr = 0;
349
350update:
351 zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
352 return mnum;
353}
354
355/** Tree updater, providing best match */
356static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches,
357 const U32 minMatchLen)
358{
359 if (ip < zc->base + zc->nextToUpdate)
360 return 0; /* skipped area */
361 ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
362 return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen);
363}
364
365static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
366 const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
367 ZSTD_match_t *matches, const U32 minMatchLen)
368{
369 switch (matchLengthSearch) {
370 case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
371 default:
372 case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
373 case 5: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
374 case 7:
375 case 6: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
376 }
377}
378
379/** Tree updater, providing best match */
380static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls,
381 ZSTD_match_t *matches, const U32 minMatchLen)
382{
383 if (ip < zc->base + zc->nextToUpdate)
384 return 0; /* skipped area */
385 ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
386 return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen);
387}
388
389static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
390 const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
391 ZSTD_match_t *matches, const U32 minMatchLen)
392{
393 switch (matchLengthSearch) {
394 case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
395 default:
396 case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
397 case 5: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
398 case 7:
399 case 6: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
400 }
401}
402
403/*-*******************************
404* Optimal parser
405*********************************/
406FORCE_INLINE
407void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
408{
409 seqStore_t *seqStorePtr = &(ctx->seqStore);
410 const BYTE *const istart = (const BYTE *)src;
411 const BYTE *ip = istart;
412 const BYTE *anchor = istart;
413 const BYTE *const iend = istart + srcSize;
414 const BYTE *const ilimit = iend - 8;
415 const BYTE *const base = ctx->base;
416 const BYTE *const prefixStart = base + ctx->dictLimit;
417
418 const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
419 const U32 sufficient_len = ctx->params.cParams.targetLength;
420 const U32 mls = ctx->params.cParams.searchLength;
421 const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
422
423 ZSTD_optimal_t *opt = seqStorePtr->priceTable;
424 ZSTD_match_t *matches = seqStorePtr->matchTable;
425 const BYTE *inr;
426 U32 offset, rep[ZSTD_REP_NUM];
427
428 /* init */
429 ctx->nextToUpdate3 = ctx->nextToUpdate;
430 ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
431 ip += (ip == prefixStart);
432 {
433 U32 i;
434 for (i = 0; i < ZSTD_REP_NUM; i++)
435 rep[i] = ctx->rep[i];
436 }
437
438 /* Match Loop */
439 while (ip < ilimit) {
440 U32 cur, match_num, last_pos, litlen, price;
441 U32 u, mlen, best_mlen, best_off, litLength;
442 memset(opt, 0, sizeof(ZSTD_optimal_t));
443 last_pos = 0;
444 litlen = (U32)(ip - anchor);
445
446 /* check repCode */
447 {
448 U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
449 for (i = (ip == anchor); i < last_i; i++) {
450 const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
451 if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) &&
452 (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
453 mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch;
454 if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
455 best_mlen = mlen;
456 best_off = i;
457 cur = 0;
458 last_pos = 1;
459 goto _storeSequence;
460 }
461 best_off = i - (ip == anchor);
462 do {
463 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
464 if (mlen > last_pos || price < opt[mlen].price)
465 SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
466 mlen--;
467 } while (mlen >= minMatch);
468 }
469 }
470 }
471
472 match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
473
474 if (!last_pos && !match_num) {
475 ip++;
476 continue;
477 }
478
479 if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
480 best_mlen = matches[match_num - 1].len;
481 best_off = matches[match_num - 1].off;
482 cur = 0;
483 last_pos = 1;
484 goto _storeSequence;
485 }
486
487 /* set prices using matches at position = 0 */
488 best_mlen = (last_pos) ? last_pos : minMatch;
489 for (u = 0; u < match_num; u++) {
490 mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
491 best_mlen = matches[u].len;
492 while (mlen <= best_mlen) {
493 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
494 if (mlen > last_pos || price < opt[mlen].price)
495 SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
496 mlen++;
497 }
498 }
499
500 if (last_pos < minMatch) {
501 ip++;
502 continue;
503 }
504
505 /* initialize opt[0] */
506 {
507 U32 i;
508 for (i = 0; i < ZSTD_REP_NUM; i++)
509 opt[0].rep[i] = rep[i];
510 }
511 opt[0].mlen = 1;
512 opt[0].litlen = litlen;
513
514 /* check further positions */
515 for (cur = 1; cur <= last_pos; cur++) {
516 inr = ip + cur;
517
518 if (opt[cur - 1].mlen == 1) {
519 litlen = opt[cur - 1].litlen + 1;
520 if (cur > litlen) {
521 price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
522 } else
523 price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
524 } else {
525 litlen = 1;
526 price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
527 }
528
529 if (cur > last_pos || price <= opt[cur].price)
530 SET_PRICE(cur, 1, 0, litlen, price);
531
532 if (cur == last_pos)
533 break;
534
535 if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
536 continue;
537
538 mlen = opt[cur].mlen;
539 if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
540 opt[cur].rep[2] = opt[cur - mlen].rep[1];
541 opt[cur].rep[1] = opt[cur - mlen].rep[0];
542 opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
543 } else {
544 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
545 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
546 opt[cur].rep[0] =
547 ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
548 }
549
550 best_mlen = minMatch;
551 {
552 U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
553 for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */
554 const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
555 if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) &&
556 (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
557 mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch;
558
559 if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
560 best_mlen = mlen;
561 best_off = i;
562 last_pos = cur + 1;
563 goto _storeSequence;
564 }
565
566 best_off = i - (opt[cur].mlen != 1);
567 if (mlen > best_mlen)
568 best_mlen = mlen;
569
570 do {
571 if (opt[cur].mlen == 1) {
572 litlen = opt[cur].litlen;
573 if (cur > litlen) {
574 price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
575 best_off, mlen - MINMATCH, ultra);
576 } else
577 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
578 } else {
579 litlen = 0;
580 price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
581 }
582
583 if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
584 SET_PRICE(cur + mlen, mlen, i, litlen, price);
585 mlen--;
586 } while (mlen >= minMatch);
587 }
588 }
589 }
590
591 match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
592
593 if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
594 best_mlen = matches[match_num - 1].len;
595 best_off = matches[match_num - 1].off;
596 last_pos = cur + 1;
597 goto _storeSequence;
598 }
599
600 /* set prices using matches at position = cur */
601 for (u = 0; u < match_num; u++) {
602 mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
603 best_mlen = matches[u].len;
604
605 while (mlen <= best_mlen) {
606 if (opt[cur].mlen == 1) {
607 litlen = opt[cur].litlen;
608 if (cur > litlen)
609 price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
610 matches[u].off - 1, mlen - MINMATCH, ultra);
611 else
612 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
613 } else {
614 litlen = 0;
615 price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
616 }
617
618 if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
619 SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
620
621 mlen++;
622 }
623 }
624 }
625
626 best_mlen = opt[last_pos].mlen;
627 best_off = opt[last_pos].off;
628 cur = last_pos - best_mlen;
629
630 /* store sequence */
631_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
632 opt[0].mlen = 1;
633
634 while (1) {
635 mlen = opt[cur].mlen;
636 offset = opt[cur].off;
637 opt[cur].mlen = best_mlen;
638 opt[cur].off = best_off;
639 best_mlen = mlen;
640 best_off = offset;
641 if (mlen > cur)
642 break;
643 cur -= mlen;
644 }
645
646 for (u = 0; u <= last_pos;) {
647 u += opt[u].mlen;
648 }
649
650 for (cur = 0; cur < last_pos;) {
651 mlen = opt[cur].mlen;
652 if (mlen == 1) {
653 ip++;
654 cur++;
655 continue;
656 }
657 offset = opt[cur].off;
658 cur += mlen;
659 litLength = (U32)(ip - anchor);
660
661 if (offset > ZSTD_REP_MOVE_OPT) {
662 rep[2] = rep[1];
663 rep[1] = rep[0];
664 rep[0] = offset - ZSTD_REP_MOVE_OPT;
665 offset--;
666 } else {
667 if (offset != 0) {
668 best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
669 if (offset != 1)
670 rep[2] = rep[1];
671 rep[1] = rep[0];
672 rep[0] = best_off;
673 }
674 if (litLength == 0)
675 offset--;
676 }
677
678 ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
679 ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
680 anchor = ip = ip + mlen;
681 }
682 } /* for (cur=0; cur < last_pos; ) */
683
684 /* Save reps for next block */
685 {
686 int i;
687 for (i = 0; i < ZSTD_REP_NUM; i++)
688 ctx->repToConfirm[i] = rep[i];
689 }
690
691 /* Last Literals */
692 {
693 size_t const lastLLSize = iend - anchor;
694 memcpy(seqStorePtr->lit, anchor, lastLLSize);
695 seqStorePtr->lit += lastLLSize;
696 }
697}
698
699FORCE_INLINE
700void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
701{
702 seqStore_t *seqStorePtr = &(ctx->seqStore);
703 const BYTE *const istart = (const BYTE *)src;
704 const BYTE *ip = istart;
705 const BYTE *anchor = istart;
706 const BYTE *const iend = istart + srcSize;
707 const BYTE *const ilimit = iend - 8;
708 const BYTE *const base = ctx->base;
709 const U32 lowestIndex = ctx->lowLimit;
710 const U32 dictLimit = ctx->dictLimit;
711 const BYTE *const prefixStart = base + dictLimit;
712 const BYTE *const dictBase = ctx->dictBase;
713 const BYTE *const dictEnd = dictBase + dictLimit;
714
715 const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
716 const U32 sufficient_len = ctx->params.cParams.targetLength;
717 const U32 mls = ctx->params.cParams.searchLength;
718 const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
719
720 ZSTD_optimal_t *opt = seqStorePtr->priceTable;
721 ZSTD_match_t *matches = seqStorePtr->matchTable;
722 const BYTE *inr;
723
724 /* init */
725 U32 offset, rep[ZSTD_REP_NUM];
726 {
727 U32 i;
728 for (i = 0; i < ZSTD_REP_NUM; i++)
729 rep[i] = ctx->rep[i];
730 }
731
732 ctx->nextToUpdate3 = ctx->nextToUpdate;
733 ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
734 ip += (ip == prefixStart);
735
736 /* Match Loop */
737 while (ip < ilimit) {
738 U32 cur, match_num, last_pos, litlen, price;
739 U32 u, mlen, best_mlen, best_off, litLength;
740 U32 curr = (U32)(ip - base);
741 memset(opt, 0, sizeof(ZSTD_optimal_t));
742 last_pos = 0;
743 opt[0].litlen = (U32)(ip - anchor);
744
745 /* check repCode */
746 {
747 U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
748 for (i = (ip == anchor); i < last_i; i++) {
749 const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
750 const U32 repIndex = (U32)(curr - repCur);
751 const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
752 const BYTE *const repMatch = repBase + repIndex;
753 if ((repCur > 0 && repCur <= (S32)curr) &&
754 (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
755 && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
756 /* repcode detected we should take it */
757 const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
758 mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
759
760 if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
761 best_mlen = mlen;
762 best_off = i;
763 cur = 0;
764 last_pos = 1;
765 goto _storeSequence;
766 }
767
768 best_off = i - (ip == anchor);
769 litlen = opt[0].litlen;
770 do {
771 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
772 if (mlen > last_pos || price < opt[mlen].price)
773 SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
774 mlen--;
775 } while (mlen >= minMatch);
776 }
777 }
778 }
779
780 match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */
781
782 if (!last_pos && !match_num) {
783 ip++;
784 continue;
785 }
786
787 {
788 U32 i;
789 for (i = 0; i < ZSTD_REP_NUM; i++)
790 opt[0].rep[i] = rep[i];
791 }
792 opt[0].mlen = 1;
793
794 if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
795 best_mlen = matches[match_num - 1].len;
796 best_off = matches[match_num - 1].off;
797 cur = 0;
798 last_pos = 1;
799 goto _storeSequence;
800 }
801
802 best_mlen = (last_pos) ? last_pos : minMatch;
803
804 /* set prices using matches at position = 0 */
805 for (u = 0; u < match_num; u++) {
806 mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
807 best_mlen = matches[u].len;
808 litlen = opt[0].litlen;
809 while (mlen <= best_mlen) {
810 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
811 if (mlen > last_pos || price < opt[mlen].price)
812 SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
813 mlen++;
814 }
815 }
816
817 if (last_pos < minMatch) {
818 ip++;
819 continue;
820 }
821
822 /* check further positions */
823 for (cur = 1; cur <= last_pos; cur++) {
824 inr = ip + cur;
825
826 if (opt[cur - 1].mlen == 1) {
827 litlen = opt[cur - 1].litlen + 1;
828 if (cur > litlen) {
829 price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
830 } else
831 price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
832 } else {
833 litlen = 1;
834 price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
835 }
836
837 if (cur > last_pos || price <= opt[cur].price)
838 SET_PRICE(cur, 1, 0, litlen, price);
839
840 if (cur == last_pos)
841 break;
842
843 if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
844 continue;
845
846 mlen = opt[cur].mlen;
847 if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
848 opt[cur].rep[2] = opt[cur - mlen].rep[1];
849 opt[cur].rep[1] = opt[cur - mlen].rep[0];
850 opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
851 } else {
852 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
853 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
854 opt[cur].rep[0] =
855 ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
856 }
857
858 best_mlen = minMatch;
859 {
860 U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
861 for (i = (mlen != 1); i < last_i; i++) {
862 const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
863 const U32 repIndex = (U32)(curr + cur - repCur);
864 const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
865 const BYTE *const repMatch = repBase + repIndex;
866 if ((repCur > 0 && repCur <= (S32)(curr + cur)) &&
867 (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
868 && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
869 /* repcode detected */
870 const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
871 mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
872
873 if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
874 best_mlen = mlen;
875 best_off = i;
876 last_pos = cur + 1;
877 goto _storeSequence;
878 }
879
880 best_off = i - (opt[cur].mlen != 1);
881 if (mlen > best_mlen)
882 best_mlen = mlen;
883
884 do {
885 if (opt[cur].mlen == 1) {
886 litlen = opt[cur].litlen;
887 if (cur > litlen) {
888 price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
889 best_off, mlen - MINMATCH, ultra);
890 } else
891 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
892 } else {
893 litlen = 0;
894 price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
895 }
896
897 if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
898 SET_PRICE(cur + mlen, mlen, i, litlen, price);
899 mlen--;
900 } while (mlen >= minMatch);
901 }
902 }
903 }
904
905 match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
906
907 if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
908 best_mlen = matches[match_num - 1].len;
909 best_off = matches[match_num - 1].off;
910 last_pos = cur + 1;
911 goto _storeSequence;
912 }
913
914 /* set prices using matches at position = cur */
915 for (u = 0; u < match_num; u++) {
916 mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
917 best_mlen = matches[u].len;
918
919 while (mlen <= best_mlen) {
920 if (opt[cur].mlen == 1) {
921 litlen = opt[cur].litlen;
922 if (cur > litlen)
923 price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
924 matches[u].off - 1, mlen - MINMATCH, ultra);
925 else
926 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
927 } else {
928 litlen = 0;
929 price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
930 }
931
932 if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
933 SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
934
935 mlen++;
936 }
937 }
938 } /* for (cur = 1; cur <= last_pos; cur++) */
939
940 best_mlen = opt[last_pos].mlen;
941 best_off = opt[last_pos].off;
942 cur = last_pos - best_mlen;
943
944 /* store sequence */
945_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
946 opt[0].mlen = 1;
947
948 while (1) {
949 mlen = opt[cur].mlen;
950 offset = opt[cur].off;
951 opt[cur].mlen = best_mlen;
952 opt[cur].off = best_off;
953 best_mlen = mlen;
954 best_off = offset;
955 if (mlen > cur)
956 break;
957 cur -= mlen;
958 }
959
960 for (u = 0; u <= last_pos;) {
961 u += opt[u].mlen;
962 }
963
964 for (cur = 0; cur < last_pos;) {
965 mlen = opt[cur].mlen;
966 if (mlen == 1) {
967 ip++;
968 cur++;
969 continue;
970 }
971 offset = opt[cur].off;
972 cur += mlen;
973 litLength = (U32)(ip - anchor);
974
975 if (offset > ZSTD_REP_MOVE_OPT) {
976 rep[2] = rep[1];
977 rep[1] = rep[0];
978 rep[0] = offset - ZSTD_REP_MOVE_OPT;
979 offset--;
980 } else {
981 if (offset != 0) {
982 best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
983 if (offset != 1)
984 rep[2] = rep[1];
985 rep[1] = rep[0];
986 rep[0] = best_off;
987 }
988
989 if (litLength == 0)
990 offset--;
991 }
992
993 ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
994 ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
995 anchor = ip = ip + mlen;
996 }
997 } /* for (cur=0; cur < last_pos; ) */
998
999 /* Save reps for next block */
1000 {
1001 int i;
1002 for (i = 0; i < ZSTD_REP_NUM; i++)
1003 ctx->repToConfirm[i] = rep[i];
1004 }
1005
1006 /* Last Literals */
1007 {
1008 size_t lastLLSize = iend - anchor;
1009 memcpy(seqStorePtr->lit, anchor, lastLLSize);
1010 seqStorePtr->lit += lastLLSize;
1011 }
1012}
1013
1014#endif /* ZSTD_OPT_H_91842398743 */
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-28 17:40:08 -0500