From 78a2e08becf98748c61c86619f9f9402309289a3 Mon Sep 17 00:00:00 2001 From: Teemu Takaluoma Date: Thu, 17 Jun 2021 16:50:30 +0300 Subject: [PATCH] mbed-cloud-client-example 4.9.1 --- CHANGELOG.md | 4 + delta-tool/bsdiff/Makefile | 137 -- delta-tool/bsdiff/bsdiff.c | 607 ----- delta-tool/bsdiff/bsdiff.h | 46 - delta-tool/bsdiff/bspatch_main.c | 172 -- delta-tool/include/bspatch.h | 131 -- delta-tool/include/bspatch_private.h | 120 - delta-tool/include/lz4.h | 660 ------ delta-tool/include/varint.h | 40 - delta-tool/mbed_lib.json | 10 - delta-tool/source/bspatch.c | 770 ------- delta-tool/source/lz4.c | 1972 ----------------- delta-tool/source/varint.c | 152 -- delta-tool/tools/delta-tool.py | 192 -- delta-tool/tools/delta-vendor-info.asn1 | 14 - delta-tool/tools/delta_vendor_info.py | 30 - .../tools/generate-delta-and-manifest.sh | 105 - mbed-cloud-client.lib | 2 +- west.yml | 2 +- 19 files changed, 6 insertions(+), 5160 deletions(-) delete mode 100755 delta-tool/bsdiff/Makefile delete mode 100755 delta-tool/bsdiff/bsdiff.c delete mode 100755 delta-tool/bsdiff/bsdiff.h delete mode 100755 delta-tool/bsdiff/bspatch_main.c delete mode 100755 delta-tool/include/bspatch.h delete mode 100644 delta-tool/include/bspatch_private.h delete mode 100755 delta-tool/include/lz4.h delete mode 100644 delta-tool/include/varint.h delete mode 100644 delta-tool/mbed_lib.json delete mode 100755 delta-tool/source/bspatch.c delete mode 100755 delta-tool/source/lz4.c delete mode 100644 delta-tool/source/varint.c delete mode 100644 delta-tool/tools/delta-tool.py delete mode 100644 delta-tool/tools/delta-vendor-info.asn1 delete mode 100644 delta-tool/tools/delta_vendor_info.py delete mode 100755 delta-tool/tools/generate-delta-and-manifest.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index aa52723..8a5e701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for Pelion Device Management Client example application +## Release 4.9.1 (15.06.2021) + +No changes. + ## Release 4.9.0 (20.05.2021) * [Mbed OS] Increased `SN_COAP_BLOCKWISE_MAX_TIME_DATA_STORED` stored time to 15min for mesh to account for long retransmission chains during blockwise transfers. diff --git a/delta-tool/bsdiff/Makefile b/delta-tool/bsdiff/Makefile deleted file mode 100755 index 790e9c7..0000000 --- a/delta-tool/bsdiff/Makefile +++ /dev/null @@ -1,137 +0,0 @@ -CFLAGS += -O0 -Wall -pedantic -g -ggdb -Wextra -std=gnu99 -Wmissing-prototypes -Wstrict-prototypes -ldl -DLZ4_MEMORY_USAGE=10 -Werror=implicit-function-declaration -Werror=return-type -# -bsdiff: bsdiff.c ../source/varint.c lz4.o - ${CC} ${CFLAGS} -D BSDIFF_EXECUTABLE $^ -o $@ -I../include - -bspatch: bspatch_main.c ../source/bspatch.c ../include/bspatch.h ../include/bspatch_private.h lz4.o ../source/varint.c - ${CC} ${CFLAGS} $^ -o $@ -I../include -DBS_PATCH_COMPILE_TIME_MEMORY_ALLOC=1024 - -bspatch_dynamic_alloc: bspatch_main.c ../source/bspatch.c ../include/bspatch.h ../include/bspatch_private.h lz4.o ../source/varint.c - ${CC} ${CFLAGS} $^ -o $@ -I../include -DBS_PATCH_COMPILE_TIME_MEMORY_ALLOC=0 - -bspatch_compile_time_memory1024: bspatch_main.c ../source/bspatch.c ../include/bspatch.h ../include/bspatch_private.h lz4.o ../source/varint.c - ${CC} ${CFLAGS} $^ -o $@ -I../include -DBS_PATCH_COMPILE_TIME_MEMORY_ALLOC=1024 - -bspatch_compile_time_memory131072: bspatch_main.c ../source/bspatch.c ../include/bspatch.h ../include/bspatch_private.h lz4.o ../source/varint.c - ${CC} ${CFLAGS} $^ -o $@ -I../include -DBS_PATCH_COMPILE_TIME_MEMORY_ALLOC=131072 - - -bspatch_compile_time_memory131072test: bspatch_compile_time_memory131072 - rm -f test-folder/* - @echo "******" - @echo "Test 9" - @echo "******" - ./bsdiff ./test_data/rootfs-yocto-original-8.tar ./test_data/rootfs-yocto-new-8.tar ./test-folder/raspi.bsdiff 65536 - ./bspatch_compile_time_memory131072 ./test_data/rootfs-yocto-original-8.tar ./test-folder/raspi_image.tar ./test-folder/raspi.bsdiff - diff ./test_data/rootfs-yocto-new-8.tar ./test-folder/raspi_image.tar - - @echo "Test 6: app update bspatch frame 65536" - @echo "******" - ./bsdiff ./test_data/app_update_old.bin ./test_data/app_update_new.bin ./test-folder/app_update_diff_65536.bsdiff 65536 - ./bspatch_compile_time_memory131072 ./test_data/app_update_old.bin ./test-folder/app_update_patched_65536.bin ./test-folder/app_update_diff_65536.bsdiff - diff ./test_data/app_update_new.bin ./test-folder/app_update_patched_65536.bin - #lz4 ./test-folder/app_update_diff_65536.bsdiff test-folder/app_update_diff_65536.bsdiff.lz4 - #lz4 ./test-folder/app_update_patched_65536.bin test-folder/app_update_patched_65536.bin.lz4 - -lz4.o: ../source/lz4.c ../include/lz4.h - ${CC} -c ../source/lz4.c -I../include -DLZ4_MEMORY_USAGE=10 - -#varint.o: ../source/varint.c ../include/varint.h -# ${CC} -c ../source/varint.c -I../include - -.PHONY: test -test: bsdiff bspatch just-test - -.PHONY: test_dynamic_alloc -test_dynamic_alloc: bsdiff bspatch bspatch_dynamic_alloc justdynamictest - -#these test now run with default framesize of 512B -.PHONY: just-test -just-test: - rm -f test-folder/* - mkdir -p test-folder - - - @echo "*********************************" - ls -la test-folder - - @echo "******" - @echo "Test 1: simple 1" - @echo "******" - ./bsdiff ./test_data/simple1.txt ./test_data/simple1_new.txt ./test-folder/simple1_diff.bsdiff - ./bspatch ./test_data/simple1.txt ./test-folder/simple1_patched.bin ./test-folder/simple1_diff.bsdiff - diff ./test_data/simple1_new.txt ./test-folder/simple1_patched.bin - - @echo "******" - @echo "Test 3: string tweak" - @echo "******" - ./bsdiff ./test_data/string_tweak_old.bin ./test_data/string_tweak_new.bin ./test-folder/string_tweak_diff.bsdiff - ./bspatch ./test_data/string_tweak_old.bin ./test-folder/string_tweak_patched.bin ./test-folder/string_tweak_diff.bsdiff - diff ./test_data/string_tweak_new.bin ./test-folder/string_tweak_patched.bin - lz4 ./test-folder/string_tweak_diff.bsdiff test-folder/string_tweak_diff.bsdiff.lz4 - lz4 ./test-folder/string_tweak_patched.bin test-folder/string_tweak_patched.bin.lz4 - - @echo "******" - @echo "Test 4: driver add" - @echo "******" - ./bsdiff ./test_data/driver_add_old.bin ./test_data/driver_add_new.bin ./test-folder/driver_add_diff.bsdiff - ./bspatch ./test_data/driver_add_old.bin ./test-folder/driver_add_patched.bin ./test-folder/driver_add_diff.bsdiff - diff ./test_data/driver_add_new.bin ./test-folder/driver_add_patched.bin - lz4 ./test-folder/driver_add_diff.bsdiff test-folder/driver_add_diff.bsdiff.lz4 - lz4 ./test-folder/driver_add_patched.bin test-folder/driver_add_patched.bin.lz4 - - @echo "******" - @echo "Test 5: app update bs patch frame 512" - @echo "******" - ./bsdiff ./test_data/app_update_old.bin ./test_data/app_update_new.bin ./test-folder/app_update_diff.bsdiff - ./bspatch ./test_data/app_update_old.bin ./test-folder/app_update_patched.bin ./test-folder/app_update_diff.bsdiff - diff ./test_data/app_update_new.bin ./test-folder/app_update_patched.bin - lz4 ./test-folder/app_update_diff.bsdiff test-folder/app_update_diff.bsdiff.lz4 - lz4 ./test-folder/app_update_patched.bin test-folder/app_update_patched.bin.lz4 - - - - @echo "Test 7" - @echo "******" - ./bsdiff ./test_data/old1.bin ./test_data/new1.bin ./test-folder/diff1.bsdiff - ./bspatch ./test_data/old1.bin ./test-folder/patched1.bin ./test-folder/diff1.bsdiff - diff ./test_data/new1.bin ./test-folder/patched1.bin - #do check how much smaller it could have been - lz4 ./test-folder/diff1.bsdiff test-folder/diff1.bsdiff.lz4 - lz4 ./test-folder/patched1.bin test-folder/patched1.bin.lz4 - - @echo "Test 8" - @echo "******" - ./bsdiff ./test_data/old2.bin ./test_data/new2.bin ./test-folder/diff2.bsdiff - ./bspatch ./test_data/old2.bin ./test-folder/patched2.bin ./test-folder/diff2.bsdiff - diff ./test_data/new2.bin ./test-folder/patched2.bin - #do check how much smaller it could have been - lz4 ./test-folder/diff2.bsdiff test-folder/diff2.bsdiff.lz4 - lz4 ./test-folder/patched2.bin test-folder/patched2.bin.lz4 - - #do check how much smaller it could have been - #lz4 ./test-folder/diff2.bsdiff test-folder/diff2.bsdiff.lz4 - #lz4 ./test-folder/patched2.bin test-folder/patched2.bin.lz4 - - @echo "*********************************" - ls -la test-folder - -.PHONY: clean -clean: - rm -f bsdiff - rm -f bspatch - rm -f test-folder/* - rm -f *.o - -.PHONY: all -all: bsdiff bspatch - -.PHONY: lines -lines: - cloc . --by-file - -.PHONY: size -size: all - nm --print-size --size-sort bspatch - ldd bspatch - #size -t `ldd bspatch | awk '/=> \// {print $3;}'` diff --git a/delta-tool/bsdiff/bsdiff.c b/delta-tool/bsdiff/bsdiff.c deleted file mode 100755 index 5398ad2..0000000 --- a/delta-tool/bsdiff/bsdiff.c +++ /dev/null @@ -1,607 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * Copyright 2012 Matthew Endsley - * Copyright (c) 2018-2019 ARM Limited - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "bsdiff.h" -#include "varint.h" -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wlong-long" -#include "lz4.h" -#pragma GCC diagnostic pop - -#define MIN(x,y) (((x)<(y)) ? (x) : (y)) -#define MAX_FRAME_SIZE_DEFAULT 512 -//#define TRACING -#ifdef TRACING -#define log(...) printf(__VA_ARGS__) -#else -#define log(...) -#endif - -static void split(int64_t *I, int64_t *V, int64_t start, int64_t len, int64_t h) -{ - int64_t i, j, k, x, tmp, jj, kk; - - if (len < 16) { - for (k = start; k < start + len; k += j) { - j = 1; - x = V[I[k] + h]; - for (i = 1; k + i < start + len; i++) { - if (V[I[k + i] + h] < x) { - x = V[I[k + i] + h]; - j = 0; - }; - if (V[I[k + i] + h] == x) { - tmp = I[k + j]; - I[k + j] = I[k + i]; - I[k + i] = tmp; - j++; - }; - }; - for (i = 0; i < j; i++){ - V[I[k + i]] = k + j - 1; - } - if (j == 1){ - I[k] = -1; - } - }; - return; - }; - - x = V[I[start + len / 2] + h]; - jj = 0; - kk = 0; - for (i = start; i < start + len; i++) { - if (V[I[i] + h] < x){ - jj++; - } - if (V[I[i] + h] == x){ - kk++; - } - }; - jj += start; - kk += jj; - - i = start; - j = 0; - k = 0; - while (i < jj) { - if (V[I[i] + h] < x) { - i++; - } else if (V[I[i] + h] == x) { - tmp = I[i]; - I[i] = I[jj + j]; - I[jj + j] = tmp; - j++; - } else { - tmp = I[i]; - I[i] = I[kk + k]; - I[kk + k] = tmp; - k++; - }; - }; - - while (jj + j < kk) { - if (V[I[jj + j] + h] == x) { - j++; - } else { - tmp = I[jj + j]; - I[jj + j] = I[kk + k]; - I[kk + k] = tmp; - k++; - }; - }; - - if (jj > start){ - split(I, V, start, jj - start, h); - } - - for (i = 0; i < kk - jj; i++){ - V[I[jj + i]] = kk - 1; - } - if (jj == kk - 1){ - I[jj] = -1; - } - - if (start + len > kk){ - split(I, V, kk, start + len - kk, h); - } -} - -static void qsufsort(int64_t *I, int64_t *V, const uint8_t *old, int64_t oldsize) -{ - int64_t buckets[256]; - int64_t i, h, len; - - for (i = 0; i < 256; i++) - buckets[i] = 0; - for (i = 0; i < oldsize; i++) - buckets[old[i]]++; - for (i = 1; i < 256; i++) - buckets[i] += buckets[i - 1]; - for (i = 255; i > 0; i--) - buckets[i] = buckets[i - 1]; - buckets[0] = 0; - - for (i = 0; i < oldsize; i++) - I[++buckets[old[i]]] = i; - I[0] = oldsize; - for (i = 0; i < oldsize; i++) - V[i] = buckets[old[i]]; - V[oldsize] = 0; - for (i = 1; i < 256; i++) - if (buckets[i] == buckets[i - 1] + 1) - I[buckets[i]] = -1; - I[0] = -1; - - for (h = 1; I[0] != -(oldsize + 1); h += h) { - len = 0; - for (i = 0; i < oldsize + 1;) { - if (I[i] < 0) { - len -= I[i]; - i -= I[i]; - } else { - if (len) - I[i - len] = -len; - len = V[I[i]] + 1 - i; - split(I, V, i, len, h); - i += len; - len = 0; - }; - }; - if (len) - I[i - len] = -len; - }; - - for (i = 0; i < oldsize + 1; i++) - I[V[i]] = i; -} - -static int64_t matchlen(const uint8_t *old, int64_t oldsize, const uint8_t *new, int64_t newsize) -{ - int64_t i; - - for (i = 0; (i < oldsize) && (i < newsize); i++) - if (old[i] != new[i]) - break; - - return i; -} - -static int64_t search(const int64_t *I, const uint8_t *old, int64_t oldsize, const uint8_t *new, int64_t newsize, - int64_t st, int64_t en, int64_t *pos) -{ - int64_t x, y; - - if (en - st < 2) { - x = matchlen(old + I[st], oldsize - I[st], new, newsize); - y = matchlen(old + I[en], oldsize - I[en], new, newsize); - - if (x > y) { - *pos = I[st]; - return x; - } else { - *pos = I[en]; - return y; - } - }; - - x = st + (en - st) / 2; - if (memcmp(old + I[x], new, MIN(oldsize - I[x], newsize)) < 0) { - return search(I, old, oldsize, new, newsize, x, en, pos); - } else { - return search(I, old, oldsize, new, newsize, st, x, pos); - }; -} - -static void offtout(int64_t x, uint8_t *buf) -{ - int64_t y; - - if (x < 0) - y = -x; - else - y = x; - - buf[0] = y % 256; - y -= buf[0]; - y = y / 256; - buf[1] = y % 256; - y -= buf[1]; - y = y / 256; - buf[2] = y % 256; - y -= buf[2]; - y = y / 256; - buf[3] = y % 256; - y -= buf[3]; - y = y / 256; - buf[4] = y % 256; - y -= buf[4]; - y = y / 256; - buf[5] = y % 256; - y -= buf[5]; - y = y / 256; - buf[6] = y % 256; - y -= buf[6]; - y = y / 256; - buf[7] = y % 256; - - if (x < 0) - buf[7] |= 0x80; -} - -/** - * Write given buffer to stream in frames, with MAX_FRAME_SIZE as maximum deCompressBuffer and undeCompressBuffer size. - * Store the biggest deCompressBuffer frame size to max_deCompressBuffer_size, if it's bigger than current value. - * @return 0 on success, negative on error. - */ -static int writedeCompressBuffer(struct bsdiff_stream* stream, const void* buffer, int64_t length, - int64_t* max_deCompressBuffer_size, int64_t max_frame_size) -{ - int src_ptr; - char* temp; - int deCompressBuffer_size; - int written = 0; - uint8_t buf[8]; - - if (length == 0) - return 0; - - temp = stream->malloc(max_frame_size); - src_ptr = MIN(length, max_frame_size); - - do { - deCompressBuffer_size = LZ4_compress_destSize(buffer, temp, &src_ptr, max_frame_size); - - if (deCompressBuffer_size == 0) { - stream->free(temp); - return -1; - } - - int encodedSize = encode_unsigned_varint(deCompressBuffer_size, buf, sizeof(buf)); - - log("writedeCompressBuffer %llu compressedSize %u (var intS %u)\n", length, deCompressBuffer_size, encodedSize); - if (encodedSize <= 0) { - return encodedSize; // error in encoding - } - stream->write(stream, buf, encodedSize); - - stream->write(stream, temp, deCompressBuffer_size); - - buffer = (char*) buffer + src_ptr; - written += src_ptr; - - src_ptr = MIN(length - written, max_frame_size); - - if (deCompressBuffer_size > *max_deCompressBuffer_size){ - *max_deCompressBuffer_size = deCompressBuffer_size; - } - - } while (written != length); - - stream->free(temp); - return 0; -} - -struct bsdiff_request { - const uint8_t* old; - int64_t oldsize; - const uint8_t* new; - int64_t newsize; - struct bsdiff_stream* stream; - int64_t *I; - uint8_t *buffer; -}; - -static int bsdiff_internal(const struct bsdiff_request req, int64_t* max_deCompressBuffer_size, - const int64_t max_frame_size) -{ - int64_t *I, *V; - int64_t scan, pos, len; - int64_t lastscan, lastpos, lastoffset; - int64_t oldscore, scsc; - int64_t s, Sf, lenf, Sb, lenb; - int64_t overlap, Ss, lens; - int64_t i; - uint8_t *buffer; - uint8_t buf[8 * 3]; - - if ((V = req.stream->malloc((req.oldsize + 1) * sizeof(int64_t))) == NULL){ - return -1; - } - I = req.I; - - qsufsort(I, V, req.old, req.oldsize); - req.stream->free(V); - - buffer = req.buffer; - - /* Compute the differences, writing ctrl as we go */ - scan = 0; - len = 0; - pos = 0; - lastscan = 0; - lastpos = 0; - lastoffset = 0; - while (scan < req.newsize) { - oldscore = 0; - - for (scsc = scan += len; scan < req.newsize; scan++) { - len = search(I, req.old, req.oldsize, req.new + scan, req.newsize - scan, 0, req.oldsize, &pos); - - for (; scsc < scan + len; scsc++) - if ((scsc + lastoffset < req.oldsize) && (req.old[scsc + lastoffset] == req.new[scsc])) - oldscore++; - - if (((len == oldscore) && (len != 0)) || (len > oldscore + 8)) - break; - - if ((scan + lastoffset < req.oldsize) && (req.old[scan + lastoffset] == req.new[scan])) - oldscore--; - }; - - if ((len != oldscore) || (scan == req.newsize)) { - s = 0; - Sf = 0; - lenf = 0; - for (i = 0; (lastscan + i < scan) && (lastpos + i < req.oldsize);) { - if (req.old[lastpos + i] == req.new[lastscan + i]) - s++; - i++; - if (s * 2 - i > Sf * 2 - lenf) { - Sf = s; - lenf = i; - }; - }; - - lenb = 0; - if (scan < req.newsize) { - s = 0; - Sb = 0; - for (i = 1; (scan >= lastscan + i) && (pos >= i); i++) { - if (req.old[pos - i] == req.new[scan - i]) - s++; - if (s * 2 - i > Sb * 2 - lenb) { - Sb = s; - lenb = i; - }; - }; - }; - - if (lastscan + lenf > scan - lenb) { - overlap = (lastscan + lenf) - (scan - lenb); - s = 0; - Ss = 0; - lens = 0; - for (i = 0; i < overlap; i++) { - if (req.new[lastscan + lenf - overlap + i] == req.old[lastpos + lenf - overlap + i]) - s++; - if (req.new[scan - lenb + i] == req.old[pos - lenb + i]) - s--; - if (s > Ss) { - Ss = s; - lens = i + 1; - }; - }; - - lenf += lens - overlap; - lenb -= lens; - }; - - int64_t diff_str_len = lenf; - int64_t extra_str_len_y = (scan - lenb) - (lastscan + lenf); - int64_t old_file_ctrl_off_set_jump = (pos - lenb) - (lastpos + lenf); - - int posInBuf = 0; - int encodedSize = encode_unsigned_varint(diff_str_len, &buf[posInBuf], 8); - - assert(encodedSize > 0 && encodedSize <= 8); - posInBuf += encodedSize; - - encodedSize = encode_unsigned_varint(extra_str_len_y, &buf[posInBuf], 8); - assert(encodedSize > 0 && encodedSize <= 8); - posInBuf += encodedSize; - - encodedSize = encode_signed_varint(old_file_ctrl_off_set_jump, &buf[posInBuf], 8); - assert(encodedSize > 0 && encodedSize <= 8); - posInBuf += encodedSize; - - log( "DIFF_STR_LEN_X: %lld EXTRA_STR_LEN_Y: %lld OLD_FILE_CTRL_OFF_SET_JUMP %lld (encoded to %i)\n", - lenf, (scan - lenb) - (lastscan + lenf), (pos - lenb) - (lastpos + lenf), posInBuf); - - /* Write control data */ - req.stream->write(req.stream, buf, posInBuf); - - /* Write diff data */ - for (i = 0; i < lenf; i++){ - buffer[i] = req.new[lastscan + i] - req.old[lastpos + i]; - } - if (writedeCompressBuffer(req.stream, buffer, lenf, max_deCompressBuffer_size, max_frame_size)) { - return -1; - } - - /* Write extra data */ - for (i = 0; i < (scan - lenb) - (lastscan + lenf); i++){ - buffer[i] = req.new[lastscan + lenf + i]; - } - if (writedeCompressBuffer(req.stream, buffer, (scan - lenb) - (lastscan + lenf), max_deCompressBuffer_size, - max_frame_size)) { - return -1; - } - - lastscan = scan - lenb; - lastpos = pos - lenb; - lastoffset = pos - scan; - }; - }; - - return 0; -} - -int bsdiff(const uint8_t* old, int64_t oldsize, const uint8_t* new, int64_t newsize, struct bsdiff_stream* stream, - int64_t* max_deCompressBuffer_size, const int64_t max_frame_size) -{ - int result; - struct bsdiff_request req; - - if ((req.I = stream->malloc((oldsize + 1) * sizeof(int64_t))) == NULL) { - return -1; - } - - if ((req.buffer = stream->malloc(newsize + 1)) == NULL) { - stream->free(req.I); - return -1; - } - - req.old = old; - req.oldsize = oldsize; - req.new = new; - req.newsize = newsize; - req.stream = stream; - - result = bsdiff_internal(req, max_deCompressBuffer_size, max_frame_size); - - stream->free(req.buffer); - stream->free(req.I); - - return result; -} - -#if defined(BSDIFF_EXECUTABLE) - -#include - -#include -#include -#include -#include -#include - -static int file_write(struct bsdiff_stream* stream, const void* buffer, - uint64_t size) { - return fwrite(buffer, 1, size, (FILE*) stream->opaque) == size ? 0 : -1; -} - -int main(int argc, char *argv[]) { - int fd; - uint8_t *old, *new; - off_t oldsize, newsize; - uint8_t buf[24]; - FILE * pf; - struct bsdiff_stream stream; - int64_t max_deCompressBuffer_size = 0; - int64_t patch_file_size = 0; - int64_t max_frame_size = MAX_FRAME_SIZE_DEFAULT; - - stream.malloc = malloc; - stream.free = free; - stream.write = file_write; - - if (argc < 4 || argc > 5) { - errx(1, "usage: %s oldfile newfile patchfile \n", - argv[0]); - } - - if (argc == 5) { - max_frame_size = strtol(argv[4], 0, 10); - } else { - printf("Using default max frame size of %d.\n", MAX_FRAME_SIZE_DEFAULT); - } - - if (max_frame_size < 64) { - err(1, "Please define max frame size as integer >= 64"); - } - - /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if (((fd = open(argv[1], O_RDONLY, 0)) < 0) - || ((oldsize = lseek(fd, 0, SEEK_END)) == -1) - || ((old = malloc(oldsize + 1)) == NULL) - || (lseek(fd, 0, SEEK_SET) != 0) - || (read(fd, old, oldsize) != oldsize) || (close(fd) == -1)) { - err(1, "%s", argv[1]); - } - - /* Allocate newsize+1 bytes instead of newsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if (((fd = open(argv[2], O_RDONLY, 0)) < 0) - || ((newsize = lseek(fd, 0, SEEK_END)) == -1) - || ((new = malloc(newsize + 1)) == NULL) - || (lseek(fd, 0, SEEK_SET) != 0) - || (read(fd, new, newsize) != newsize) || (close(fd) == -1)) { - err(1, "%s", argv[2]); - } - - /* Create the patch file */ - if ((pf = fopen(argv[3], "w")) == NULL) { - err(1, "%s", argv[3]); - } - - /* Write header (signature+newsize+max undeCompressBuffer+maxdeCompressBuffer)*/ - offtout(newsize, buf); - offtout(max_frame_size, buf + 8); - offtout(max_deCompressBuffer_size, buf + 16); - if (fwrite("PELION/BSDIFF001", 16, 1, pf) != 1 - || fwrite(buf, sizeof(buf), 1, pf) != 1) - err(1, "Failed to write header"); - - stream.opaque = pf; - if (bsdiff(old, oldsize, new, newsize, &stream, &max_deCompressBuffer_size, - max_frame_size)) { - err(1, "bsdiff"); - } - - /* Go back to header and fill the maxdeCompressBuffer properly */ - offtout(max_deCompressBuffer_size, buf); - fseek(pf, 32, SEEK_SET); - fwrite(buf, 1, 8, pf); - - fseek(pf, 0, SEEK_END); - patch_file_size = ftell(pf); - - if (fclose(pf)) { - err(1, "fclose"); - } - - printf( - "Wrote diff file %s, size %ld. Max undeCompressBuffer frame size was %ld, max deCompressBuffer frame size was %ld.\n", - argv[3], patch_file_size, max_frame_size, - max_deCompressBuffer_size); - - /* Free the memory we used */ - free(old); - free(new); - - return 0; -} - -#endif diff --git a/delta-tool/bsdiff/bsdiff.h b/delta-tool/bsdiff/bsdiff.h deleted file mode 100755 index 8062d0a..0000000 --- a/delta-tool/bsdiff/bsdiff.h +++ /dev/null @@ -1,46 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * Copyright 2012 Matthew Endsley - * Copyright (c) 2018-2019 ARM Limited - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef BSDIFF_H -#define BSDIFF_H - -# include -# include - -struct bsdiff_stream { - void* opaque; - - void* (*malloc)(size_t size); - void (*free)(void* ptr); - int (*write)(struct bsdiff_stream* stream, const void* buffer, - uint64_t size); -}; - -int bsdiff(const uint8_t* old, int64_t oldsize, const uint8_t* new, int64_t newsize, struct bsdiff_stream* stream, int64_t* max_deCompressBuffer_size, const int64_t max_frame_size); - -#endif diff --git a/delta-tool/bsdiff/bspatch_main.c b/delta-tool/bsdiff/bspatch_main.c deleted file mode 100755 index afb1113..0000000 --- a/delta-tool/bsdiff/bspatch_main.c +++ /dev/null @@ -1,172 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * Copyright 2012 Matthew Endsley - * Copyright (c) 2018-2019 ARM Limited - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include - -#include -#include -#include -#include -#include -#include - -#include "bspatch.h" - -#define MAX_NEXT_SIZE 512 - -int64_t batch_read_size; - -static int virtualDataAmount = 0; -static int nextEventToPostToBsPatch = 0; -static int readCounter = 0; - -static int seek_old(const struct bspatch_stream* stream, int64_t seek_diff) { - - FILE* f = ((FILE**) (ARM_BS_GetOpaque(stream)))[1]; - assert("failure in seek" && fseek(f, seek_diff, SEEK_CUR)==0); - - nextEventToPostToBsPatch = EBSAPI_SEEK_OLD_DONE; - return EBSAPI_OPERATION_OLD_FILE_SEEK_WILL_COMPLETE_LATER; -} - -static int read_old(const struct bspatch_stream* stream, void* buffer, - uint64_t length) { - FILE* f = ((FILE**) ( ARM_BS_GetOpaque(stream) ))[1]; - unsigned int read = (unsigned int)fread(buffer, 1, length, f); - if ( read != length) - printf("Read old err: %u (%lu) - %ld\n", read, length, - ftell(f)); - - assert(read == length); - if( readCounter % 2 == 0) { - nextEventToPostToBsPatch = EBSAPI_READ_OLD_DONE; - return EBSAPI_OPERATION_OLD_FILE_READ_WILL_COMPLETE_LATER; - } else { - return EBSAPI_OPERATION_DONE_IMMEDIATELY; - } -} - -int PATCH_TRACING = 0; -static int read_patch(const struct bspatch_stream* stream, void* buffer, - uint64_t length) { - readCounter ++; - virtualDataAmount -= length; - int read; - batch_read_size -= length; - FILE* f = ((FILE**) ARM_BS_GetOpaque(stream))[0]; - read = fread(buffer, 1, length, f); - - if (read < 0 || (unsigned int) read != length) - printf("Read patch err: %d (%d)\n", read, 0); - - assert(nextEventToPostToBsPatch == 0); - - if(PATCH_TRACING) { - printf(" (%u)B\n",(uint32_t)length); - PATCH_TRACING = 0; - } - // lets fake later completion here every other time - if( readCounter % 2 == 0) { - nextEventToPostToBsPatch = EBSAPI_READ_PATCH_DONE; - return EBSAPI_OPERATION_PATCH_READ_WILL_COMPLETE_LATER; - } else { - return EBSAPI_OPERATION_DONE_IMMEDIATELY; - } -} - -static int write_new(const struct bspatch_stream* stream, void* buffer, - uint64_t length) { - - FILE* f = ((FILE**) ARM_BS_GetOpaque(stream))[2]; - assert(fwrite(buffer, 1, length, f) == length); - - if( readCounter % 2 == 0) { - nextEventToPostToBsPatch = EBSAPI_WRITE_NEW_DONE; - return EBSAPI_OPERATION_NEW_FILE_WRITE_WILL_COMPLETE_LATER; - } else { - return EBSAPI_OPERATION_DONE_IMMEDIATELY; - } - -} - -// lets make this global to have size better visible in compile time -struct bspatch_stream bs_patch; - -#include "bspatch_private.h" -int main(int argc, char * argv[]) { - FILE* f[3]; - - int status; - - if (argc != 4) - errx(1, "usage: %s oldfile newfile patchfile\n", argv[0]); - - /* Open patch file */ - if ((f[0] = fopen(argv[3], "rb")) == NULL) - err(1, "fopen(%s)", argv[3]); - - /* Open old file */ - if ((f[1] = fopen(argv[1], "rb")) == NULL) - err(1, "fopen(%s)", argv[1]); - - /* Open new file */ - if ((f[2] = fopen(argv[2], "wbx")) == NULL) - err(1, "fopen(%s)", argv[2]); - - ARM_BS_Init(&bs_patch, f, - read_patch, - read_old, - seek_old, - write_new ); - - printf("BSPatch Size: %lu B\n", sizeof(bs_patch)); - - /* Run bspatching */ - status = ARM_BS_ProcessPatchEvent(&bs_patch, EBSAPI_START_PATCH_PROCESSING); - if(status != EBSAPI_PATCH_DONE) { - do { - assert(nextEventToPostToBsPatch); - bs_patch_api_event_t eventToPost = nextEventToPostToBsPatch; - nextEventToPostToBsPatch = 0; // zero the global to make asserts on implementation to work. assumes to be zero when new request arrives - status = ARM_BS_ProcessPatchEvent(&bs_patch, eventToPost); - } while (status > 0 && status != EBSAPI_PATCH_DONE); - } - - status = ARM_BS_Free(&bs_patch); - - if (status != 0) - errx(1, "bspatch fail: %d", status); - - /* close file handles */ - fclose(f[2]); - fclose(f[1]); - fclose(f[0]); - - return 0; -} - diff --git a/delta-tool/include/bspatch.h b/delta-tool/include/bspatch.h deleted file mode 100755 index a0cf824..0000000 --- a/delta-tool/include/bspatch.h +++ /dev/null @@ -1,131 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * Copyright 2012 Matthew Endsley - * Copyright (c) 2018-2019 ARM Limited - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef BSPATCH_H -#define BSPATCH_H - -#include - - -// BS_PATCH_COMPILE_TIME_MEMORY_ALLOC this allows allocating memory to bsdiff during compile time -// basically to be safe this should be 2*framesize defined to bsdiff. -//#define BS_PATCH_COMPILE_TIME_MEMORY_ALLOC 1024 -#ifndef BS_PATCH_COMPILE_TIME_MEMORY_ALLOC -#warning "using default memory allocation" -#define BS_PATCH_COMPILE_TIME_MEMORY_ALLOC 1024 -#endif - - -struct bspatch_stream; - -// todo compile time assert from some header -#define COMPILE_TIME_ASSERT(condition) _impl_CASSERT_LINE(condition,__LINE__,__FILE__) - -#define _impl_PASTE(a,b) a##b -#define _impl_CASSERT_LINE(predicate, line, file) \ - typedef char _impl_PASTE(assertion_failed_##file##_,line)[2*!!(predicate)-1]; - -#if defined( BS_PATCH_COMPILE_TIME_MEMORY_ALLOC ) && (BS_PATCH_COMPILE_TIME_MEMORY_ALLOC>0) -COMPILE_TIME_ASSERT(BS_PATCH_COMPILE_TIME_MEMORY_ALLOC%sizeof(int64_t)==0) -#endif - -// events used to feed ARM_BS_ProcessPatchEvent -typedef enum { - EBSAPI_START_PATCH_PROCESSING = 300, - EBSAPI_READ_PATCH_DONE, - EBSAPI_READ_OLD_DONE, - EBSAPI_SEEK_OLD_DONE, - EBSAPI_WRITE_NEW_DONE -}bs_patch_api_event_t; - -typedef enum { - // errors - EBSAPI_ERR_INVALID_STATE = -7, - EBSAPI_ERR_UNEXPECTED_EVENT = -6, - EBSAPI_ERR_ALREADY_INIT = -5, - EBSAPI_ERR_OUT_OF_MEMORY = -4, - EBSAPI_ERR_PARAMETERS = -3, /* Values in struct bspatch_stream didn't make sense (callback was NULL, too small buffer_size... */ - EBSAPI_ERR_FILE_IO= -2, /* One of the read/write calls returned an error. */ - EBSAPI_ERR_CORRUPTED_PATCH = -1, - // code for sync completion - EBSAPI_OPERATION_DONE_IMMEDIATELY = 0, - // returned when patching is ready and no more events are waited - EBSAPI_PATCH_DONE, - // used by corresponding BS API functions to indicate asynch completion of operation - EBSAPI_OPERATION_PATCH_READ_WILL_COMPLETE_LATER = 400, - EBSAPI_OPERATION_OLD_FILE_READ_WILL_COMPLETE_LATER, - EBSAPI_OPERATION_OLD_FILE_SEEK_WILL_COMPLETE_LATER, - EBSAPI_OPERATION_NEW_FILE_WRITE_WILL_COMPLETE_LATER -}bs_patch_api_return_code_t; - -typedef struct bspatch_stream bspatch_stream; - -// functions pointer prototypes for BS_PATCH_API -typedef bs_patch_api_return_code_t (*read_patch_f)(const bspatch_stream* stream, void* buffer, uint64_t length); -typedef bs_patch_api_return_code_t (*read_old_f)(const bspatch_stream* stream, void* buffer, uint64_t length); -typedef bs_patch_api_return_code_t (*seek_old_f)(const bspatch_stream* stream, int64_t seek_diff); -typedef bs_patch_api_return_code_t (*write_new_f)(const bspatch_stream* stream, void* buffer, uint64_t length); - -/** - * Initialize the bspatch control structure. - * @param stream, a pointer to memory allocated for BsPatch structure. This will need to be valid for life time of patching - * @param opaque, pointer to anything the implementor of rpf, rof, sof and wnf functions will need. This can be accessed with name opaque - * bspatch_stream structure that is delivered to each call of rpf, rof, sof and wnf. - * @param rpf, a function pointer to a function capable in reading patch data - * @param rof, a function pointer to a function capable reading old file to be patched - * @param sof, a function pointer to a function capable seeking the position in old file to be patched - * @param wnf, a function pointer to a function capable writing new resulting file of patching - */ -void ARM_BS_Init( bspatch_stream* stream, void* opaque, - read_patch_f rpf, read_old_f rof, - seek_old_f sof, write_new_f wnf); - -/** - * Handles patch processing events. - * @param stream control structure, with callbacks for data i/o - * @param bsApiEvent @see BsPatchApiEvent, indication of asynch operation completing or EBsAPI_StartPatchProcessing to start patching - * @return @see BsPatchApiReturnCode. EBsApiPatchDone if success and whole new file has been written, or code indicating asynch completion - * from one of the API calls or any other non synch success code returned by API - */ -bs_patch_api_return_code_t ARM_BS_ProcessPatchEvent(bspatch_stream* stream, bs_patch_api_event_t bsApiEvent); - -/** - * Gets opaque pointer from BS patch - * @param stream pointer to relevant bspatch instance - * @return opaque pointer given to bspatch during initialization with ARM_BS_Init . - */ -void* ARM_BS_GetOpaque(const bspatch_stream* stream); - -/** - * Frees the resources allocated by bspatch - * @param stream pointer to relevant bspatch instance - * @return 0 - */ -int ARM_BS_Free(bspatch_stream* stream); - -#endif diff --git a/delta-tool/include/bspatch_private.h b/delta-tool/include/bspatch_private.h deleted file mode 100644 index 7ca0356..0000000 --- a/delta-tool/include/bspatch_private.h +++ /dev/null @@ -1,120 +0,0 @@ -// ---------------------------------------------------------------------------- -// Copyright 2019 ARM Ltd. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------- - -#ifndef INCLUDE_BSPATCH_PRIVATE_H_ -#define INCLUDE_BSPATCH_PRIVATE_H_ - - -#include "bspatch.h" - -/* Patch applied succesfully, but new file is not ready yet. User should re-fill patch buffer and call bspatch again. */ -#define BSPATCH_NEED_MORE_PATCH_DATA 1 - -/* Patch applied succesfully, and new file is fully written. */ -#define BSPATCH_DONE 0 -#define FILE_MAGIC "PELION/BSDIFF001" -#define FILE_MAGIC_LEN (sizeof(FILE_MAGIC) - 1) - -#define CONTROL_LEN (8 * 3) -#define FILE_HEADER_LEN (FILE_MAGIC_LEN + 24) - -// internal states of BS patch state machine -typedef enum { - EBsInitial = 100, - EBsInitialPatchReadDone, - EBsAllocWorkingBuffer, - EBsReadInitialHeader, - EBsReadControlSegmentHeader, - EBsReadControlSegment, - EBsReadDataSegment, - EBspatch_processDiffBytes_readHeaderPiece, - EBspatch_processDiffBytes_processSinglePiece, - EBspatch_processDiffBytes_processSinglePieceContinue, - EBspatch_processDiffBytes_processSinglePieceInit, - EBspatch_processDiffBytes_processSinglePieceContinue2, - EBspatch_processDiffBytes_processSinglePieceContinue_writePart, - EBspatch_processDiffBytes_processSinglePieceContinue_postActions, - - EBsProcessExtraLen, - EBsProcessExtraLen_readHeader, - EBsProcessExtraLen_singleItem, - EBsProcessExtraLen_singleItemContinue, - EBsProcessExtraLen_singleItemContinuePostStep, - EBsProcessExtraLen_postStep, - - EBspatch_read_frame_len, - EBspatch_read_frame_len_piece_read, - EBspatch_read_varintPiece, - EReadCtrl_diff_str_len, - EReadCtrl_extra_str_len_y, - EReadCtrl_old_file_ctrl_off_set_jump, - EBsPatch_process_varintPiece - -} bs_patch_state_t; - -/* Control structure for bspatch - * opaque Is not used by bspatch but can be used e.g. by caller to store file handle(s). - * read_patch When called, should read next length bytes from the patch file into buffer and return 0 on success. - * read_old When called, should read next length bytes from the base file into buffer and return 0 on success. - * seek_old When called, should reposition the old file with seek_diff amount in relation to current position and return 0 on success. - * write_new When called, should write length bytes from buffer into end of the new file and return 0 on success. - */ -struct bspatch_stream { - - /* To be filled by ARM_BS_Init only thing used outside */ - void* opaque; - read_patch_f read_patch; - read_old_f read_old; - seek_old_f seek_old; - write_new_f write_new; - - // private members to BS patching. // not to be used by API implementor for other than debug - int64_t undeCompressBuffer_len; - int64_t total_undeCompressBuffer; /* keeping track of total undeCompressBuffer bytes */ - uint64_t var_int; - uint64_t var_int_len; - uint64_t frame_len; - int64_t newpos; /* keeping track of position of the resulting file */ - int64_t i; - int64_t ctrl[3]; - - uint8_t* nonCompressedDataBuffer; /* buffer for undeCompressBuffer bytes */ - uint8_t* bufferForCompressedData; /* buffer to store deCompressBuffer frame for decompression */ - - uint32_t progress; - uint32_t new_size; - // for compress support - uint32_t max_compressedDataBuffer; - uint32_t max_deCompressBuffer; - uint32_t readBufferCurrentSize; - uint32_t readRequestSize ; - uint8_t header[FILE_HEADER_LEN]; // potential padding here - bs_patch_state_t next_state; - bs_patch_api_event_t expectedExternalEvent; - bs_patch_state_t stateAfterReadVarInt; - uint32_t isSignedVarInt; - -#if defined( BS_PATCH_COMPILE_TIME_MEMORY_ALLOC ) && (BS_PATCH_COMPILE_TIME_MEMORY_ALLOC>0) - int64_t bsMemoryBuffer[BS_PATCH_COMPILE_TIME_MEMORY_ALLOC/sizeof(int64_t)]; // - uint8_t allignmentBuffer[7+7]; // to allow space to alling to 8 byte boundary properly. -#endif -}; - - - -#endif /* INCLUDE_BSPATCH_PRIVATE_H_ */ diff --git a/delta-tool/include/lz4.h b/delta-tool/include/lz4.h deleted file mode 100755 index e873c13..0000000 --- a/delta-tool/include/lz4.h +++ /dev/null @@ -1,660 +0,0 @@ -/* - * LZ4 - Fast LZ compression algorithm - * Header File - * Copyright (C) 2011-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 - */ -#if defined (__cplusplus) -extern "C" { -#endif - -#ifndef LZ4_H_2983827168210 -#define LZ4_H_2983827168210 - -/* --- Dependency --- */ -#include /* size_t */ - -/** - Introduction - - LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, - scalable with multi-cores CPU. It features an extremely fast decoder, with speed in - multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. - - The LZ4 compression library provides in-memory compression and decompression functions. - Compression can be done in: - - a single step (described as Simple Functions) - - a single step, reusing a context (described in Advanced Functions) - - unbounded multiple steps (described as Streaming compression) - - lz4.h provides block compression functions. It gives full buffer control to user. - Decompressing an lz4-deCompressBuffer block also requires metadata (such as deCompressBuffer size). - Each application is free to encode such metadata in whichever way it wants. - - An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), - take care of encoding standard metadata alongside LZ4-deCompressBuffer blocks. - Frame format is required for interoperability. - It is delivered through a companion API, declared in lz4frame.h. - */ - -/*^*************************************************************** - * Export parameters - *****************************************************************/ -/* - * LZ4_DLL_EXPORT : - * Enable exporting of functions when building a Windows DLL - * LZ4LIB_VISIBILITY : - * Control library symbols visibility. - */ -#ifndef LZ4LIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) -# else -# define LZ4LIB_VISIBILITY -# endif -#endif -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define LZ4LIB_API LZ4LIB_VISIBILITY -#endif - -/*------ Version ------*/ -#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ - -#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) - -#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE -#define LZ4_QUOTE(str) #str -#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) -#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) - -LZ4LIB_API int LZ4_versionNumber(void); /**< library version number; useful to check dll version */ -LZ4LIB_API const char* LZ4_versionString(void); /**< library version string; useful to check dll version */ - -/*-************************************ - * Tuning parameter - **************************************/ -/*! - * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio. - * Reduced memory usage may improve speed, thanks to better cache locality. - * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache - */ -#ifndef LZ4_MEMORY_USAGE -# define LZ4_MEMORY_USAGE 14 -#endif - -/*-************************************ - * Simple Functions - **************************************/ -/*! LZ4_compress_default() : - Compresses 'srcSize' bytes from buffer 'src' - into already allocated 'dst' buffer of size 'dstCapacity'. - Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'src' into a more limited 'dst' budget, - compression stops *immediately*, and the function result is zero. - In which case, 'dst' content is undefined (invalid). - srcSize : max supported value is LZ4_MAX_INPUT_SIZE. - dstCapacity : size of buffer 'dst' (which must be already allocated) - @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails - Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). - */ -LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, - int dstCapacity); - -/*! LZ4_decompress_safe() : - deCompressBufferSize : is the exact complete size of the deCompressBuffer block. - dstCapacity : is the size of destination buffer, which must be already allocated. - @return : the number of bytes dedeCompressBuffer into destination buffer (necessarily <= dstCapacity) - If destination buffer is not large enough, decoding will stop and output an error code (negative value). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). - */ -LZ4LIB_API int LZ4_decompress_safe(const char* src, char* dst, - int deCompressBufferSize, int dstCapacity); - -/*-************************************ - * Advanced Functions - **************************************/ -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) - -/*! LZ4_compressBound() : - Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) - inputSize : max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is incorrect (too large or negative) - */ -LZ4LIB_API int LZ4_compressBound(int inputSize); - -/*! LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows selection of "acceleration" factor. - The larger the acceleration value, the faster the algorithm, but also the lesser the compression. - It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. - An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c). - */ -LZ4LIB_API int LZ4_compress_fast(const char* src, char* dst, int srcSize, - int dstCapacity, int acceleration); - -/*! LZ4_compress_fast_extState() : - * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. - * Use LZ4_sizeofState() to know how much memory must be allocated, - * and allocate it on 8-bytes boundaries (using `malloc()` typically). - * Then, provide this buffer as `void* state` to compression function. - */ -LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState(void* state, const char* src, - char* dst, int srcSize, int dstCapacity, int acceleration); - -/*! LZ4_compress_destSize() : - * Reverse the logic : compresses as much data as possible from 'src' buffer - * into already allocated buffer 'dst', of size >= 'targetDestSize'. - * This function either compresses the entire 'src' content into 'dst' if it's large enough, - * or fill 'dst' buffer completely with as much data as possible from 'src'. - * note: acceleration parameter is fixed to "default". - * - * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. - * New value is necessarily <= input value. - * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) - * or 0 if compression fails. - */ -LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, - int* srcSizePtr, int targetDstSize); - -/*! LZ4_decompress_fast() : **unsafe!** - * This function used to be a bit faster than LZ4_decompress_safe(), - * though situation has changed in recent versions, - * and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`. - * Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of deCompressBuffer data. - * As a consequence, this function is no longer recommended, and may be deprecated in future versions. - * It's last remaining specificity is that it can decompress data without knowing its deCompressBuffer size. - * - * originalSize : is the undeCompressBuffer size to regenerate. - * `dst` must be already allocated, its size must be >= 'originalSize' bytes. - * @return : number of bytes read from source buffer (== deCompressBuffer size). - * If the source stream is detected malformed, the function stops decoding and returns a negative result. - * note : This function requires undeCompressBuffer originalSize to be known in advance. - * The function never writes past the output buffer. - * However, since it doesn't know its 'src' size, it may read past the intended input. - * Also, because match offsets are not validated during decoding, - * reads from 'src' may underflow. - * Use this function in trusted environment **only**. - */ -LZ4LIB_API int LZ4_decompress_fast(const char* src, char* dst, - int originalSize); - -/*! LZ4_decompress_safe_partial() : - * Decompress an LZ4 deCompressBuffer block, of size 'srcSize' at position 'src', - * into destination buffer 'dst' of size 'dstCapacity'. - * Up to 'targetOutputSize' bytes will be decoded. - * The function stops decoding on reaching this objective, - * which can boost performance when only the beginning of a block is required. - * - * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity) - * If source stream is detected malformed, function returns a negative result. - * - * Note : @return can be < targetOutputSize, if deCompressBuffer block contains less data. - * - * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity, - * and expects targetOutputSize <= dstCapacity. - * It effectively stops decoding on reaching targetOutputSize, - * so dstCapacity is kind of redundant. - * This is because in a previous version of this function, - * decoding operation would not "break" a sequence in the middle. - * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize, - * it could write more bytes, though only up to dstCapacity. - * Some "margin" used to be required for this operation to work properly. - * This is no longer necessary. - * The function nonetheless keeps its signature, in an effort to not break API. - */ -LZ4LIB_API int LZ4_decompress_safe_partial(const char* src, char* dst, - int srcSize, int targetOutputSize, int dstCapacity); - -/*-********************************************* - * Streaming Compression Functions - ***********************************************/ -typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ - -LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); -LZ4LIB_API int LZ4_freeStream(LZ4_stream_t* streamPtr); - -/*! LZ4_resetStream() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to start compressing a new stream. - */ -LZ4LIB_API void LZ4_resetStream(LZ4_stream_t* streamPtr); - -/*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream_t. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed, and is the same as reset. - * @return : dictionary size, in bytes (necessarily <= 64 KB) - */ -LZ4LIB_API int LZ4_loadDict(LZ4_stream_t* streamPtr, const char* dictionary, - int dictSize); - -/*! LZ4_compress_fast_continue() : - * Compress 'src' content using data from previously deCompressBuffer blocks, for better compression ratio. - * 'dst' buffer must be already allocated. - * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * - * @return : size of deCompressBuffer block - * or 0 if there is an error (typically, cannot fit into 'dst'). - * - * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. - * Each block has precise boundaries. - * Each block must be dedeCompressBuffer separately, calling LZ4_decompress_*() with relevant metadata. - * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. - * - * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! - * - * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. - * Make sure that buffers are separated, by at least one byte. - * This construction ensures that each block only depends on previous block. - * - * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. - * - * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. - */ -LZ4LIB_API int LZ4_compress_fast_continue(LZ4_stream_t* streamPtr, - const char* src, char* dst, int srcSize, int dstCapacity, - int acceleration); - -/*! LZ4_saveDict() : - * If last 64KB data cannot be guaranteed to remain available at its current memory location, - * save it into a safer place (char* safeBuffer). - * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), - * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. - * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. - */ -LZ4LIB_API int LZ4_saveDict(LZ4_stream_t* streamPtr, char* safeBuffer, - int maxDictSize); - -/*-********************************************** - * Streaming Decompression Functions - * Bufferless synchronous API - ************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ - -/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : - * creation / destruction of streaming decompression tracking context. - * A tracking context can be re-used multiple times. - */ -LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); -LZ4LIB_API int LZ4_freeStreamDecode(LZ4_streamDecode_t* LZ4_stream); - -/*! LZ4_setStreamDecode() : - * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. - * Use this function to start decompression of a new stream of blocks. - * A dictionary can optionally be set. Use NULL or size 0 for a reset order. - * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. - * @return : 1 if OK, 0 if error - */ -LZ4LIB_API int LZ4_setStreamDecode(LZ4_streamDecode_t* LZ4_streamDecode, - const char* dictionary, int dictSize); - -/*! LZ4_decoderRingBufferSize() : v1.8.2+ - * Note : in a ring buffer scenario (optional), - * blocks are presumed dedeCompressBuffer next to each other - * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), - * at which stage it resumes from beginning of ring buffer. - * When setting such a ring buffer for streaming decompression, - * provides the minimum size of this ring buffer - * to be compatible with any source respecting maxBlockSize condition. - * @return : minimum ring buffer size, - * or 0 if there is an error (invalid maxBlockSize). - */ -LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); -#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ - -/*! LZ4_decompress_*_continue() : - * These decoding functions allow decompression of consecutive blocks in "streaming" mode. - * A block is an unsplittable entity, it must be presented entirely to a decompression function. - * Decompression functions only accepts one block at a time. - * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. - * If less than 64KB of data has been decoded, all the data must be present. - * - * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : - * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). - * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. - * In which case, encoding and decoding buffers do not need to be synchronized. - * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. - * - Synchronized mode : - * Decompression buffer size is _exactly_ the same as compression buffer size, - * and follows exactly same update rule (block boundaries at same positions), - * and decoding function is provided with exact dedeCompressBuffer size of each block (exception for last block of the stream), - * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). - * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. - * In which case, encoding and decoding buffers do not need to be synchronized, - * and encoding ring buffer can have any size, including small ones ( < 64 KB). - * - * Whenever these conditions are not possible, - * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, - * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. - */ -LZ4LIB_API int LZ4_decompress_safe_continue( - LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, - int srcSize, int dstCapacity); -LZ4LIB_API int LZ4_decompress_fast_continue( - LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, - int originalSize); - -/*! LZ4_decompress_*_usingDict() : - * These decoding functions work the same as - * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() - * They are stand-alone, and don't need an LZ4_streamDecode_t structure. - * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. - * Performance tip : Decompression speed can be substantially increased - * when dst == dictStart + dictSize. - */ -LZ4LIB_API int LZ4_decompress_safe_usingDict(const char* src, char* dst, - int srcSize, int dstCapcity, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict(const char* src, char* dst, - int originalSize, const char* dictStart, int dictSize); - -/*^********************************************** - * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ - -/*-************************************ - * Unstable declarations - ************************************** - * Declarations in this section must be considered unstable. - * Their signatures may change, or may be removed in the future. - * They are therefore only safe to depend on - * when the caller is statically linked against the library. - * To access their declarations, define LZ4_STATIC_LINKING_ONLY. - **************************************/ - -#ifdef LZ4_STATIC_LINKING_ONLY - -/* By default, symbols in this section aren't published into shared/dynamic libraries. - * You can override this behavior and force them to be published - * by defining LZ4_PUBLISH_STATIC_FUNCTIONS. - * Use at your own risk. - */ - -#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS -#define LZ4LIB_STATIC_API LZ4LIB_API -#else -#define LZ4LIB_STATIC_API -#endif - -/*! LZ4_resetStream_fast() : - * Use this, like LZ4_resetStream(), to prepare a context for a new chain of - * calls to a streaming API (e.g., LZ4_compress_fast_continue()). - * - * Note: - * Using this in advance of a non-streaming-compression function is redundant, - * since they all perform their own custom reset internally. - * - * Differences from LZ4_resetStream(): - * When an LZ4_stream_t is known to be in a internally coherent state, - * it can often be prepared for a new compression with almost no work, - * only sometimes falling back to the full, expensive reset - * that is always required when the stream is in an indeterminate state - * (i.e., the reset performed by LZ4_resetStream()). - * - * LZ4_streams are guaranteed to be in a valid state when: - * - returned from LZ4_createStream() - * - reset by LZ4_resetStream() - * - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged - * - the stream was in a valid state and was reset by LZ4_resetStream_fast() - * - the stream was in a valid state and was then used in any compression call - * that returned success - * - the stream was in an indeterminate state and was used in a compression - * call that fully reset the state (e.g., LZ4_compress_fast_extState()) and - * that returned success - * - * When a stream isn't known to be in a valid state, it is not safe to pass to - * any fastReset or streaming function. It must first be cleansed by the full - * LZ4_resetStream(). - */ -LZ4LIB_STATIC_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); - -/*! LZ4_compress_fast_extState_fastReset() : - * A variant of LZ4_compress_fast_extState(). - * - * Using this variant avoids an expensive initialization step. - * It is only safe to call if the state buffer is known to be correctly initialized already - * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). - * From a high level, the difference is that - * this function initializes the provided state with a call to something like LZ4_resetStream_fast() - * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). - */ -LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); - -/*! LZ4_attach_dictionary() : - * This is an experimental API that allows - * efficient use of a static dictionary many times. - * - * Rather than re-loading the dictionary buffer into a working context before - * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a - * working LZ4_stream_t, this function introduces a no-copy setup mechanism, - * in which the working stream references the dictionary stream in-place. - * - * Several assumptions are made about the state of the dictionary stream. - * Currently, only streams which have been prepared by LZ4_loadDict() should - * be expected to work. - * - * Alternatively, the provided dictionaryStream may be NULL, - * in which case any existing dictionary stream is unset. - * - * If a dictionary is provided, it replaces any pre-existing stream history. - * The dictionary contents are the only history that can be referenced and - * logically immediately precede the data deCompressBuffer in the first subsequent - * compression call. - * - * The dictionary will only remain attached to the working stream through the - * first compression call, at the end of which it is cleared. The dictionary - * stream (and source buffer) must remain in-place / accessible / unchanged - * through the completion of the first compression call on the stream. - */ -LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); - -#endif - -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions directly. - * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Accessing members will expose code to API and/or ABI break in future versions of the library. - **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ - -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; -struct LZ4_stream_t_internal { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint16_t initCheck; - uint16_t tableType; - const uint8_t* dictionary; - const LZ4_stream_t_internal* dictCtx; - uint32_t dictSize; -}; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#else - -typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; -struct LZ4_stream_t_internal { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned short initCheck; - unsigned short tableType; - const unsigned char* dictionary; - const LZ4_stream_t_internal* dictCtx; - unsigned int dictSize; -}; - -typedef struct { - const unsigned char* externalDict; - size_t extDictSize; - const unsigned char* prefixEnd; - size_t prefixSize; -}LZ4_streamDecode_t_internal; - -#endif - -/*! LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure with LZ4_resetStream() before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * it may change in a future version ! - */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) -union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; - LZ4_stream_t_internal internal_donotuse; -}; -/* previously typedef'd to LZ4_stream_t */ - -/*! LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode() before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! - */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -union LZ4_streamDecode_u { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; - LZ4_streamDecode_t_internal internal_donotuse; -}; -/* previously typedef'd to LZ4_streamDecode_t */ - -/*-************************************ - * Obsolete Functions - **************************************/ - -/*! Deprecation warnings - Should deprecation warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ -#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS -# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ -#else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) -# elif defined(_MSC_VER) -# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) -# else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) -# endif -#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ - -/* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress( - const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput( - const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState( - void* state, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState( - void* state, const char* source, char* dest, int inputSize, - int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue( - LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, - int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue( - LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, - int inputSize, int maxOutputSize); - -/* Obsolete decompression functions */ -LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress( - const char* source, char* dest, int outputSize); -LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize( - const char* source, char* dest, int isize, int maxOutputSize); - -/* Obsolete streaming functions; degraded functionality; do not use! - * - * In order to perform streaming compression, these functions depended on data - * that is no longer tracked in the state. They have been preserved as well as - * possible: using them will still produce a correct output. However, they don't - * actually retain any history between compression calls. The compression ratio - * achieved will therefore be no better than compressing each chunk - * independently. - */ -LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create( - char* inputBuffer); -LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState( - void); -LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState( - void* state, char* inputBuffer); -LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer( - void* state); - -/* Obsolete streaming decoding functions */ -LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k( - const char* src, char* dst, int deCompressBufferSize, int maxDstSize); -LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k( - const char* src, char* dst, int originalSize); - -#endif /* LZ4_H_2983827168210 */ - -#if defined (__cplusplus) -} -#endif diff --git a/delta-tool/include/varint.h b/delta-tool/include/varint.h deleted file mode 100644 index 89806e3..0000000 --- a/delta-tool/include/varint.h +++ /dev/null @@ -1,40 +0,0 @@ -// ---------------------------------------------------------------------------- -// Copyright 2019 ARM Ltd. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------- - -#ifndef VARINT_H -#define VARINT_H - -#include - -typedef enum { - ERR_BUFFER_TOO_SMALL = -7, - ERR_PARAMETTERS = -6, - OPERATION_COMPLETED = 0, - OPERATION_NEEDS_MORE_DATA = 1 -}var_int_op_code; - -// decodes varint with multiple calls one byte at a time, returns 1 of more data is needed caller should pass -// number of calls already done int count -var_int_op_code decode_unsigned_varint(unsigned char varIntByte, uint64_t* varIntValue, int count); -var_int_op_code decode_signed_varint(unsigned char varIntByte, int64_t* varIntValue, int count); - -// encodes varint to stream, BUFF_SIZE_MAX should containt maxbytes in buf to avoid overwrites -int encode_unsigned_varint(uint64_t value, unsigned char *buf, uint32_t BUFF_SIZE_MAX); -int encode_signed_varint(int64_t value, unsigned char *buf, uint32_t BUFF_SIZE_MAX); - -#endif diff --git a/delta-tool/mbed_lib.json b/delta-tool/mbed_lib.json deleted file mode 100644 index 8947fda..0000000 --- a/delta-tool/mbed_lib.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "delta-tool", - "config": { - "BS_PATCH_COMPILE_TIME_MEMORY_ALLOC": { - "help": "memory allocated statically during compile time to bspatch module, will need to be around 2* frame size defined in bsdiff during delta creation. If defined to 0, malloc will be used instead", - "macro_name": "BS_PATCH_COMPILE_TIME_MEMORY_ALLOC", - "value": 1024 - } - } -} diff --git a/delta-tool/source/bspatch.c b/delta-tool/source/bspatch.c deleted file mode 100755 index 94f8094..0000000 --- a/delta-tool/source/bspatch.c +++ /dev/null @@ -1,770 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * Copyright 2012 Matthew Endsley - * Copyright (c) 2018-2019 ARM Limited - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/*** - * TODO: - * - change to 32-bit addressing - */ -#include "bspatch.h" -#include "bspatch_private.h" -#include "varint.h" - -#include -#include -#include -#include - -//#define PATCH_STAT_COUNTING 1 -#ifndef BS_DEBUG -#define BS_DEBUG 0 -#endif - -#ifndef BS_HIGH_LEVEL_DEBUG -#define BS_HIGH_LEVEL_DEBUG 0 -#endif - -#if (BS_DEBUG==1) -// for debug -#include -#include -#define log(...) printf(__VA_ARGS__) -// for debug end -#else // remember anything inside assert will not be executed in product code as it is done like this. -// so do not do like assert(readSomething() == true); as things inside assert will not be included in final compile -#define log(...) -#define assert(condition) - -#endif // BS_DEBUG - -#if BS_DEBUG -//extern int PATCH_TRACING; -#define TRACE_PATCH_CONTROL_READ(...) log(__VA_ARGS__)//PATCH_TRACING = 1; -#else -#define TRACE_PATCH_CONTROL_READ(...) -#endif // BS_DEBUG - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wlong-long" -#include "lz4.h" -#pragma GCC diagnostic pop - -#define MIN(x,y) (((x)<(y)) ? (x) : (y)) - -#define DIFF_STR_LEN_X 0 -#define EXTRA_STR_LEN_Y 1 -#define OLD_FILE_CTRL_OFF_SET_JUMP 2 -#define BS_PATCH_HEADERLEN 8 - -// internal helper functions for BS Patching -int bspatch_readInitialHeaderAndContinue(bspatch_stream* stream); - -int bspatch_allocWorkingBuffers(bspatch_stream* stream); - -int read_controldata(bspatch_stream* stream); -int read_controldata_header(struct bspatch_stream* stream); -int read_controldata_process(struct bspatch_stream* stream); -int sendPatchReadRequest(const bspatch_stream* stream, void* buffer, uint64_t length); -int bspatch_readInitialHeaderAndContinueReadOfPatchDone(bspatch_stream* stream); -int read_deCompressBuffer_process(struct bspatch_stream* stream, int64_t frame_len); -//void process_read_frame_len(struct bspatch_stream* stream); -int bspatch_processSingeExtraStrLenCompress(struct bspatch_stream* stream); -int setExpectedExternalEventByState(struct bspatch_stream* stream, int status, bs_patch_state_t nextState, - bs_patch_api_event_t expectedExternalEvent); - -// api to hide direct function pointer usage, return code will indicate either async completion synch completion or error -int sendWriteNewRequest(const struct bspatch_stream* stream, void* buffer, uint64_t length); -int sendSeekOldRequest(const struct bspatch_stream* stream, int64_t seek_diff); -int sendReadOldRequest(const struct bspatch_stream* stream, void* buffer, uint64_t length); -int bspatch_processDiffBytesPost(struct bspatch_stream* stream); - -int isPatchingDone(bspatch_stream* stream); -uint64_t allignTo8ByteBoundary(uint64_t address); -int readVarIntEventified(struct bspatch_stream* stream, int isSigned); - -#if PATCH_STAT_COUNTING -// note stats counting is now partly broken as varint are not calcualted correctly to header stats -typedef struct { - uint control_header_count; - uint control_header_bytes; - - uint control_data_count; - uint control_data_bytes; - - uint total_diff_str_len_x; - uint total_extra_str_len_y; - - uint max_compressedDataBuffer; - uint max_deCompressBuffer; - - uint frame_header_count; - uint frame_header_bytes; - - uint compressed_frame_count; - uint compressed_frame_bytes; - - uint non_compressed_frame_count; - uint non_compressed_frame_bytes; - - uint initial_header_bytes; - -} -BsPatchStatistics; - -static BsPatchStatistics bsStats = {0}; -#include -#define LOG_STATS printf -void printStats(); -void printStats() { - /* - uint initial_header_bytes; - */ - LOG_STATS("* ********** BS STATS START ***********\n"); - LOG_STATS("* max_compressedDataBuffer: %u max_deCompressBuffer: %u B\n", bsStats.max_compressedDataBuffer, bsStats.max_deCompressBuffer); - LOG_STATS("* initial_header_bytes %u\n", bsStats.initial_header_bytes ); - LOG_STATS("* ctrlHeader_count %u bytes %u B\n", bsStats.control_header_count, bsStats.control_header_bytes); - LOG_STATS("* frame_header_count %u frame_header_bytes %u B\n", bsStats.frame_header_count, bsStats.frame_header_bytes); - - LOG_STATS("* compressed_frame_count %u compressed_frame_bytes %u B\n", bsStats.compressed_frame_count, bsStats.compressed_frame_bytes); - - LOG_STATS("* total_diff_str_len_x %u B\n", bsStats.total_diff_str_len_x); - LOG_STATS("* total_extra_str_len_y %u B\n", bsStats.total_extra_str_len_y); - - uint totalUnCompPayloadBytes = bsStats.total_diff_str_len_x+bsStats.total_extra_str_len_y; - uint compressionEfficiency =(bsStats.compressed_frame_bytes*100)/totalUnCompPayloadBytes; - LOG_STATS("* compressionEfficiency %u %% (smaller is better)\n", compressionEfficiency); - - uint headerBytesSum = bsStats.initial_header_bytes+bsStats.control_header_bytes+bsStats.frame_header_bytes; - uint headerRatioToCompressedPayload = (headerBytesSum*100)/bsStats.compressed_frame_bytes; - LOG_STATS("* headerRatioToCompressedPayload %u %%\n", headerRatioToCompressedPayload); - - LOG_STATS("* ********** BS STATS END ***********\n"); - -} - -#endif - -static int64_t offtin(uint8_t* buf) -{ - int64_t y; - - y = buf[7] & 0x7F; - y = y * 256; - y += buf[6]; - y = y * 256; - y += buf[5]; - y = y * 256; - y += buf[4]; - y = y * 256; - y += buf[3]; - y = y * 256; - y += buf[2]; - y = y * 256; - y += buf[1]; - y = y * 256; - y += buf[0]; - - if (buf[7] & 0x80) - y = -y; - - return y; -} - -/** - * Read deCompressBuffer frame from the patch stream using deCompressBuffer as storage, uncompress and store to compressedDataBuffer. - * @param compressedDataBuffer pre-allocated buffer where undeCompressBuffer frame should be stored - * @param deCompressBuffer pre-allocated buffer where deCompressBuffer frame can be stored - * @param stream - * @param max_compressedDataBuffer length of compressedDataBuffer buffer - * @param max_deCompressBuffer length of deCompressBuffer buffer - * @param plain_done the amount of bytes that was dedeCompressBuffer from frame - * @return 0 if succesfull, BSPATCH error code if failure. - */ -static int read_deCompressBuffer(struct bspatch_stream* stream, int64_t frame_len) -{ - if (frame_len > stream->max_deCompressBuffer) - return EBSAPI_ERR_CORRUPTED_PATCH; - -#if PATCH_STAT_COUNTING - - bsStats.compressed_frame_count+=1; - bsStats.compressed_frame_bytes+=frame_len; -#endif - - return sendPatchReadRequest(stream, stream->bufferForCompressedData, frame_len); -} - -// todo not used to anything yeat. made for non compress support -/* - static int read_unCompressedDataToBuffer(struct bspatch_stream* stream, - int64_t frame_len) { - if (frame_len > stream->max_compressedDataBuffer) - return EBSAPI_ERR_CORRUPTED_PATCH; - - #if PATCH_STAT_COUNTING - - bsStats.non_compressed_frame_count+=1; - bsStats.non_compressed_frame_bytes+=frame_len; - #endif - - return sendPatchReadRequest(stream, stream->nonCompressedDataBuffer, frame_len); - } - */ - -int read_deCompressBuffer_process(struct bspatch_stream* stream, int64_t frame_len) -{ - stream->undeCompressBuffer_len = LZ4_decompress_safe((char*) stream->bufferForCompressedData, - (char*) stream->nonCompressedDataBuffer, frame_len, stream->max_compressedDataBuffer); - - return stream->undeCompressBuffer_len > 0 ? 0 : EBSAPI_ERR_CORRUPTED_PATCH; -} - -int read_controldata_process(struct bspatch_stream* stream) -{ -#if (BS_HIGH_LEVEL_DEBUG) - int32_t headerDataSize = stream->nonCompressedDataBuffer[0]; -#endif - - if (stream->nonCompressedDataBuffer[0] == 24) { - - stream->ctrl[DIFF_STR_LEN_X] = offtin(stream->bufferForCompressedData); - stream->ctrl[EXTRA_STR_LEN_Y] = offtin(stream->bufferForCompressedData + 8); - stream->ctrl[OLD_FILE_CTRL_OFF_SET_JUMP] = offtin(stream->bufferForCompressedData + 16); - } else { - if (LZ4_decompress_safe((char*) stream->bufferForCompressedData, (char*) stream->nonCompressedDataBuffer, - stream->nonCompressedDataBuffer[0], 24) != 24) { - return EBSAPI_ERR_CORRUPTED_PATCH; - } - - stream->ctrl[DIFF_STR_LEN_X] = offtin(stream->nonCompressedDataBuffer); - stream->ctrl[EXTRA_STR_LEN_Y] = offtin(stream->nonCompressedDataBuffer + 8); - stream->ctrl[OLD_FILE_CTRL_OFF_SET_JUMP] = offtin(stream->nonCompressedDataBuffer + 16); - } - -#if (BS_HIGH_LEVEL_DEBUG) - printf("read_controldata_process DIFF_STR_LEN_X %" PRId64 " EXTRA_STR_LEN_Y %" PRId64 " OLD_FILE_CTRL_OFF_SET_JUMP %"PRId64" (headerdatasize: %u)\n", - stream->ctrl[DIFF_STR_LEN_X], - stream->ctrl[EXTRA_STR_LEN_Y], - stream->ctrl[OLD_FILE_CTRL_OFF_SET_JUMP], headerDataSize); -#endif - -#if PATCH_STAT_COUNTING - bsStats.total_diff_str_len_x+=stream->ctrl[DIFF_STR_LEN_X]; - bsStats.total_extra_str_len_y+=stream->ctrl[EXTRA_STR_LEN_Y]; -#endif - - return 0; -} - -uint64_t allignTo8ByteBoundary(uint64_t address) -{ - if (address % 8 != 0) { - address += 8 - address % 8; - } - return address; -} - -int bspatch_allocWorkingBuffers(struct bspatch_stream* stream) -{ - int result = 0; - - if (!stream) { - return EBSAPI_ERR_PARAMETERS; - } - log("bspatch_allocWorkingBuffers max_compressedDataBuffer %u max_deCompressBuffer %u\n", - stream->max_compressedDataBuffer, stream->max_deCompressBuffer ); - -#if PATCH_STAT_COUNTING - bsStats.max_compressedDataBuffer = stream->max_compressedDataBuffer; - bsStats.max_deCompressBuffer = stream->max_deCompressBuffer; -#endif - -#if defined( BS_PATCH_COMPILE_TIME_MEMORY_ALLOC) && (BS_PATCH_COMPILE_TIME_MEMORY_ALLOC > 0) - if (BS_PATCH_COMPILE_TIME_MEMORY_ALLOC < (stream->max_compressedDataBuffer + stream->max_deCompressBuffer)) { - log("BS_PATCH_COMPILE_TIME_MEMORY_ALLOC %u < (%u + %u)\n", BS_PATCH_COMPILE_TIME_MEMORY_ALLOC, stream->max_compressedDataBuffer, stream->max_deCompressBuffer); - return EBSAPI_ERR_OUT_OF_MEMORY; - } - stream->nonCompressedDataBuffer = (uint8_t*) allignTo8ByteBoundary((uint64_t) & (stream->bsMemoryBuffer)); // this should already be alligned - - uint32_t diff1 = (uint64_t)stream->nonCompressedDataBuffer-(uint64_t)stream->bsMemoryBuffer; - - stream->bufferForCompressedData = ((uint8_t*) &(stream->bsMemoryBuffer)) + stream->max_compressedDataBuffer; - - - uint8_t* allignedAdress = (uint8_t*) allignTo8ByteBoundary((uint64_t) stream->bufferForCompressedData); - - uint32_t diff2 = (uint64_t)allignedAdress-(uint64_t)stream->bufferForCompressedData; - stream->bufferForCompressedData = allignedAdress; - log("align extra1 %u align extra2 %u", diff1, diff2); - - if ((stream->bufferForCompressedData + stream->max_deCompressBuffer) - > ((uint8_t*) stream) + sizeof(bspatch_stream)) { - log("not enough memory bufferComp %u decomp %u size %u stream %u allingmentbuffer u%\n", - stream->bufferForCompressedData, stream->max_deCompressBuffer, - sizeof(bspatch_stream), (uint32_t)stream, sizeof(stream->allignmentBuffer)); - return EBSAPI_ERR_OUT_OF_MEMORY; - } - return result; -#else - stream->nonCompressedDataBuffer = malloc(stream->max_compressedDataBuffer); // todo these are likely named wrong way - stream->bufferForCompressedData = malloc(stream->max_deCompressBuffer); - - if (stream->nonCompressedDataBuffer == 0 || stream->bufferForCompressedData == 0) { - result = EBSAPI_ERR_OUT_OF_MEMORY; - } - return result; -#endif // BS_PATCH_COMPILE_TIME_MEMORY_ALLOC -} - -int bspatch_processDiffBytesPost(struct bspatch_stream* stream) -{ - /* Adjust pointers */ - stream->newpos += stream->ctrl[DIFF_STR_LEN_X]; - return 0; -} - -int bspatch_processSingeExtraStrLenCompress(struct bspatch_stream* stream) -{ - int result = 0; - - result = read_deCompressBuffer_process(stream, stream->frame_len); - - if (result) - return result; - - return sendWriteNewRequest(stream, stream->nonCompressedDataBuffer, stream->undeCompressBuffer_len); -} - -// onle there helpers should access function pointers in BS API -int sendPatchReadRequest(const bspatch_stream* stream, void* buffer, uint64_t length) -{ - return stream->read_patch(stream, buffer, length); -} - -int sendSeekOldRequest(const struct bspatch_stream* stream, int64_t seek_diff) -{ - return stream->seek_old(stream, seek_diff); -} - -int sendReadOldRequest(const struct bspatch_stream* stream, void* buffer, uint64_t length) -{ - return stream->read_old(stream, buffer, length); -} - -int sendWriteNewRequest(const struct bspatch_stream* stream, void* buffer, uint64_t length) -{ - return stream->write_new(stream, buffer, length); -} - -int bspatch_readInitialHeaderAndContinue(bspatch_stream* stream) -{ - if (stream->read_old == 0 || stream->read_patch == 0 || stream->seek_old == 0 || stream->write_new == 0) - return EBSAPI_ERR_PARAMETERS; - - if (stream->new_size == 0) { - /* First call, read header */ -#if PATCH_STAT_COUNTING - bsStats.initial_header_bytes+=FILE_HEADER_LEN; -#endif - - return sendPatchReadRequest(stream, stream->header, FILE_HEADER_LEN); - } else { - return EBSAPI_ERR_ALREADY_INIT; - } -} - -int bspatch_readInitialHeaderAndContinueReadOfPatchDone(bspatch_stream* stream) -{ - /* Check for appropriate magic */ - if (memcmp(stream->header, FILE_MAGIC, FILE_MAGIC_LEN) != 0) { - return EBSAPI_ERR_CORRUPTED_PATCH; - } - - /* Read new file length from header */ - int64_t newSize64 = offtin(stream->header + FILE_MAGIC_LEN); - if (newSize64 < 0 || newSize64 > UINT32_MAX) { - return EBSAPI_ERR_CORRUPTED_PATCH; - } - stream->new_size = (uint32_t) newSize64; - - /* Read max undeCompressBuffer frame size from header */ - int64_t max_compressDataBuffer64 = offtin(stream->header + FILE_MAGIC_LEN + 8); - if (max_compressDataBuffer64 < 0 || max_compressDataBuffer64 > UINT32_MAX) { - return EBSAPI_ERR_CORRUPTED_PATCH; - } - stream->max_compressedDataBuffer = (uint32_t) max_compressDataBuffer64; - - /* Read max deCompressBuffer frame size from header */ - int64_t max_deCompressBuffer64 = offtin(stream->header + FILE_MAGIC_LEN + 8); - - if (max_deCompressBuffer64 < 0 || max_deCompressBuffer64 > UINT32_MAX) { - return EBSAPI_ERR_CORRUPTED_PATCH; - } - stream->max_deCompressBuffer = (uint32_t) max_deCompressBuffer64; - - /* Sanity checks */ - if (stream->max_deCompressBuffer > stream->max_compressedDataBuffer || stream->max_compressedDataBuffer < 64 - || stream->max_deCompressBuffer < 1) { - return EBSAPI_ERR_CORRUPTED_PATCH; - } - return 0; -} - -int setExpectedExternalEventByState(struct bspatch_stream* stream, int status, bs_patch_state_t nextState, - bs_patch_api_event_t expectedExternalEvent) -{ - if (status < 0) { - return 1; - } - - switch (status) { - case EBSAPI_OPERATION_DONE_IMMEDIATELY: - stream->expectedExternalEvent = 0; - stream->next_state = nextState; - return 0; - case EBSAPI_OPERATION_PATCH_READ_WILL_COMPLETE_LATER: - case EBSAPI_OPERATION_OLD_FILE_READ_WILL_COMPLETE_LATER: - case EBSAPI_OPERATION_OLD_FILE_SEEK_WILL_COMPLETE_LATER: - case EBSAPI_OPERATION_NEW_FILE_WRITE_WILL_COMPLETE_LATER: - stream->expectedExternalEvent = expectedExternalEvent; - stream->next_state = nextState; - return 1; - default: - assert(0); - return 1; - } - return 1; -} - -int isPatchingDone(bspatch_stream* stream) -{ - if (stream->new_size != 0 && stream->newpos >= stream->new_size) - return 1; - else - return 0; -} - -/// API FUNCTIONS VISIBLE TO OUTSIDE INTERFACE -void ARM_BS_Init(bspatch_stream* stream, void* opaque, read_patch_f rpf, read_old_f rof, seek_old_f sof, - write_new_f wnf) -{ - assert(stream && rpf && rof && sof && wnf); - memset(stream, 0, sizeof(struct bspatch_stream)); - stream->opaque = opaque; - stream->read_patch = rpf; - stream->read_old = rof; - stream->seek_old = sof; - stream->write_new = wnf; - stream->expectedExternalEvent = EBSAPI_START_PATCH_PROCESSING; - stream->next_state = EBsInitial; - stream->frame_len = 0; -} - -/** - * Main state machine functionality for eventified BS patching. Currently all state transitions are visible inside this - * function with below helper macros - */ -#define WAIT_FOR_PATCH_DATA(next_state) \ -assert("wrongAssumedStatus" && (status== 0 || status == EBSAPI_OPERATION_PATCH_READ_WILL_COMPLETE_LATER) || status < 0); \ -if(setExpectedExternalEventByState(stream, status, next_state, EBSAPI_READ_PATCH_DONE)) {SET_NEXT_STATE_NOCHECK(next_state);} else {/*internal state transition assumed*/;} - -#define WAIT_FOR_SEEK_OLD(next_state) \ -assert("wrongAssumedStatus" && (status== 0 || status == EBSAPI_OPERATION_OLD_FILE_SEEK_WILL_COMPLETE_LATER) || status < 0); \ -if (setExpectedExternalEventByState(stream, status, next_state, EBSAPI_SEEK_OLD_DONE)) {SET_NEXT_STATE_NOCHECK(next_state);} else {/*internal state transition assumed*/;} - -#define WAIT_FOR_READ_OLD(next_state) \ -assert("wrongAssumedStatus" && (status== 0 || status == EBSAPI_OPERATION_OLD_FILE_READ_WILL_COMPLETE_LATER) || status < 0); \ -if (setExpectedExternalEventByState(stream, status, next_state, EBSAPI_READ_OLD_DONE)) {SET_NEXT_STATE_NOCHECK(next_state);} else {/*internal state transition assumed*/;} - -#define WAIT_FOR_WRITE_NEW(next_state) \ -assert("wrongAssumedStatus" && (status== 0 || status == EBSAPI_OPERATION_NEW_FILE_WRITE_WILL_COMPLETE_LATER) || status < 0); \ -if (setExpectedExternalEventByState(stream, status, next_state, EBSAPI_WRITE_NEW_DONE)) {SET_NEXT_STATE_NOCHECK(next_state);} else {/*internal state transition assumed*/;} - -#define SET_NEXT_STATE_NOCHECK(new_state) stream->next_state=new_state; - -#define SET_NEXT_STATE(new_state) if(status!=0){log ("invalid status %d\n", status);}assert("error status in setting next state" && status==0); SET_NEXT_STATE_NOCHECK(new_state); - -bs_patch_api_return_code_t ARM_BS_ProcessPatchEvent(bspatch_stream* stream, bs_patch_api_event_t bsApiEvent) -{ - if (bsApiEvent != stream->expectedExternalEvent) { - return EBSAPI_ERR_UNEXPECTED_EVENT; - } else { - stream->expectedExternalEvent = 0; // received expected external event - } - int status = 0; - do { - status = 0; - //log("bs:state:%d\n", stream->next_state); - switch (stream->next_state) { - case EBsInitial: - TRACE_PATCH_CONTROL_READ("EBsInitial\n"); - status = bspatch_readInitialHeaderAndContinue(stream); - WAIT_FOR_PATCH_DATA(EBsInitialPatchReadDone) - break; - case EBsInitialPatchReadDone: - status = bspatch_readInitialHeaderAndContinueReadOfPatchDone(stream); - SET_NEXT_STATE(EBsAllocWorkingBuffer) - // no patch data reading wait - break; - case EBsAllocWorkingBuffer: - status = bspatch_allocWorkingBuffers(stream); - /* next process patch and write to new file */ - SET_NEXT_STATE(EBsReadControlSegmentHeader) - break; - case EBsReadControlSegmentHeader: - TRACE_PATCH_CONTROL_READ("EBsReadControlSegmentHeader\n"); - SET_NEXT_STATE(EBsReadControlSegment) - break; - case EBsReadControlSegment: - TRACE_PATCH_CONTROL_READ("EBsReadControlSegment\n"); - SET_NEXT_STATE(EReadCtrl_diff_str_len) - break; - case EReadCtrl_diff_str_len: - stream->stateAfterReadVarInt = EReadCtrl_extra_str_len_y; - - stream->isSignedVarInt = 0; - status = sendPatchReadRequest(stream, stream->nonCompressedDataBuffer, 1); - WAIT_FOR_PATCH_DATA(EBsPatch_process_varintPiece) - break; - case EReadCtrl_extra_str_len_y: - stream->ctrl[DIFF_STR_LEN_X] = stream->var_int; - - stream->stateAfterReadVarInt = EReadCtrl_old_file_ctrl_off_set_jump; - stream->isSignedVarInt = 0; - status = sendPatchReadRequest(stream, stream->nonCompressedDataBuffer, 1); - WAIT_FOR_PATCH_DATA(EBsPatch_process_varintPiece) - break; - case EReadCtrl_old_file_ctrl_off_set_jump: - stream->ctrl[EXTRA_STR_LEN_Y] = stream->var_int; - - stream->stateAfterReadVarInt = EBsReadDataSegment; - stream->isSignedVarInt = 1; - status = sendPatchReadRequest(stream, stream->nonCompressedDataBuffer, 1); - WAIT_FOR_PATCH_DATA(EBsPatch_process_varintPiece) - break; - case EBsReadDataSegment: - stream->ctrl[OLD_FILE_CTRL_OFF_SET_JUMP] = stream->var_int; - // Sanity-check - if (stream->newpos + stream->ctrl[DIFF_STR_LEN_X] > stream->new_size) { - status = EBSAPI_ERR_CORRUPTED_PATCH; - } - - log("DIFF_STR_LEN_X:%ld\n", stream->ctrl[DIFF_STR_LEN_X]); - log("EXTRA_STR_LEN_Y: %ld\n", stream->ctrl[EXTRA_STR_LEN_Y]); - log("OLD_FILE_CTRL_OFF_SET_JUMP: %ld\n", stream->ctrl[OLD_FILE_CTRL_OFF_SET_JUMP]); - - if (stream->ctrl[DIFF_STR_LEN_X] > 0) { - stream->total_undeCompressBuffer = 0; - SET_NEXT_STATE(EBspatch_processDiffBytes_readHeaderPiece); - } else { - SET_NEXT_STATE(EBsProcessExtraLen); - } - break; - case EBspatch_processDiffBytes_readHeaderPiece: - TRACE_PATCH_CONTROL_READ("EBspatch_processDiffBytes_readHeaderPiece\n"); - stream->stateAfterReadVarInt = EBspatch_processDiffBytes_processSinglePiece; - SET_NEXT_STATE(EBspatch_read_frame_len) - ; - break; - case EBspatch_read_frame_len: // read frame header (encoded with varint so might need multiple rounds) - stream->isSignedVarInt = 0; - status = sendPatchReadRequest(stream, stream->nonCompressedDataBuffer, 1); - WAIT_FOR_PATCH_DATA(EBsPatch_process_varintPiece) - break; - case EBspatch_read_varintPiece: - status = sendPatchReadRequest(stream, stream->nonCompressedDataBuffer, 1); - WAIT_FOR_PATCH_DATA(EBsPatch_process_varintPiece) - break; - case EBsPatch_process_varintPiece: - status = readVarIntEventified(stream, stream->isSignedVarInt); - break; - case EBspatch_processDiffBytes_processSinglePiece: - TRACE_PATCH_CONTROL_READ("EBspatch_processDiffBytes_processSinglePiece\n"); - stream->frame_len = stream->var_int; - status = read_deCompressBuffer(stream, stream->frame_len); - WAIT_FOR_PATCH_DATA(EBspatch_processDiffBytes_processSinglePieceContinue) - break; - case EBspatch_processDiffBytes_processSinglePieceContinue: - status = read_deCompressBuffer_process(stream, stream->frame_len); - log("extractedSize: %u\n", stream->undeCompressBuffer_len); - SET_NEXT_STATE(EBspatch_processDiffBytes_processSinglePieceInit) - break; - case EBspatch_processDiffBytes_processSinglePieceInit: - stream->i = 0; - SET_NEXT_STATE(EBspatch_processDiffBytes_processSinglePieceContinue2) - break; - case EBspatch_processDiffBytes_processSinglePieceContinue2: - if (stream->i < stream->undeCompressBuffer_len) { - stream->readRequestSize = 0; - uint32_t dataLeft = stream->undeCompressBuffer_len - stream->i; - if (stream->max_deCompressBuffer <= dataLeft) { - stream->readRequestSize = stream->max_deCompressBuffer; - } else { - stream->readRequestSize = dataLeft; - } - status = sendReadOldRequest(stream, stream->bufferForCompressedData, /*1*/stream->readRequestSize); - WAIT_FOR_READ_OLD(EBspatch_processDiffBytes_processSinglePieceContinue_writePart); // recursive event to loopify - } else { - status = 0; - SET_NEXT_STATE(EBspatch_processDiffBytes_processSinglePieceContinue_postActions); - } - break; - case EBspatch_processDiffBytes_processSinglePieceContinue_writePart: - - for (uint32_t i = 0; i < stream->readRequestSize; i++) { - uint8_t newByte = stream->bufferForCompressedData[i] - + stream->nonCompressedDataBuffer[stream->i + i]; - stream->bufferForCompressedData[i] = newByte; - } - status = sendWriteNewRequest(stream, stream->bufferForCompressedData, stream->readRequestSize); - stream->i += stream->readRequestSize; - WAIT_FOR_WRITE_NEW(EBspatch_processDiffBytes_processSinglePieceContinue2) - break; - case EBspatch_processDiffBytes_processSinglePieceContinue_postActions: - stream->total_undeCompressBuffer += stream->undeCompressBuffer_len; - status = 0; - if (stream->total_undeCompressBuffer < stream->ctrl[DIFF_STR_LEN_X]) { - SET_NEXT_STATE(EBspatch_processDiffBytes_readHeaderPiece); - } else { - status = bspatch_processDiffBytesPost(stream); - SET_NEXT_STATE(EBsProcessExtraLen); - } - break; - case EBsProcessExtraLen: - // Sanity-check - status = 0; - if (stream->newpos + stream->ctrl[EXTRA_STR_LEN_Y] > stream->new_size) { - status = EBSAPI_ERR_CORRUPTED_PATCH; - break; - } - - // Read and write extra string - if (stream->ctrl[EXTRA_STR_LEN_Y] > 0) { - stream->total_undeCompressBuffer = 0; - - SET_NEXT_STATE(EBsProcessExtraLen_readHeader); - } else { - SET_NEXT_STATE(EBsProcessExtraLen_postStep); - } - break; - case EBsProcessExtraLen_readHeader: - TRACE_PATCH_CONTROL_READ("EBsProcessExtraLen_readHeader\n"); - stream->stateAfterReadVarInt = EBsProcessExtraLen_singleItem; - SET_NEXT_STATE(EBspatch_read_frame_len) - break; - case EBsProcessExtraLen_singleItem: - stream->frame_len = stream->var_int; - TRACE_PATCH_CONTROL_READ("EBsProcessExtraLen_singleItem frame len %llu\n", stream->frame_len); - status = read_deCompressBuffer(stream, stream->frame_len); - WAIT_FOR_PATCH_DATA(EBsProcessExtraLen_singleItemContinue) - break; - case EBsProcessExtraLen_singleItemContinue: - status = bspatch_processSingeExtraStrLenCompress(stream); - WAIT_FOR_WRITE_NEW(EBsProcessExtraLen_singleItemContinuePostStep) - break; - case EBsProcessExtraLen_singleItemContinuePostStep: - stream->total_undeCompressBuffer += stream->undeCompressBuffer_len; - - if (stream->total_undeCompressBuffer < stream->ctrl[EXTRA_STR_LEN_Y]) { - SET_NEXT_STATE(EBsProcessExtraLen_readHeader); - } else { - SET_NEXT_STATE(EBsProcessExtraLen_postStep); - } - break; - case EBsProcessExtraLen_postStep: - // Adjust pointers - stream->newpos += stream->ctrl[EXTRA_STR_LEN_Y]; - status = sendSeekOldRequest(stream, stream->ctrl[OLD_FILE_CTRL_OFF_SET_JUMP]); - WAIT_FOR_SEEK_OLD(EBsReadControlSegmentHeader) - break; - - default: - assert("unknown state" && 0); - status = EBSAPI_ERR_INVALID_STATE; - break; - } - } while ((!isPatchingDone(stream) && status == 0)); - - if (isPatchingDone(stream)) { -#if PATCH_STAT_COUNTING - printStats(); -#endif - - return EBSAPI_PATCH_DONE; - } else { - return status; - } -} - -int readVarIntEventified(struct bspatch_stream* stream, int isSigned) -{ - int status = 0; - if (isSigned) { - status = decode_signed_varint(*(stream->nonCompressedDataBuffer), (int64_t*) &stream->var_int, - stream->var_int_len); - } else { - status = decode_unsigned_varint(*(stream->nonCompressedDataBuffer), &stream->var_int, stream->var_int_len); - } - stream->var_int_len++; - - if (stream->var_int_len >= 8 || status < 0) { - status = EBSAPI_ERR_CORRUPTED_PATCH; - } - - if (status == OPERATION_NEEDS_MORE_DATA) { - status = 0; - SET_NEXT_STATE(EBspatch_read_varintPiece); - } else { - stream->var_int_len = 0; - assert(stream->stateAfterReadVarInt); - SET_NEXT_STATE(stream->stateAfterReadVarInt); // next state depends on state that started frame len reading - stream->stateAfterReadVarInt = 0; - } - return status; -} - -void* ARM_BS_GetOpaque(const struct bspatch_stream* stream) -{ - return stream->opaque; -} - -int ARM_BS_Free(struct bspatch_stream* stream) -{ -#if defined( BS_PATCH_COMPILE_TIME_MEMORY_ALLOC) && (BS_PATCH_COMPILE_TIME_MEMORY_ALLOC > 0) - // nothing to free really but we can lose the pointers at least - stream->nonCompressedDataBuffer = 0; - stream->bufferForCompressedData = 0; -#else - - free(stream->nonCompressedDataBuffer); - free(stream->bufferForCompressedData); - stream->nonCompressedDataBuffer = 0; - stream->bufferForCompressedData = 0; -#endif - - return 0; -} diff --git a/delta-tool/source/lz4.c b/delta-tool/source/lz4.c deleted file mode 100755 index a299a72..0000000 --- a/delta-tool/source/lz4.c +++ /dev/null @@ -1,1972 +0,0 @@ -/* - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ - - -/*-************************************ -* Tuning parameters -**************************************/ -/* - * LZ4_HEAPMODE : - * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). - */ -#ifndef LZ4_HEAPMODE -# define LZ4_HEAPMODE 0 -#endif - -/* - * ACCELERATION_DEFAULT : - * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 - */ -#define ACCELERATION_DEFAULT 1 - - -/*-************************************ -* CPU Feature Detection -**************************************/ -/* LZ4_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which assembly generation depends on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ -# if defined(__GNUC__) && \ - ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ - || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define LZ4_FORCE_MEMORY_ACCESS 2 -# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) -# define LZ4_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/* - * LZ4_FORCE_SW_BITCOUNT - * Define this parameter if your target system or compiler does not support hardware bit count - */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - - -/*-************************************ -* Dependency -**************************************/ -#define LZ4_STATIC_LINKING_ONLY -#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ -#include "lz4.h" -/* see also "memory routines" below */ - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ -#endif /* _MSC_VER */ - -#ifndef LZ4_FORCE_INLINE -# ifdef _MSC_VER /* Visual Studio */ -# define LZ4_FORCE_INLINE static __forceinline -# else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define LZ4_FORCE_INLINE static inline -# endif -# else -# define LZ4_FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -# endif /* _MSC_VER */ -#endif /* LZ4_FORCE_INLINE */ - -/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE - * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy, - * together with a simple 8-byte copy loop as a fall-back path. - * However, this optimization hurts the decompression speed by >30%, - * because the execution does not go to the optimized loop - * for typical compressible data, and all of the preamble checks - * before going to the fall-back path become useless overhead. - * This optimization happens only with the -O3 flag, and -O2 generates - * a simple 8-byte copy loop. - * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy - * functions are annotated with __attribute__((optimize("O2"))), - * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute - * of LZ4_wildCopy does not affect the compression speed. - */ -#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) -# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE -#else -# define LZ4_FORCE_O2_GCC_PPC64LE -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static -#endif - -#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#ifndef likely -#define likely(expr) expect((expr) != 0, 1) -#endif -#ifndef unlikely -#define unlikely(expr) expect((expr) != 0, 0) -#endif - - -/*-************************************ -* Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#define ALLOC(s) malloc(s) -#define ALLOC_AND_ZERO(s) calloc(1,s) -#define FREEMEM(p) free(p) -#include /* memset, memcpy */ -#define MEM_INIT(p,v,s) memset((p),(v),(s)) - - -/*-************************************ -* Basic Types -**************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef uintptr_t uptrval; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef size_t uptrval; /* generally true, except OpenVMS-64 */ -#endif - -#if defined(__x86_64__) - typedef U64 reg_t; /* 64-bits in x32 mode */ -#else - typedef size_t reg_t; /* 32-bits in x32 mode */ -#endif - -/*-************************************ -* Reading and writing into memory -**************************************/ -static unsigned LZ4_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - - -#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) -/* lie to the compiler about data alignment; use with caution */ - -static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } -static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } -static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } - -static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } - -#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; - -static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } - -static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } - -#else /* safe and portable access through memcpy() */ - -static U16 LZ4_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static U32 LZ4_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static reg_t LZ4_read_ARCH(const void* memPtr) -{ - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static void LZ4_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -static void LZ4_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* LZ4_FORCE_MEMORY_ACCESS */ - - -static U16 LZ4_readLE16(const void* memPtr) -{ - if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); - } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); - } -} - -static void LZ4_writeLE16(void* memPtr, U16 value) -{ - if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - } -} - -/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE -void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) -{ - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; - - do { memcpy(d,s,8); d+=8; s+=8; } while (d=1) -# include -#else -# ifndef assert -# define assert(condition) ((void)0) -# endif -#endif - -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ - -#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) -# include -static int g_debuglog_enable = 1; -# define DEBUGLOG(l, ...) { \ - if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif - - -/*-************************************ -* Common functions -**************************************/ -static unsigned LZ4_NbCommonBytes (reg_t val) -{ - if (LZ4_isLittleEndian()) { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, - 0, 3, 1, 3, 1, 4, 2, 7, - 0, 2, 3, 6, 1, 5, 3, 5, - 1, 3, 4, 4, 2, 5, 6, 7, - 7, 0, 1, 2, 3, 3, 4, 6, - 2, 6, 5, 5, 3, 4, 5, 6, - 7, 1, 2, 4, 6, 4, 4, 5, - 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, - 3, 2, 2, 1, 3, 2, 0, 1, - 3, 3, 1, 2, 2, 2, 2, 0, - 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else /* Big Endian CPU */ { - if (sizeof(val)==8) { /* 64-bits */ -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. - Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. - Note that this code path is never triggered in 32-bits mode. */ - unsigned r; - if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } - } -} - -#define STEPSIZE sizeof(reg_t) -LZ4_FORCE_INLINE -unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) -{ - const BYTE* const pStart = pIn; - - if (likely(pIn < pInLimit-(STEPSIZE-1))) { - reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); - if (!diff) { - pIn+=STEPSIZE; pMatch+=STEPSIZE; - } else { - return LZ4_NbCommonBytes(diff); - } } - - while (likely(pIn < pInLimit-(STEPSIZE-1))) { - reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); - if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } - pIn += LZ4_NbCommonBytes(diff); - return (unsigned)(pIn - pStart); - } - - if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } - if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } - if ((pIn compression run slower on incompressible data */ - - -/*-************************************ -* Local Structures and types -**************************************/ -typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive; -typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; - -/** - * This enum distinguishes several different modes of accessing previous - * content in the stream. - * - * - noDict : There is no preceding content. - * - withPrefix64k : Table entries up to ctx->dictSize before the current blob - * blob being deCompressBuffer are valid and refer to the preceding - * content (of length ctx->dictSize), which is available - * contiguously preceding in memory the content currently - * being deCompressBuffer. - * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere - * else in memory, starting at ctx->dictionary with length - * ctx->dictSize. - * - usingDictCtx : Like usingExtDict, but everything concerning the preceding - * content is in a separate context, pointed to by - * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table - * entries in the current context that refer to positions - * preceding the beginning of the current compression are - * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx - * ->dictSize describe the location and size of the preceding - * content, and matches are found by looking in the ctx - * ->dictCtx->hashTable. - */ -typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; -typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; - - -/*-************************************ -* Local Utils -**************************************/ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - - -/*-************************************ -* Internal Definitions used in Tests -**************************************/ -#if defined (__cplusplus) -extern "C" { -#endif - -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); - -int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize); - -#if defined (__cplusplus) -} -#endif - -/*-****************************** -* Compression functions -********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) -{ - if (tableType == byU16) - return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); - else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); -} - -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) -{ - const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) { - const U64 prime5bytes = 889523592379ULL; - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - } else { - const U64 prime8bytes = 11400714785074694791ULL; - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); - } -} - -LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) -{ - if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); - return LZ4_hash4(LZ4_read32(p), tableType); -} - -static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) -{ - switch (tableType) - { - default: /* fallthrough */ - case clearedTable: /* fallthrough */ - case byPtr: { /* illegal! */ assert(0); return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } - case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } - } -} - -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, - void* tableBase, tableType_t const tableType, - const BYTE* srcBase) -{ - switch (tableType) - { - case clearedTable: { /* illegal! */ assert(0); return; } - case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } - } -} - -LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); -} - -/* LZ4_getIndexOnHash() : - * Index of match position registered in hash table. - * hash position must be calculated by using base+index, or dictBase+index. - * Assumption 1 : only valid if tableType == byU32 or byU16. - * Assumption 2 : h is presumed valid (within limits of hash table) - */ -static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) -{ - LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); - if (tableType == byU32) { - const U32* const hashTable = (const U32*) tableBase; - assert(h < (1U << (LZ4_MEMORY_USAGE-2))); - return hashTable[h]; - } - if (tableType == byU16) { - const U16* const hashTable = (const U16*) tableBase; - assert(h < (1U << (LZ4_MEMORY_USAGE-1))); - return hashTable[h]; - } - assert(0); return 0; /* forbidden case */ -} - -static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ -} - -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, - const void* tableBase, tableType_t tableType, - const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); -} - -LZ4_FORCE_INLINE void LZ4_prepareTable( - LZ4_stream_t_internal* const cctx, - const int inputSize, - const tableType_t tableType) { - /* If the table hasn't been used, it's guaranteed to be zeroed out, and is - * therefore safe to use no matter what mode we're in. Otherwise, we figure - * out if it's safe to leave as is or whether it needs to be reset. - */ - if (cctx->tableType != clearedTable) { - if (cctx->tableType != tableType - || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU) - || (tableType == byU32 && cctx->currentOffset > 1 GB) - || tableType == byPtr - || inputSize >= 4 KB) - { - DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); - MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); - cctx->currentOffset = 0; - cctx->tableType = clearedTable; - } else { - DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); - } - } - - /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster - * than compressing without a gap. However, compressing with - * currentOffset == 0 is faster still, so we preserve that case. - */ - if (cctx->currentOffset != 0 && tableType == byU32) { - DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); - cctx->currentOffset += 64 KB; - } - - /* Finally, clear history */ - cctx->dictCtx = NULL; - cctx->dictionary = NULL; - cctx->dictSize = 0; -} - -/** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ -LZ4_FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, - int *inputConsumed, /* only written when outputLimited == fillOutput */ - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dictDirective, - const dictIssue_directive dictIssue, - const U32 acceleration) -{ - const BYTE* ip = (const BYTE*) source; - - U32 const startIndex = cctx->currentOffset; - const BYTE* base = (const BYTE*) source - startIndex; - const BYTE* lowLimit; - - const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; - const BYTE* const dictionary = - dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; - const U32 dictSize = - dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; - const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ - - int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); - U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ - const BYTE* const dictEnd = dictionary + dictSize; - const BYTE* anchor = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; - const BYTE* const matchlimit = iend - LASTLITERALS; - - /* the dictCtx currentOffset is indexed on the start of the dictionary, - * while a dictionary in the current context precedes the currentOffset */ - const BYTE* dictBase = (dictDirective == usingDictCtx) ? - dictionary + dictSize - dictCtx->currentOffset : - dictionary + dictSize - startIndex; - - BYTE* op = (BYTE*) dest; - BYTE* const olimit = op + maxOutputSize; - - U32 offset = 0; - U32 forwardH; - - DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); - /* Init conditions */ - if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ - assert(acceleration >= 1); - - lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); - - /* Update context state */ - if (dictDirective == usingDictCtx) { - /* Subsequent linked blocks can't use the dictionary. */ - /* Instead, they use the block we just deCompressBuffer. */ - cctx->dictCtx = NULL; - cctx->dictSize = (U32)inputSize; - } else { - cctx->dictSize += (U32)inputSize; - } - cctx->currentOffset += (U32)inputSize; - cctx->tableType = (U16)tableType; - - if (inputSizehashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - if (tableType == byPtr) { - const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; - assert(ip < mflimitPlusOne); - - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - - } while ( (match+MAX_DISTANCE < ip) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - - } else { /* byU32, byU16 */ - - const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - U32 const current = (U32)(forwardIp - base); - U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); - assert(matchIndex <= current); - assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; - assert(ip < mflimitPlusOne); - - if (dictDirective == usingDictCtx) { - if (matchIndex < startIndex) { - /* there was no match, try the dictionary */ - assert(tableType == byU32); - matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); - match = dictBase + matchIndex; - matchIndex += dictDelta; /* make dictCtx index comparable with current context */ - lowLimit = dictionary; - } else { - match = base + matchIndex; - lowLimit = (const BYTE*)source; - } - } else if (dictDirective==usingExtDict) { - if (matchIndex < startIndex) { - DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); - assert(startIndex - matchIndex >= MINMATCH); - match = dictBase + matchIndex; - lowLimit = dictionary; - } else { - match = base + matchIndex; - lowLimit = (const BYTE*)source; - } - } else { /* single continuous memory segment */ - match = base + matchIndex; - } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); - - if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ - assert(matchIndex < current); - if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */ - if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */ - - if (LZ4_read32(match) == LZ4_read32(ip)) { - if (maybe_extMem) offset = current - matchIndex; - break; /* match found */ - } - - } while(1); - } - - /* Catch up */ - while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literals */ - { unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited == limitedOutput) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if ((outputLimited == fillOutput) && - (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { - op--; - goto _last_literals; - } - if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; - *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< olimit)) { - /* the match was too close to the end, rewind and go to last literals */ - op = token; - goto _last_literals; - } - - /* Encode Offset */ - if (maybe_extMem) { /* static test */ - DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); - assert(offset <= MAX_DISTANCE && offset > 0); - LZ4_writeLE16(op, (U16)offset); op+=2; - } else { - DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); - assert(ip-match <= MAX_DISTANCE); - LZ4_writeLE16(op, (U16)(ip - match)); op+=2; - } - - /* Encode MatchLength */ - { unsigned matchCode; - - if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) - && (lowLimit==dictionary) /* match within extDict */ ) { - const BYTE* limit = ip + (dictEnd-match); - assert(dictEnd > match); - if (limit > matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip==limit) { - unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; - } - DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); - } else { - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); - } - - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) { - if (outputLimited == limitedOutput) - return 0; - if (outputLimited == fillOutput) { - /* Match description too long : reduce it */ - U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255; - ip -= matchCode - newMatchCode; - matchCode = newMatchCode; - } - } - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) { - op+=4; - LZ4_write32(op, 0xFFFFFFFF); - matchCode -= 4*255; - } - op += matchCode / 255; - *op++ = (BYTE)(matchCode % 255); - } else - *token += (BYTE)(matchCode); - } - - anchor = ip; - - /* Test end of chunk */ - if (ip >= mflimitPlusOne) break; - - /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); - - /* Test next position */ - if (tableType == byPtr) { - - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE >= ip) - && (LZ4_read32(match) == LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - } else { /* byU32, byU16 */ - - U32 const h = LZ4_hashPosition(ip, tableType); - U32 const current = (U32)(ip-base); - U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); - assert(matchIndex < current); - if (dictDirective == usingDictCtx) { - if (matchIndex < startIndex) { - /* there was no match, try the dictionary */ - matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); - match = dictBase + matchIndex; - lowLimit = dictionary; /* required for match length counter */ - matchIndex += dictDelta; - } else { - match = base + matchIndex; - lowLimit = (const BYTE*)source; /* required for match length counter */ - } - } else if (dictDirective==usingExtDict) { - if (matchIndex < startIndex) { - match = dictBase + matchIndex; - lowLimit = dictionary; /* required for match length counter */ - } else { - match = base + matchIndex; - lowLimit = (const BYTE*)source; /* required for match length counter */ - } - } else { /* single memory segment */ - match = base + matchIndex; - } - LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); - assert(matchIndex < current); - if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) - && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current)) - && (LZ4_read32(match) == LZ4_read32(ip)) ) { - token=op++; - *token=0; - if (maybe_extMem) offset = current - matchIndex; - DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", - (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); - goto _next_match; - } - } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - - } - -_last_literals: - /* Encode Last Literals */ - { size_t lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { - if (outputLimited == fillOutput) { - /* adapt lastRun to fill 'dst' */ - lastRun = (olimit-op) - 1; - lastRun -= (lastRun+240)/255; - } - if (outputLimited == limitedOutput) - return 0; - } - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRun<internal_donotuse; - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - LZ4_resetStream((LZ4_stream_t*)state); - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) { - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); - } - } else { - if (inputSize < LZ4_64Klimit) {; - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); - } - } -} - -/** - * LZ4_compress_fast_extState_fastReset() : - * A variant of LZ4_compress_fast_extState(). - * - * Using this variant avoids an expensive initialization step. It is only safe - * to call if the state buffer is known to be correctly initialized already - * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of - * "correctly initialized"). - */ -int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - if (dstCapacity >= LZ4_compressBound(srcSize)) { - if (srcSize < LZ4_64Klimit) { - const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, srcSize, tableType); - if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); - } else { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); - } - } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; - LZ4_prepareTable(ctx, srcSize, tableType); - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); - } - } else { - if (srcSize < LZ4_64Klimit) { - const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, srcSize, tableType); - if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); - } else { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); - } - } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; - LZ4_prepareTable(ctx, srcSize, tableType); - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); - } - } -} - - -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - int result; -#if (LZ4_HEAPMODE) - LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ - if (ctxPtr == NULL) return 0; -#else - LZ4_stream_t ctx; - LZ4_stream_t* const ctxPtr = &ctx; -#endif - result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); - -#if (LZ4_HEAPMODE) - FREEMEM(ctxPtr); -#endif - return result; -} - - -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); -} - - -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} - - -/* Note!: This function leaves the stream in an unclean/broken state! - * It is not safe to subsequently use the same state with a _fastReset() or - * _continue() call without resetting it. */ -static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) -{ - LZ4_resetStream(state); - - if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); - } else { - if (*srcSizePtr < LZ4_64Klimit) { - return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); - } else { - tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; - return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, tableType, noDict, noDictIssue, 1); - } } -} - - -int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) -{ -#if (LZ4_HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ - if (ctx == NULL) return 0; -#else - LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; -#endif - - int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); - -#if (LZ4_HEAPMODE) - FREEMEM(ctx); -#endif - return result; -} - - - -/*-****************************** -* Streaming functions -********************************/ - -LZ4_stream_t* LZ4_createStream(void) -{ - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - DEBUGLOG(4, "LZ4_createStream %p", lz4s); - if (lz4s == NULL) return NULL; - LZ4_resetStream(lz4s); - return lz4s; -} - -void LZ4_resetStream (LZ4_stream_t* LZ4_stream) -{ - DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); -} - -void LZ4_resetStream_fast(LZ4_stream_t* ctx) { - LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); -} - -int LZ4_freeStream (LZ4_stream_t* LZ4_stream) -{ - if (!LZ4_stream) return 0; /* support free on NULL */ - DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); - FREEMEM(LZ4_stream); - return (0); -} - - -#define HASH_UNIT sizeof(reg_t) -int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) -{ - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; - const tableType_t tableType = byU32; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); - - /* It's necessary to reset the context, - * and not just continue it with prepareTable() - * to avoid any risk of generating overflowing matchIndex - * when compressing using this dictionary */ - LZ4_resetStream(LZ4_dict); - - /* We always increment the offset by 64 KB, since, if the dict is longer, - * we truncate it to the last 64k, and if it's shorter, we still want to - * advance by a whole window length so we can provide the guarantee that - * there are only valid offsets in the window, which allows an optimization - * in LZ4_compress_fast_continue() where it uses noDictIssue even when the - * dictionary isn't a full 64k. */ - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - base = dictEnd - 64 KB - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += 64 KB; - dict->tableType = tableType; - - if (dictSize < (int)HASH_UNIT) { - return 0; - } - - while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, tableType, base); - p+=3; - } - - return dict->dictSize; -} - -void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { - if (dictionary_stream != NULL) { - /* If the current offset is zero, we will never look in the - * external dictionary context, since there is no value a table - * entry can take that indicate a miss. In that case, we need - * to bump the offset to something non-zero. - */ - if (working_stream->internal_donotuse.currentOffset == 0) { - working_stream->internal_donotuse.currentOffset = 64 KB; - } - working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse); - } else { - working_stream->internal_donotuse.dictCtx = NULL; - } -} - - -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) -{ - if (LZ4_dict->currentOffset + nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ - /* rescale hash table */ - U32 const delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - DEBUGLOG(4, "LZ4_renormDictT"); - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; - else LZ4_dict->hashTable[i] -= delta; - } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } -} - - -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - const tableType_t tableType = byU32; - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - /* invalidate tiny dictionaries */ - if ( (streamPtr->dictSize-1 < 4) /* intentional underflow */ - && (dictEnd != (const BYTE*)source) ) { - DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); - streamPtr->dictSize = 0; - streamPtr->dictionary = (const BYTE*)source; - dictEnd = (const BYTE*)source; - } - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); - else - return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); - } - - /* external dictionary mode */ - { int result; - if (streamPtr->dictCtx) { - /* We depend here on the fact that dictCtx'es (produced by - * LZ4_loadDict) guarantee that their tables contain no references - * to offsets between dictCtx->currentOffset - 64 KB and - * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe - * to use noDictIssue even when the dict isn't a full 64 KB. - */ - if (inputSize > 4 KB) { - /* For compressing large blobs, it is faster to pay the setup - * cost to copy the dictionary's tables into the active context, - * so that the compression loop is only looking into one table. - */ - memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); - } else { - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); - } - } else { - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); - } else { - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); - } - } - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - return result; - } -} - - -/* Hidden debug function, to force-test external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; - int result; - - LZ4_renormDictT(streamPtr, srcSize); - - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { - result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); - } else { - result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); - } - - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)srcSize; - - return result; -} - - -/*! LZ4_saveDict() : - * If previously deCompressBuffer data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -{ - LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; - - return dictSize; -} - - - -/*-******************************* - * Decompression functions - ********************************/ - -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; - -#undef MIN -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -/*! LZ4_decompress_generic() : - * This generic decompression function covers all use cases. - * It shall be instantiated several times, using different sets of directives. - * Note that it is important for performance that this function really get inlined, - * in order to remove useless branches during compilation optimization. - */ -LZ4_FORCE_INLINE int -LZ4_decompress_generic( - const char* const src, - char* const dst, - int srcSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ - - endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */ - earlyEnd_directive partialDecoding, /* full, partial */ - dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) -{ - if (src == NULL) return -1; - - { const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; - - const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; - const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; - const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; - - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - - /* Set up the "end" pointers for the shortcut. */ - const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; - const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; - - DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); - - /* Special cases */ - assert(lowPrefix <= op); - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); - if ((endOnInput) && unlikely(srcSize==0)) return -1; - - /* Main Loop : decode sequences */ - while (1) { - const BYTE* match; - size_t offset; - - unsigned const token = *ip++; - size_t length = token >> ML_BITS; /* literal length */ - - assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ - - /* A two-stage shortcut for the most common case: - * 1) If the literal length is 0..14, and there is enough space, - * enter the shortcut and copy 16 bytes on behalf of the literals - * (in the fast mode, only 8 bytes can be safely copied this way). - * 2) Further if the match length is 4..18, copy 18 bytes in a similar - * manner; but we ensure that there's enough space in the output for - * those 18 bytes earlier, upon entering the shortcut (in other words, - * there is a combined check for both stages). - */ - if ( (endOnInput ? length != RUN_MASK : length <= 8) - /* strictly "less than" on input, to re-enter the loop with at least one byte */ - && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { - /* Copy the literals */ - memcpy(op, ip, endOnInput ? 16 : 8); - op += length; ip += length; - - /* The second stage: prepare for match copying, decode full info. - * If it doesn't work out, the info won't be wasted. */ - length = token & ML_MASK; /* match length */ - offset = LZ4_readLE16(ip); ip += 2; - match = op - offset; - assert(match <= op); /* check overflow */ - - /* Do not deal with overlapping matches. */ - if ( (length != ML_MASK) - && (offset >= 8) - && (dict==withPrefix64k || match >= lowPrefix) ) { - /* Copy the match. */ - memcpy(op + 0, match + 0, 8); - memcpy(op + 8, match + 8, 8); - memcpy(op +16, match +16, 2); - op += length + MINMATCH; - /* Both stages worked, load the next token. */ - continue; - } - - /* The second stage didn't work out, but the info is ready. - * Propel it right to the point of match copying. */ - goto _copy_match; - } - - /* decode literal length */ - if (length == RUN_MASK) { - unsigned s; - if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip= WILDCOPYLENGTH); - if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - memcpy(op, ip, length); - ip += length; - op += length; - if (!partialDecoding || (cpy == oend)) { - /* Necessarily EOF, due to parsing restrictions */ - break; - } - - } else { - LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ - ip += length; op = cpy; - } - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - - /* get matchlength */ - length = token & ML_MASK; - - _copy_match: - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (!partialDecoding) { - assert(oend > op); - assert(oend - op >= 4); - LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ - } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ - - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* match starting within external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ - } - - if (length <= (size_t)(lowPrefix-match)) { - /* match fits entirely within external dictionary : just copy */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match stretches into both external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - - /* partialDecoding : may not respect endBlock parsing restrictions */ - assert(op<=oend); - if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - size_t const mlen = MIN(length, (size_t)(oend-op)); - const BYTE* const matchEnd = match + mlen; - BYTE* const copyEnd = op + mlen; - if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) *op++ = *match++; - } else { - memcpy(op, match, mlen); - } - op = copyEnd; - if (op==oend) break; - continue; - } - - if (unlikely(offset<8)) { - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += inc32table[offset]; - memcpy(op+4, match, 4); - match -= dec64table[offset]; - } else { - memcpy(op, match, 8); - match += 8; - } - op += 8; - - if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (undeCompressBuffer) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op < cpy) *op++ = *match++; - } else { - memcpy(op, match, 8); - if (length > 16) LZ4_wildCopy(op+8, match+8, cpy); - } - op = cpy; /* wildcopy correction */ - } - - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - - /* Overflow error detected */ - _output_error: - return (int) (-(((const char*)ip)-src))-1; - } -} - - -/*===== Instantiate the API decoding functions. =====*/ - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe(const char* source, char* dest, int deCompressBufferSize, int maxDedeCompressBufferSize) -{ - return LZ4_decompress_generic(source, dest, deCompressBufferSize, maxDedeCompressBufferSize, - endOnInputSize, decode_full_block, noDict, - (BYTE*)dest, NULL, 0); -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_partial(const char* src, char* dst, int deCompressBufferSize, int targetOutputSize, int dstCapacity) -{ - dstCapacity = MIN(targetOutputSize, dstCapacity); - return LZ4_decompress_generic(src, dst, deCompressBufferSize, dstCapacity, - endOnInputSize, partial_decode, - noDict, (BYTE*)dst, NULL, 0); -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_fast(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, decode_full_block, withPrefix64k, - (BYTE*)dest - 64 KB, NULL, 0); -} - -/*===== Instantiate a few more decoding cases, used more than once. =====*/ - -LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */ -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int deCompressBufferSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, deCompressBufferSize, maxOutputSize, - endOnInputSize, decode_full_block, withPrefix64k, - (BYTE*)dest - 64 KB, NULL, 0); -} - -/* Another obsolete API function, paired with the previous one. */ -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) -{ - /* LZ4_decompress_fast doesn't validate match offsets, - * and thus serves well with any prefixed dictionary. */ - return LZ4_decompress_fast(source, dest, originalSize); -} - -LZ4_FORCE_O2_GCC_PPC64LE -static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int deCompressBufferSize, int maxOutputSize, - size_t prefixSize) -{ - return LZ4_decompress_generic(source, dest, deCompressBufferSize, maxOutputSize, - endOnInputSize, decode_full_block, noDict, - (BYTE*)dest-prefixSize, NULL, 0); -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, - int deCompressBufferSize, int maxOutputSize, - const void* dictStart, size_t dictSize) -{ - return LZ4_decompress_generic(source, dest, deCompressBufferSize, maxOutputSize, - endOnInputSize, decode_full_block, usingExtDict, - (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -LZ4_FORCE_O2_GCC_PPC64LE -static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, - const void* dictStart, size_t dictSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, decode_full_block, usingExtDict, - (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -/* The "double dictionary" mode, for use with e.g. ring buffers: the first part - * of the dictionary is passed as prefix, and the second via dictStart + dictSize. - * These routines are used only once, in LZ4_decompress_*_continue(). - */ -LZ4_FORCE_INLINE -int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int deCompressBufferSize, int maxOutputSize, - size_t prefixSize, const void* dictStart, size_t dictSize) -{ - return LZ4_decompress_generic(source, dest, deCompressBufferSize, maxOutputSize, - endOnInputSize, decode_full_block, usingExtDict, - (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); -} - -LZ4_FORCE_INLINE -int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize, - size_t prefixSize, const void* dictStart, size_t dictSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, decode_full_block, usingExtDict, - (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); -} - -/*===== streaming decompression functions =====*/ - -LZ4_streamDecode_t* LZ4_createStreamDecode(void) -{ - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); - return lz4s; -} - -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) -{ - if (!LZ4_stream) return 0; /* support free on NULL */ - FREEMEM(LZ4_stream); - return 0; -} - -/*! LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * @return : 1 if OK, 0 if error - */ -int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; -} - -/*! LZ4_decoderRingBufferSize() : - * when setting a ring buffer for streaming decompression (optional scenario), - * provides the minimum size of this ring buffer - * to be compatible with any source respecting maxBlockSize condition. - * Note : in a ring buffer scenario, - * blocks are presumed dedeCompressBuffer next to each other. - * When not enough space remains for next block (remainingSize < maxBlockSize), - * decoding resumes from beginning of ring buffer. - * @return : minimum ring buffer size, - * or 0 if there is an error (invalid maxBlockSize). - */ -int LZ4_decoderRingBufferSize(int maxBlockSize) -{ - if (maxBlockSize < 0) return 0; - if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; - if (maxBlockSize < 16) maxBlockSize = 16; - return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setStreamDecode() -*/ -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int deCompressBufferSize, int maxOutputSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixSize == 0) { - /* The first call, no dictionary yet. */ - assert(lz4sd->extDictSize == 0); - result = LZ4_decompress_safe(source, dest, deCompressBufferSize, maxOutputSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } else if (lz4sd->prefixEnd == (BYTE*)dest) { - /* They're rolling the current segment. */ - if (lz4sd->prefixSize >= 64 KB - 1) - result = LZ4_decompress_safe_withPrefix64k(source, dest, deCompressBufferSize, maxOutputSize); - else if (lz4sd->extDictSize == 0) - result = LZ4_decompress_safe_withSmallPrefix(source, dest, deCompressBufferSize, maxOutputSize, - lz4sd->prefixSize); - else - result = LZ4_decompress_safe_doubleDict(source, dest, deCompressBufferSize, maxOutputSize, - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } else { - /* The buffer wraps around, or they're switching to another buffer. */ - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_safe_forceExtDict(source, dest, deCompressBufferSize, maxOutputSize, - lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixSize == 0) { - assert(lz4sd->extDictSize == 0); - result = LZ4_decompress_fast(source, dest, originalSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } else if (lz4sd->prefixEnd == (BYTE*)dest) { - if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) - result = LZ4_decompress_fast(source, dest, originalSize); - else - result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_fast_extDict(source, dest, originalSize, - lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int deCompressBufferSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return LZ4_decompress_safe(source, dest, deCompressBufferSize, maxOutputSize); - if (dictStart+dictSize == dest) { - if (dictSize >= 64 KB - 1) - return LZ4_decompress_safe_withPrefix64k(source, dest, deCompressBufferSize, maxOutputSize); - return LZ4_decompress_safe_withSmallPrefix(source, dest, deCompressBufferSize, maxOutputSize, dictSize); - } - return LZ4_decompress_safe_forceExtDict(source, dest, deCompressBufferSize, maxOutputSize, dictStart, dictSize); -} - -int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) -{ - if (dictSize==0 || dictStart+dictSize == dest) - return LZ4_decompress_fast(source, dest, originalSize); - return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize); -} - - -/*=************************************************* -* Obsolete Functions -***************************************************/ -/* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_default(source, dest, inputSize, maxOutputSize); -} -int LZ4_compress(const char* source, char* dest, int inputSize) -{ - return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); -} -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) -{ - return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); -} -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) -{ - return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); -} -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) -{ - return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); -} -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) -{ - return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); -} - -/* -These decompression functions are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is totally equivalent to LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe -*/ -int LZ4_uncompress (const char* source, char* dest, int outputSize) -{ - return LZ4_decompress_fast(source, dest, outputSize); -} -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) -{ - return LZ4_decompress_safe(source, dest, isize, maxOutputSize); -} - -/* Obsolete Streaming functions */ - -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -int LZ4_resetStreamState(void* state, char* inputBuffer) -{ - (void)inputBuffer; - LZ4_resetStream((LZ4_stream_t*)state); - return 0; -} - -void* LZ4_create (char* inputBuffer) -{ - (void)inputBuffer; - return LZ4_createStream(); -} - -char* LZ4_slideInputBuffer (void* state) -{ - /* avoid const char * -> char * conversion warning */ - return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; -} - -#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/delta-tool/source/varint.c b/delta-tool/source/varint.c deleted file mode 100644 index 4a2149f..0000000 --- a/delta-tool/source/varint.c +++ /dev/null @@ -1,152 +0,0 @@ -/*- - * Copyright (c) 2018-2019 ARM Limited - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include - -unsigned char TOP_BIT_ON_BYTE = (128); -unsigned char TOP_BIT_OFF_BYTE = (127); - -// return 0 if decode completed otherwise 1 negative on error -var_int_op_code decode_unsigned_varint(unsigned char varIntByte, uint64_t* varIntValue, int count) -{ - var_int_op_code returnValue = OPERATION_COMPLETED; - - if (count > 9 || varIntValue == 0) { - return ERR_PARAMETTERS; - } - - if (count == 0) { - *varIntValue = 0; - } - - int hasMore = varIntByte & TOP_BIT_ON_BYTE; - varIntByte &= TOP_BIT_OFF_BYTE; - - uint64_t byteAs64int = varIntByte; - - for (int i = 0; i < count; i++) { - byteAs64int <<= 7; - } - *varIntValue |= byteAs64int; - - if (hasMore) { - returnValue = OPERATION_NEEDS_MORE_DATA; - } else { - returnValue = OPERATION_COMPLETED; - } - - return returnValue; -} - -int encode_unsigned_varint(uint64_t value, unsigned char *buf, uint32_t BUFF_SIZE_MAX) -{ - unsigned int pos = 0; - do { - if (pos >= BUFF_SIZE_MAX) { - return ERR_BUFFER_TOO_SMALL; // protecting buf from overwrite - } - buf[pos] = (char) value; - value >>= 7; - if (value > 0) { - buf[pos] |= TOP_BIT_ON_BYTE; - } else { - buf[pos] &= TOP_BIT_OFF_BYTE; - } - pos++; - } while (value > 0); - - return pos; -} - -int encode_signed_varint(int64_t value, unsigned char *buf, uint32_t BUFF_SIZE_MAX) -{ - unsigned int pos = 0; - - if (value < 0) { - value = value * -1; // change value to positive number. - value <<= 1; - value |= 1; // set lowest bit 1 if it was negative; - } else { - value <<= 1; // lower bit set to 0 if not negative. - } - - do { - if (pos >= BUFF_SIZE_MAX) { - return ERR_BUFFER_TOO_SMALL; // protecting buf from overwrite - } - buf[pos] = (char) value; - value >>= 7; - if (value > 0) { - buf[pos] |= TOP_BIT_ON_BYTE; - } else { - buf[pos] &= TOP_BIT_OFF_BYTE; - } - pos++; - } while (value > 0); - - return pos; -} - -var_int_op_code decode_signed_varint(unsigned char varIntByte, int64_t* varIntValue, int count) -{ - var_int_op_code returnValue = OPERATION_COMPLETED; - - if (count > 9 || varIntValue == 0) { - return ERR_PARAMETTERS; - } - - if (count == 0) { - *varIntValue = 0; - } - - int hasMore = varIntByte & TOP_BIT_ON_BYTE; - varIntByte &= TOP_BIT_OFF_BYTE; - - uint64_t byteAs64int = varIntByte; - - for (int i = 0; i < count; i++) { - byteAs64int <<= 7; - } - *varIntValue |= byteAs64int; - - if (hasMore) { - returnValue = OPERATION_NEEDS_MORE_DATA; - } else { - returnValue = OPERATION_COMPLETED; - if (*varIntValue & 1) // this is negative value - { - *varIntValue >>= 1; - *varIntValue = *varIntValue * -1; - } else { - *varIntValue >>= 1; // positive value just eat the lowest bit away. - } - } - - return returnValue; -} - diff --git a/delta-tool/tools/delta-tool.py b/delta-tool/tools/delta-tool.py deleted file mode 100644 index f84bf2f..0000000 --- a/delta-tool/tools/delta-tool.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -""" -Copyright (c) 2019 ARM Limited - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -LIBRARIES BUILD -""" -'''Creates a delta patch - -This tool creates a delta patch by calling an underlying binary diff -implementation, then calculating any related information that is needed by the -manifest tool for manifest creation. -''' - -__version__ = '0.0.1' - -import sys -import os -from delta_vendor_info import DeltaInfo -from pyasn1.codec.der import encoder as der_encoder -from pyasn1.codec.native.decoder import decode as native_decode -import binascii -import argparse -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat import backends -import json -import subprocess -import traceback -import warnings -import filecmp - -def digest_file(fd, digest_algo): - fd.seek(0) - md = hashes.Hash(digest_algo(), backends.default_backend()) - flen = 0 - while True: - d = fd.read(8192) - if not d: - break - flen += len(d) - md.update(d) - fd.seek(0) - return md.finalize(), flen - -# return percentage value, how much value2 is from value1 -def diff_percentage(value1, value2): - return 100 * float(value2)/float(value1) - -def parse_arguments(): - parser = argparse.ArgumentParser(description = 'Create a delta patch') - - - parser.add_argument("original", metavar='ORIGINAL_IMAGE-binary', - help='Image currently running on the devices to be updated', - type=argparse.FileType('rb')) - - parser.add_argument("new", metavar='NEW_IMAGE', - help='New image to which update the devices to', - type=argparse.FileType('rb')) - - parser.add_argument('--version', action='version', version=__version__, - help='Display the version' - ) - - group = parser.add_mutually_exclusive_group() - - group.add_argument('-f', '--files', - help='Do not run bsdiff, use files as input arguments will not invoke bsdiff directly. Delta file passed must already be created', action='store_true') - - group.add_argument('-b', '--bsdiff-binary', metavar='BSDIFF_BINARY-executable', - default=os.path.join(os.path.dirname(os.path.realpath(__file__)),'..','bsdiff','bsdiff'), - help='Path the the arm-bsdiff binary to invoke') - - parser.add_argument('-s', '--block-size', metavar='SIZE', - default=512, help='Block size to pass to arm-bsdiff. This must match to memory available to update code (BS_PATCH_COMPILE_TIME_MEMORY_ALLOC or dynamic memory from malloc if compile time is not defined).\n' - 'Higher number will generate slightly smaller patch files but require more memory from patching side') - - parser.add_argument('-d','--delta', metavar='delta-binary-file', - help='Specify the delta output file', default='delta.patch') - parser.add_argument('-i','--input-config', metavar='manifest-tools-input-json', - help='Specify the config input file. stdin by default', - type=argparse.FileType('r'), default=sys.stdin, required=False) - parser.add_argument('-o','--output-config', metavar='manifest-tool-output-json', - help='Specify the config output file. stdout by default', - type=argparse.FileType('w'), default=sys.stdout, required=False) - - group.add_argument('-c', '--size-check', - help='Do not run delta image size check. Otherwise will report error if delta is larger than normal update', action='store_true') - - return parser.parse_args() - - - -def main(): - args = parse_arguments() - config = {} - if not args.input_config.isatty(): - content = args.input_config.read() - if content and len(content) >= 2: #The minimum size of a JSON file is 2: '{}' - config.update(json.loads(content)) - - original_digest, original_size = digest_file(args.original, hashes.SHA256) - new_digest, new_size = digest_file(args.new, hashes.SHA256) - - # Call Delta Function Here if -b is used - if args.files is not True: - #todo for piped version printing anything from progress is impossible - #print ("Generating diff with with "+args.bsdiff_binary) - proc = subprocess.Popen([args.bsdiff_binary, - args.original.name, - args.new.name, - args.delta, - str(args.block_size)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout_data, stderr_data = proc.communicate() - - rc = proc.wait() - if rc: - print('ERROR: {} returned error code {} with output:'.format(proc.args[0], rc)) - print(stdout_data) - print(stderr_data) - sys.exit(rc) - # Delta function done! - - new_digest, new_size = digest_file(args.new, hashes.SHA256) - delta_digest, delta_size = (None, None) - with open(args.delta, 'rb') as fd: - delta_digest, delta_size = digest_file(fd, hashes.SHA256) - - fd.close() - diff = diff_percentage(new_size,delta_size) - diff_str = '%.2f'%(diff) - # Check delta size -c is not used - if args.size_check is not True: - if diff >= 100: - print('Difference with delta image and update image is more than 100 percentage! Delta-file not generated. To override this behavior use -c argument') - os.remove(args.delta) - raise Exception('Difference with delta image and update image is more than 100 percentage! Percentage is: '+diff_str) - - # if delta is more than 60% from update image, show warning - if diff >= 60: - warnings.warn('Difference with delta image and update image is more than 60 percentage. Percentage is: ' + diff_str) - - if (filecmp.cmp(args.new.name, args.original.name) == True): - warnings.warn('New and old file are binary same. This will generate delta that will not change the original image. This is probably an error') - - deltainfo_py = { - 'deltaVariant': 'arm-stream-diff-lz4', - 'precursorDigest': original_digest, - 'deltaDigest': delta_digest, - 'deltaSize': delta_size, - } - - schema = DeltaInfo() - deltainfo_asn1 = native_decode(deltainfo_py, schema) - deltainfo_der = der_encoder.encode(deltainfo_asn1) - - # if delta is more than 60% from update image, show warning - if diff >= 60: - warnings.warn('Delta image size is larger than 60% of full new image size') - - ucc = { - 'payloadHash': binascii.b2a_hex(delta_digest).decode('utf-8'), - 'payloadSize': delta_size, - 'payloadFile' : args.delta, - 'payloadFormat': 'bsdiff-stream', - 'installedFile' : args.new.name, - 'deltaFile' : args.delta, - 'precursorFile' : args.original.name, - } - config.update(ucc) - args.output_config.write(json.dumps(config)) - - -if __name__ == '__main__': - try: - main() - except Exception: - traceback.print_exc() - sys.exit(1) diff --git a/delta-tool/tools/delta-vendor-info.asn1 b/delta-tool/tools/delta-vendor-info.asn1 deleted file mode 100644 index 93fe2bf..0000000 --- a/delta-tool/tools/delta-vendor-info.asn1 +++ /dev/null @@ -1,14 +0,0 @@ --- VendorInfo definition file for stream-bsdiff delta with LZ4 compression in ASN.1 (v. 0.1.0) -VendorInfoDeltaSchema DEFINITIONS IMPLICIT TAGS ::= BEGIN - -DeltaInfo ::= SEQUENCE { - deltaVariant ENUMERATED { - reserved(0), - arm-stream-diff-lz4(1) - }, - precursorDigest OCTET STRING, - deltaDigest OCTET STRING, - deltaSize INTEGER -} - -END diff --git a/delta-tool/tools/delta_vendor_info.py b/delta-tool/tools/delta_vendor_info.py deleted file mode 100644 index 78b5bc9..0000000 --- a/delta-tool/tools/delta_vendor_info.py +++ /dev/null @@ -1,30 +0,0 @@ -#---------------------------------------------------------------------------- -# The confidential and proprietary information contained in this file may -# only be used by a person authorised under and to the extent permitted -# by a subsisting licensing agreement from ARM Limited or its affiliates. -# -# (C) COPYRIGHT 2019 ARM Limited or its affiliates. -# ALL RIGHTS RESERVED -# -# This entire notice must be reproduced on all copies of this file -# and copies of this file may only be made by a person if such person is -# permitted to do so under the terms of a subsisting license agreement -# from ARM Limited or its affiliates. -#---------------------------------------------------------------------------- -# -*- coding: utf-8 -*- -# -# This file has been generated using asn1ate (v ) from 'delta-vendor-info.asn1' -# Last Modified on 2019-02-28 09:57:38.442092 -from pyasn1.type import univ, char, namedtype, namedval, tag, constraint, useful - - -class DeltaInfo(univ.Sequence): - pass - - -DeltaInfo.componentType = namedtype.NamedTypes( - namedtype.NamedType('deltaVariant', univ.Enumerated(namedValues=namedval.NamedValues(('reserved', 0), ('arm-stream-diff-lz4', 1)))), - namedtype.NamedType('precursorDigest', univ.OctetString()), - namedtype.NamedType('deltaDigest', univ.OctetString()), - namedtype.NamedType('deltaSize', univ.Integer()) -) diff --git a/delta-tool/tools/generate-delta-and-manifest.sh b/delta-tool/tools/generate-delta-and-manifest.sh deleted file mode 100755 index 4986bbd..0000000 --- a/delta-tool/tools/generate-delta-and-manifest.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -# --address https://api.us-east-1.mbedcloud.com -set -e -set -x -BINARY=null -OLD_BINARY=null -INPUT_MANIFEST_JSON=null -DELTA_BINARY=null -ADDRESS="https://api.us-east-1.mbedcloud.com" -APIKEY=null - -function help { - echo "Available mandatory parameters are:" - echo "-b|--binary " - echo "-o|--old-binary " - echo "-i|--manifest-input-file " - echo "-k|--apikey " - echo "Available optional parameters are:" - echo "-d|--delta-binary . If delta binary is not given, delta image is named BINARY_delta.bin" - echo "-a|--address
. If not given, uses production API" - exit -} - -while [[ $# -gt 1 ]] -do -key="$1" - case $key in - - -b|--binary) - BINARY="$2" - BINARY_NAME="$(basename $BINARY .bin)" - echo "Using binary $BINARY with name $BINARY_NAME." - shift - ;; - - -o|--old-binary) - OLD_BINARY="$2" - echo "Using old binary $OLD_BINARY." - shift - ;; - - -d|--delta-binary) - DELTA_BINARY="$2" - echo "Using delta binary $DELTA_BINARY." - shift - ;; - - -i|--manifest-input-file) - INPUT_MANIFEST_JSON="$2" - echo "Using manifest input file $INPUT_MANIFEST_JSON." - shift - ;; - - -a|--address) - ADDRESS="$2" - shift - ;; - - -k|--apikey) - APIKEY="$2" - shift - ;; - - *) - echo "Invalid parameter $2 given." - help - ;; - -esac -shift -done - -if [ $BINARY = "null" ] || [ $OLD_BINARY = "null" ] || [ $INPUT_MANIFEST_JSON = "null" ] || [ $APIKEY = "null" ] -then - help -fi - -# if delta file name is not given, use default -if [ $DELTA_BINARY = "null" ] -then - DELTA_BINARY=$BINARY_NAME"_delta.bin" -fi - -# create image and manifest addesses -IMAGEADDRESS=$ADDRESS"/v3/firmware-images" -MANIFESTADDRESS=$ADDRESS"/v3/firmware-manifests" - - -DELTA_BINARY_NAME="$(basename $DELTA_BINARY .bin)" - -MANIFEST=${DELTA_BINARY_NAME}.manifest -echo $MANIFEST - -BINARY_FULLPATH=$(readlink -f $BINARY) -OLD_BINARY_FULLPATH=$(readlink -f $OLD_BINARY) -DELTA_BATCH_BINARY_FULLPATH=$(readlink -f ${DELTA_BINARY}) -INPUT_MANIFEST_JSON_FULLPATH=$(readlink -f ${INPUT_MANIFEST_JSON}) -DELTA_TOOL_FULLPATH=$(readlink -f ./delta-tool.py) -echo "Generating delta $DELTA_BINARY for $BINARY (new image) from $OLD_BINARY (old image)." - -python3 $DELTA_TOOL_FULLPATH $OLD_BINARY_FULLPATH $BINARY_FULLPATH -d $DELTA_BATCH_BINARY_FULLPATH -i $INPUT_MANIFEST_JSON_FULLPATH -o delta-tool-generated.json -RANDOM_NAME="$DELTA_BINARY_NAME$$" -UPLOAD_URI="$(curl -X POST -H "Authorization: Bearer $APIKEY" -F "name=$RANDOM_NAME" -F "datafile=@$DELTA_BATCH_BINARY_FULLPATH" "$IMAGEADDRESS" | python2 -c "import sys, json; print json.load(sys.stdin)['datafile']")" -manifest-tool create -i delta-tool-generated.json -o $MANIFEST --payload-format bsdiff-stream -u $UPLOAD_URI -rm -f -r delta-tool-generated.json diff --git a/mbed-cloud-client.lib b/mbed-cloud-client.lib index 8b7b7bf..5a4d5cf 100644 --- a/mbed-cloud-client.lib +++ b/mbed-cloud-client.lib @@ -1 +1 @@ -https://github.com/PelionIoT/mbed-cloud-client/#64904a11eac01714b9131744dc0a6b181224ef09 +https://github.com/PelionIoT/mbed-cloud-client/#7e45d92cf759d62c2d881bd7be318c807d8a47c2 diff --git a/west.yml b/west.yml index e0d696e..637ddf0 100644 --- a/west.yml +++ b/west.yml @@ -10,7 +10,7 @@ manifest: - name: pelion-dm repo-path: mbed-cloud-client remote: PelionIoT - revision: 64904a11eac01714b9131744dc0a6b181224ef09 + revision: 7e45d92cf759d62c2d881bd7be318c807d8a47c2 path: modules/lib/pelion-dm - name: zephyr remote: zephyrproject-rtos