From e93625d60c2c6a506a5b62072df0bc90c0877e75 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 2 Nov 2015 09:54:07 +0100 Subject: [PATCH] fixes a bug in decompression - fixes a bug in decompression - fullbench is working now --- lib/lz5.c | 78 ++- lib/lz5hc.c | 82 +-- lib/lz5hc.h | 13 +- programs/Makefile | 15 +- programs/fullbench.c | 51 +- programs/fuzzer.c | 1233 ------------------------------------------ programs/lz5cli.c | 3 - programs/lz5io.c | 2 +- 8 files changed, 159 insertions(+), 1318 deletions(-) delete mode 100644 programs/fuzzer.c diff --git a/lib/lz5.c b/lib/lz5.c index 50156b1..faf6cba 100644 --- a/lib/lz5.c +++ b/lib/lz5.c @@ -305,6 +305,7 @@ static const int LZ5_minLength = (MFLIMIT+1); #define MAXD_LOG 22 #define MAX_DISTANCE ((1 << MAXD_LOG) - 1) +#define LZ5_DICT_SIZE (1 << MAXD_LOG) #define ML_BITS 3 #define ML_MASK ((1U<=RUN_MASK) + + if (ip-match < (1<<10)) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; + if (litLength>=RUN_MASK2) + { + int len = (int)litLength-RUN_MASK2; + *token=(RUN_MASK2<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<=RUN_MASK) + { + int len = (int)litLength-RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<>8))< 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; + if ((dictEnd - p) > LZ5_DICT_SIZE) p = dictEnd - LZ5_DICT_SIZE; + dict->currentOffset += LZ5_DICT_SIZE; base = p - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); @@ -1102,7 +1132,7 @@ static void LZ5_renormDictT(LZ5_stream_t_internal* LZ5_dict, const BYTE* src) ((size_t)LZ5_dict->currentOffset > (size_t)src)) /* address space overflow */ { /* rescale hash table */ - U32 delta = LZ5_dict->currentOffset - 64 KB; + U32 delta = LZ5_dict->currentOffset - LZ5_DICT_SIZE; const BYTE* dictEnd = LZ5_dict->dictionary + LZ5_dict->dictSize; int i; for (i=0; ihashTable[i] < delta) LZ5_dict->hashTable[i]=0; else LZ5_dict->hashTable[i] -= delta; } - LZ5_dict->currentOffset = 64 KB; - if (LZ5_dict->dictSize > 64 KB) LZ5_dict->dictSize = 64 KB; + LZ5_dict->currentOffset = LZ5_DICT_SIZE; + if (LZ5_dict->dictSize > LZ5_DICT_SIZE) LZ5_dict->dictSize = LZ5_DICT_SIZE; LZ5_dict->dictionary = dictEnd - LZ5_dict->dictSize; } } @@ -1134,7 +1164,7 @@ int LZ5_compress_fast_continue (LZ5_stream_t* LZ5_stream, const char* source, ch if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize > LZ5_DICT_SIZE) streamPtr->dictSize = LZ5_DICT_SIZE; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; streamPtr->dictionary = dictEnd - streamPtr->dictSize; } @@ -1144,7 +1174,7 @@ int LZ5_compress_fast_continue (LZ5_stream_t* LZ5_stream, const char* source, ch if (dictEnd == (const BYTE*)source) { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + if ((streamPtr->dictSize < LZ5_DICT_SIZE) && (streamPtr->dictSize < streamPtr->currentOffset)) result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); else result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); @@ -1156,7 +1186,7 @@ int LZ5_compress_fast_continue (LZ5_stream_t* LZ5_stream, const char* source, ch /* external dictionary mode */ { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + if ((streamPtr->dictSize < LZ5_DICT_SIZE) && (streamPtr->dictSize < streamPtr->currentOffset)) result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); else result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); @@ -1194,7 +1224,7 @@ int LZ5_saveDict (LZ5_stream_t* LZ5_dict, char* safeBuffer, int dictSize) LZ5_stream_t_internal* dict = (LZ5_stream_t_internal*) LZ5_dict; const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE; /* useless to define a dictionary > LZ5_DICT_SIZE */ if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; memmove(safeBuffer, previousDictEnd - dictSize, dictSize); @@ -1246,7 +1276,7 @@ FORCE_INLINE int LZ5_decompress_generic( const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const int checkOffset = ((safeDecode) && (dictSize < (int)(LZ5_DICT_SIZE))); /* Special cases */ @@ -1298,7 +1328,7 @@ FORCE_INLINE int LZ5_decompress_generic( /* copy literals */ cpy = op+length; - if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(1+1+LASTLITERALS))) ) || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH))) { if (partialDecoding) @@ -1440,7 +1470,7 @@ int LZ5_decompress_safe_partial(const char* source, char* dest, int compressedSi int LZ5_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); + return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - LZ5_DICT_SIZE), NULL, LZ5_DICT_SIZE); } @@ -1567,8 +1597,8 @@ FORCE_INLINE int LZ5_decompress_usingDict_generic(const char* source, char* dest return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + if (dictSize >= (int)(LZ5_DICT_SIZE - 1)) + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-LZ5_DICT_SIZE, NULL, 0); return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); } return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); @@ -1639,7 +1669,7 @@ void* LZ5_create (char* inputBuffer) char* LZ5_slideInputBuffer (void* LZ5_Data) { LZ5_stream_t_internal* ctx = (LZ5_stream_t_internal*)LZ5_Data; - int dictSize = LZ5_saveDict((LZ5_stream_t*)LZ5_Data, (char*)ctx->bufferStart, 64 KB); + int dictSize = LZ5_saveDict((LZ5_stream_t*)LZ5_Data, (char*)ctx->bufferStart, LZ5_DICT_SIZE); return (char*)(ctx->bufferStart + dictSize); } @@ -1647,12 +1677,12 @@ char* LZ5_slideInputBuffer (void* LZ5_Data) int LZ5_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { - return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - LZ5_DICT_SIZE, NULL, LZ5_DICT_SIZE); } int LZ5_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) { - return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); + return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - LZ5_DICT_SIZE, NULL, LZ5_DICT_SIZE); } #endif /* LZ5_COMMONDEFS_ONLY */ diff --git a/lib/lz5hc.c b/lib/lz5hc.c index 6ea046c..ae1a7c8 100644 --- a/lib/lz5hc.c +++ b/lib/lz5hc.c @@ -92,7 +92,7 @@ static const int g_maxCompressionLevel = 16; /************************************** * Local Types **************************************/ -typedef struct +struct LZ5HC_Data_s { U32* hashTable; U32* chainTable; @@ -104,7 +104,7 @@ typedef struct U32 lowLimit; /* below that point, no more dict */ U32 nextToUpdate; /* index from which to continue dictionary update */ U32 compressionLevel; -} LZ5HC_Data_Structure; +}; /************************************** @@ -584,6 +584,29 @@ int LZ5_compress_HC_extStateHC (void* state, const char* src, char* dst, int src return LZ5HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); } +int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* statePtr) +{ + statePtr->hashTable = ALLOCATOR(1, sizeof(U32)*HASHTABLESIZE); + if (!statePtr->hashTable) + return 0; + + statePtr->chainTable = ALLOCATOR(1, sizeof(U32)*MAXD); + if (!statePtr->chainTable) + { + FREEMEM(statePtr->hashTable); + statePtr->hashTable = NULL; + return 0; + } + + return 1; +} + +void LZ5_free_mem_HC(LZ5HC_Data_Structure* statePtr) +{ + if (statePtr->chainTable) FREEMEM(statePtr->chainTable); + if (statePtr->hashTable) FREEMEM(statePtr->hashTable); +} + int LZ5_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { #if LZ5HC_HEAPMODE==1 @@ -594,17 +617,13 @@ int LZ5_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int #endif int cSize = 0; - statePtr->hashTable = ALLOCATOR(1, sizeof(U32)*HASHTABLESIZE); - if (statePtr->hashTable) - { - statePtr->chainTable = ALLOCATOR(1, sizeof(U32)*MAXD); - if (statePtr->chainTable) - { - cSize = LZ5_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); - FREEMEM(statePtr->chainTable); - } - FREEMEM(statePtr->hashTable); - } + + if (!LZ5_alloc_mem_HC(statePtr)) + return 0; + + cSize = LZ5_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); + + LZ5_free_mem_HC(statePtr); #if LZ5HC_HEAPMODE==1 free(statePtr); @@ -624,29 +643,19 @@ LZ5_streamHC_t* LZ5_createStreamHC(void) if (!statePtr) return NULL; - statePtr->hashTable = ALLOCATOR(1, sizeof(U32)*HASHTABLESIZE); - if (!statePtr->hashTable) + if (!LZ5_alloc_mem_HC(statePtr)) { FREEMEM(statePtr); return NULL; } - - statePtr->chainTable = ALLOCATOR(1, sizeof(U32)*MAXD); - if (!statePtr->chainTable) - { - FREEMEM(statePtr->hashTable); - FREEMEM(statePtr); - return NULL; - } return (LZ5_streamHC_t*) statePtr; } -int LZ5_freeStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr) +int LZ5_freeStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr) { LZ5HC_Data_Structure* statePtr = (LZ5HC_Data_Structure*)LZ5_streamHCPtr; - FREEMEM(statePtr->chainTable); - FREEMEM(statePtr->hashTable); + LZ5_free_mem_HC(statePtr); free(LZ5_streamHCPtr); return 0; } @@ -663,10 +672,10 @@ void LZ5_resetStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr, int compressionLevel) int LZ5_loadDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, const char* dictionary, int dictSize) { LZ5HC_Data_Structure* ctxPtr = (LZ5HC_Data_Structure*) LZ5_streamHCPtr; - if (dictSize > 64 KB) + if (dictSize > LZ5_DICT_SIZE) { - dictionary += dictSize - 64 KB; - dictSize = 64 KB; + dictionary += dictSize - LZ5_DICT_SIZE; + dictSize = LZ5_DICT_SIZE; } LZ5HC_init (ctxPtr, (const BYTE*)dictionary); if (dictSize >= 4) LZ5HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); @@ -702,7 +711,7 @@ static int LZ5_compressHC_continue_generic (LZ5HC_Data_Structure* ctxPtr, if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; - if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE; LZ5_loadDictHC((LZ5_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } @@ -742,7 +751,7 @@ int LZ5_saveDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, char* safeBuffer, int dictS { LZ5HC_Data_Structure* streamPtr = (LZ5HC_Data_Structure*)LZ5_streamHCPtr; int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); - if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; memmove(safeBuffer, streamPtr->end - dictSize, dictSize); @@ -757,3 +766,14 @@ int LZ5_saveDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, char* safeBuffer, int dictS return dictSize; } +/*********************************** +* Deprecated Functions +***********************************/ +/* Deprecated compression functions */ +/* These functions are planned to start generate warnings by r131 approximately */ +int LZ5_compressHC(const char* src, char* dst, int srcSize) { return LZ5_compress_HC (src, dst, srcSize, LZ5_compressBound(srcSize), 0); } +int LZ5_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ5_compressHC_continue (LZ5_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ5_compress_HC_continue (ctx, src, dst, srcSize, LZ5_compressBound(srcSize)); } +int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } +int LZ5_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ5_compress_HC_extStateHC (state, src, dst, srcSize, LZ5_compressBound(srcSize), 0); } +int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } diff --git a/lib/lz5hc.h b/lib/lz5hc.h index 29be73d..23d37fc 100644 --- a/lib/lz5hc.h +++ b/lib/lz5hc.h @@ -66,6 +66,10 @@ LZ5_compress_HC : Decompression functions are provided within LZ5 source code (see "lz5.h") (BSD license) */ +typedef struct LZ5HC_Data_s LZ5HC_Data_Structure; + +int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* statePtr); +void LZ5_free_mem_HC(LZ5HC_Data_Structure* statePtr); int LZ5_sizeofStateHC(void); int LZ5_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); @@ -165,15 +169,10 @@ int LZ5_saveDictHC (LZ5_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSi /* these functions are planned to trigger warning messages by r131 approximately */ int LZ5_compressHC (const char* source, char* dest, int inputSize); int LZ5_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -int LZ5_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -int LZ5_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -int LZ5_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -int LZ5_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -int LZ5_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); int LZ5_compressHC_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize); int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - +int LZ5_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); #if defined (__cplusplus) } diff --git a/programs/Makefile b/programs/Makefile index 806b37b..8a4bc97 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -25,8 +25,6 @@ # lz5 : Command Line Utility, supporting gzip-like arguments # lz5c : CLU, supporting also legacy lz5demo arguments # lz5c32: Same as lz5c, but forced to compile in 32-bits mode -# fuzzer : Test tool, to check lz5 integrity on target platform -# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode # frametest : Test tool, to check lz5frame integrity on target platform # frametest32: Same as frametest, but forced to compile in 32-bits mode # fullbench : Precisely measure speed for each LZ5 function variant @@ -65,9 +63,9 @@ FUZZER_TIME := -T9mn default: lz5 -m32: lz5c32 fullbench32 fuzzer32 frametest32 +m32: lz5c32 fullbench32 frametest32 -bins: lz5 lz5c fullbench fuzzer frametest datagen +bins: lz5 fullbench frametest datagen all: bins m32 @@ -80,18 +78,12 @@ lz5c : $(LZ5DIR)/lz5.o $(LZ5DIR)/lz5hc.o $(LZ5DIR)/lz5frame.o $(LZ5DIR)/xxhash. lz5c32: $(LZ5DIR)/lz5.c $(LZ5DIR)/lz5hc.c $(LZ5DIR)/lz5frame.c $(LZ5DIR)/xxhash.c bench.c lz5io.c lz5cli.c $(CC) -m32 $(FLAGS) -DENABLE_LZ5C_LEGACY_OPTIONS $^ -o $@$(EXT) -fullbench : $(LZ5DIR)/lz5.o $(LZ5DIR)/lz5hc.o $(LZ5DIR)/lz5frame.o $(LZ5DIR)/xxhash.o fullbench.c +fullbench: $(LZ5DIR)/lz5.o $(LZ5DIR)/lz5hc.o $(LZ5DIR)/lz5frame.o $(LZ5DIR)/xxhash.o fullbench.o $(CC) $(FLAGS) $^ -o $@$(EXT) fullbench32: $(LZ5DIR)/lz5.c $(LZ5DIR)/lz5hc.c $(LZ5DIR)/lz5frame.c $(LZ5DIR)/xxhash.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -fuzzer : $(LZ5DIR)/lz5.o $(LZ5DIR)/lz5hc.o $(LZ5DIR)/xxhash.o fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fuzzer32: $(LZ5DIR)/lz5.c $(LZ5DIR)/lz5hc.c $(LZ5DIR)/xxhash.c fuzzer.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - frametest: $(LZ5DIR)/lz5frame.o $(LZ5DIR)/lz5.o $(LZ5DIR)/lz5hc.o $(LZ5DIR)/xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -106,7 +98,6 @@ clean: @rm -f core *.o *.test tmp* \ lz5$(EXT) lz5c$(EXT) lz5c32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) \ frametest$(EXT) frametest32$(EXT) \ datagen$(EXT) @echo Cleaning completed diff --git a/programs/fullbench.c b/programs/fullbench.c index edfb7c5..8b71335 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -482,27 +482,57 @@ static int local_LZ5_saveDictHC(const char* in, char* out, int inSize) static int local_LZ5_compressHC_withStateHC(const char* in, char* out, int inSize) { - return LZ5_compressHC_withStateHC(&LZ5_streamHC, in, out, inSize); + int res = 0; + if (LZ5_alloc_mem_HC((LZ5HC_Data_Structure*)(&LZ5_streamHC))) + { + res = LZ5_compressHC_withStateHC(&LZ5_streamHC, in, out, inSize); + LZ5_free_mem_HC((LZ5HC_Data_Structure*)&LZ5_streamHC); + } + return res; } static int local_LZ5_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) { - return LZ5_compressHC_limitedOutput_withStateHC(&LZ5_streamHC, in, out, inSize, LZ5_compressBound(inSize)-1); + int res = 0; + if (LZ5_alloc_mem_HC((LZ5HC_Data_Structure*)(&LZ5_streamHC))) + { + res = LZ5_compressHC_limitedOutput_withStateHC(&LZ5_streamHC, in, out, inSize, LZ5_compressBound(inSize)-1); + LZ5_free_mem_HC((LZ5HC_Data_Structure*)&LZ5_streamHC); + } + return res; } static int local_LZ5_compressHC_limitedOutput(const char* in, char* out, int inSize) { - return LZ5_compressHC_limitedOutput(in, out, inSize, LZ5_compressBound(inSize)-1); + int res = 0; + if (LZ5_alloc_mem_HC((LZ5HC_Data_Structure*)(&LZ5_streamHC))) + { + res = LZ5_compressHC_limitedOutput(in, out, inSize, LZ5_compressBound(inSize)-1); + LZ5_free_mem_HC((LZ5HC_Data_Structure*)&LZ5_streamHC); + } + return res; } static int local_LZ5_compressHC_continue(const char* in, char* out, int inSize) { - return LZ5_compressHC_continue(&LZ5_streamHC, in, out, inSize); + int res = 0; + if (LZ5_alloc_mem_HC((LZ5HC_Data_Structure*)(&LZ5_streamHC))) + { + res = LZ5_compressHC_continue(&LZ5_streamHC, in, out, inSize); + LZ5_free_mem_HC((LZ5HC_Data_Structure*)&LZ5_streamHC); + } + return res; } static int local_LZ5_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) { - return LZ5_compressHC_limitedOutput_continue(&LZ5_streamHC, in, out, inSize, LZ5_compressBound(inSize)-1); + int res = 0; + if (LZ5_alloc_mem_HC((LZ5HC_Data_Structure*)(&LZ5_streamHC))) + { + res = LZ5_compressHC_limitedOutput_continue(&LZ5_streamHC, in, out, inSize, LZ5_compressBound(inSize)-1); + LZ5_free_mem_HC((LZ5HC_Data_Structure*)&LZ5_streamHC); + } + return res; } @@ -556,7 +586,9 @@ static int local_LZ5F_decompress(const char* in, char* out, int inSize, int outS size_t srcSize = inSize; size_t dstSize = outSize; size_t result; +// printf("srcSize=%d dstSize=%d\n", (int)srcSize,(int)dstSize); result = LZ5F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); +// printf("srcSize=%d dstSize=%d result=%d\n", (int)srcSize,(int)dstSize,(int)result); if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); } if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } return (int)dstSize; @@ -690,7 +722,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) case 10: compressionFunction = LZ5_compressHC; compressorName = "LZ5_compressHC"; break; case 11: compressionFunction = local_LZ5_compressHC_limitedOutput; compressorName = "LZ5_compressHC_limitedOutput"; break; - case 12 : compressionFunction = local_LZ5_compressHC_withStateHC; compressorName = "LZ5_compressHC_withStateHC"; break; + case 12: compressionFunction = local_LZ5_compressHC_withStateHC; compressorName = "LZ5_compressHC_withStateHC"; break; case 13: compressionFunction = local_LZ5_compressHC_limitedOutput_withStateHC; compressorName = "LZ5_compressHC_limitedOutput_withStateHC"; break; case 14: compressionFunction = local_LZ5_compressHC_continue; initFunction = local_LZ5_resetStreamHC; compressorName = "LZ5_compressHC_continue"; break; case 15: compressionFunction = local_LZ5_compressHC_limitedOutput_continue; initFunction = local_LZ5_resetStreamHC; compressorName = "LZ5_compressHC_limitedOutput_continue"; break; @@ -702,7 +734,11 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) LZ5_loadDict(&LZ5_stream, chunkP[0].origBuffer, chunkP[0].origSize); break; case 41: compressionFunction = local_LZ5_saveDictHC; compressorName = "LZ5_saveDictHC"; - LZ5_loadDictHC(&LZ5_streamHC, chunkP[0].origBuffer, chunkP[0].origSize); + if (LZ5_alloc_mem_HC((LZ5HC_Data_Structure*)(&LZ5_streamHC))) + { + LZ5_loadDictHC(&LZ5_streamHC, chunkP[0].origBuffer, chunkP[0].origSize); + LZ5_free_mem_HC((LZ5HC_Data_Structure*)&LZ5_streamHC); + } break; case 60: DISPLAY("Obsolete compression functions : \n"); continue; case 61: compressionFunction = LZ5_compress; compressorName = "LZ5_compress"; break; @@ -805,6 +841,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) free(chunkP); return 1; } + // printf("(int)errorCode=%d benchedSize=%d\n", (int)errorCode, (int)benchedSize); chunkP[0].origSize = (int)benchedSize; chunkP[0].compressedSize = (int)errorCode; nbChunks = 1; diff --git a/programs/fuzzer.c b/programs/fuzzer.c deleted file mode 100644 index 7e8b847..0000000 --- a/programs/fuzzer.c +++ /dev/null @@ -1,1233 +0,0 @@ -/* - fuzzer.c - Fuzzer test tool for LZ5 - Copyright (C) Yann Collet 2012-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ5 source mirror : https://github.com/inikep/lz5 - - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c -*/ - -/************************************** -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ -#endif - -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if defined(_MSC_VER) || defined(_WIN32) -# define FUZ_LEGACY_TIMER 1 -#endif - - -/************************************** -* Includes -**************************************/ -#include -#include /* fgets, sscanf */ -#include /* strcmp */ -#include "lz5.h" -#include "lz5hc.h" -#include "xxhash.h" - -/* Use ftime() if gettimeofday() is not available on your target */ -#if defined(FUZ_LEGACY_TIMER) -# include /* timeb, ftime */ -#else -# include /* gettimeofday */ -#endif - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - -/************************************** -* Constants -**************************************/ -#ifndef LZ5_VERSION -# define LZ5_VERSION "" -#endif - -#define NB_ATTEMPTS (1<<16) -#define COMPRESSIBLE_NOISE_LENGTH (1 << 21) -#define FUZ_MAX_BLOCK_SIZE (1 << 17) -#define FUZ_MAX_DICT_SIZE (1 << 15) -#define FUZ_COMPRESSIBILITY_DEFAULT 60 -#define PRIME1 2654435761U -#define PRIME2 2246822519U -#define PRIME3 3266489917U - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - - -/***************************************** -* Macros -*****************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static int g_displayLevel = 2; -static const U32 g_refreshRate = 250; -static U32 g_time = 0; - - -/********************************************************* -* Fuzzer functions -*********************************************************/ -#if defined(FUZ_LEGACY_TIMER) - -static U32 FUZ_GetMilliStart(void) -{ - struct timeb tb; - U32 nCount; - ftime( &tb ); - nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); - return nCount; -} - -#else - -static U32 FUZ_GetMilliStart(void) -{ - struct timeval tv; - U32 nCount; - gettimeofday(&tv, NULL); - nCount = (U32) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); - return nCount; -} - -#endif - - -static U32 FUZ_GetMilliSpan(U32 nTimeStart) -{ - U32 nCurrent = FUZ_GetMilliStart(); - U32 nSpan = nCurrent - nTimeStart; - if (nTimeStart > nCurrent) - nSpan += 0x100000 * 1000; - return nSpan; -} - -static U32 FUZ_rotl32(U32 u32, U32 nbBits) -{ - return ((u32 << nbBits) | (u32 >> (32 - nbBits))); -} - -static U32 FUZ_rand(U32* src) -{ - U32 rand32 = *src; - rand32 *= PRIME1; - rand32 ^= PRIME2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32; -} - - -#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) -#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) -{ - BYTE* BBuffer = (BYTE*)buffer; - size_t pos = 0; - U32 P32 = (U32)(32768 * proba); - - /* First Bytes */ - while (pos < 20) - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - - while (pos < bufferSize) - { - /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) - { - /* Copy (within 64K) */ - size_t match, d; - size_t length = FUZ_RANDLENGTH + 4; - size_t offset = FUZ_RAND15BITS + 1; - while (offset > pos) offset >>= 1; - d = pos + length; - while (d > bufferSize) d = bufferSize; - match = pos - offset; - while (pos < d) BBuffer[pos++] = BBuffer[match++]; - } - else - { - /* Literal (noise) */ - size_t d; - size_t length = FUZ_RANDLENGTH; - d = pos + length; - if (d > bufferSize) d = bufferSize; - while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } -} - - -#define MAX_NB_BUFF_I134 150 -#define BLOCKSIZE_I134 (32 MB) -static int FUZ_AddressOverflow(void) -{ - char* buffers[MAX_NB_BUFF_I134+1]; - int i, nbBuff=0; - int highAddress = 0; - - DISPLAY("Overflow tests : "); - - /* Only possible in 32-bits */ - if (sizeof(void*)==8) - { - DISPLAY("64 bits mode : no overflow \n"); - fflush(stdout); - return 0; - } - - buffers[0] = (char*)malloc(BLOCKSIZE_I134); - buffers[1] = (char*)malloc(BLOCKSIZE_I134); - if ((!buffers[0]) || (!buffers[1])) - { - DISPLAY("not enough memory for tests \n"); - return 0; - } - - for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) - { - DISPLAY("%3i \b\b\b\b", nbBuff); - buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); - if (buffers[nbBuff]==NULL) goto _endOfTests; - //DISPLAY("%08X ", (U32)(size_t)(buffers[nbBuff])); fflush(stdout); - - if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress)) - { - DISPLAY("high address detected : "); - fflush(stdout); - highAddress=1; - } - - { - size_t sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512); - int nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); - char* input = buffers[nbBuff-1]; - char* output = buffers[nbBuff]; - int r; - input[0] = (char)0xF0; // Literal length overflow - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff; - r = LZ5_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - input[0] = (char)0x1F; // Match length overflow - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ5_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - - output = buffers[nbBuff-2]; // Reverse in/out pointer order - input[0] = (char)0xF0; // Literal length overflow - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - r = LZ5_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - input[0] = (char)0x1F; // Match length overflow - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ5_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - } - } - - nbBuff++; -_endOfTests: - for (i=0 ; i g_refreshRate) || (g_displayLevel>=3)) - { - g_time = FUZ_GetMilliStart(); - DISPLAY("\r%5u ", testNb); - if (g_displayLevel>=3) fflush(stdout); - } -} - - -static void FUZ_findDiff(const void* buff1, const void* buff2) -{ - const BYTE* b1 = (const BYTE*)buff1; - const BYTE* b2 = (const BYTE*)buff2; - size_t i=0; - while (b1[i]==b2[i]) i++; - DISPLAY("Wrong Byte at position %u\n", (unsigned)i); -} - - -static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration) -{ - unsigned long long bytes = 0; - unsigned long long cbytes = 0; - unsigned long long hcbytes = 0; - unsigned long long ccbytes = 0; - void* CNBuffer; - char* compressedBuffer; - char* decodedBuffer; -# define FUZ_max LZ5_COMPRESSBOUND(LEN) - int ret; - unsigned cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } - void* stateLZ5 = malloc(LZ5_sizeofState()); - void* stateLZ5HC = malloc(LZ5_sizeofStateHC()); - LZ5_stream_t LZ5dict; - LZ5_streamHC_t LZ5dictHC; - U32 crcOrig, crcCheck; - U32 coreRandState = seed; - U32 randState = coreRandState ^ PRIME3; - int result = 0; - const U32 startTime = FUZ_GetMilliStart(); - - - /* init */ - memset(&LZ5dict, 0, sizeof(LZ5dict)); - duration *= 1000; - - /* Create compressible test buffer */ - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - compressedBuffer = (char*)malloc(LZ5_compressBound(FUZ_MAX_BLOCK_SIZE)); - decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); - - /* move to startCycle */ - for (cycleNb = 0; cycleNb < startCycle; cycleNb++) - { - (void)FUZ_rand(&coreRandState); - - if (0) /* some problems are related to dictionary re-use; in this case, enable this loop */ - { - int dictSize, blockSize, blockStart; - char* dict; - char* block; - FUZ_displayUpdate(cycleNb); - randState = coreRandState ^ PRIME3; - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - LZ5_loadDict(&LZ5dict, dict, dictSize); - LZ5_compress_continue(&LZ5dict, block, compressedBuffer, blockSize); - LZ5_loadDict(&LZ5dict, dict, dictSize); - LZ5_compress_continue(&LZ5dict, block, compressedBuffer, blockSize); - LZ5_loadDict(&LZ5dict, dict, dictSize); - LZ5_compress_continue(&LZ5dict, block, compressedBuffer, blockSize); - } - } - - /* Main test loop */ - for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetMilliSpan(startTime) < duration) ; cycleNb++) - { - U32 testNb = 0; - char* dict; - char* block; - int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; - int blockContinueCompressedSize; - - FUZ_displayUpdate(cycleNb); - (void)FUZ_rand(&coreRandState); - randState = coreRandState ^ PRIME3; - - /* Select block to test */ - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - - /* Compression tests */ - - /* Test compression destSize */ - FUZ_DISPLAYTEST; - { - int srcSize = blockSize; - int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; - compressedBuffer[targetSize] = endCheck; - ret = LZ5_compress_destSize(block, compressedBuffer, &srcSize, targetSize); - FUZ_CHECKTEST(ret > targetSize, "LZ5_compress_destSize() result larger than dst buffer !"); - FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ5_compress_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(srcSize > blockSize, "LZ5_compress_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); - if (targetSize>0) - { - FUZ_CHECKTEST((ret==0), "LZ5_compress_destSize() compression failed"); - /* check correctness */ - FUZ_DISPLAYTEST; - - crcOrig = XXH32(block, srcSize, 0); - compressedSize = ret; - endCheck = FUZ_rand(&randState) & 255; - decodedBuffer[srcSize] = endCheck; - ret = LZ5_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ5_decompress_safe() failed on data compressed by LZ5_compress_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ5_decompress_safe() failed : did not fully decompressed data"); - FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ5_decompress_safe() overwrite dst buffer !"); - crcCheck = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_safe() corrupted decoded data"); - - DISPLAYLEVEL(5, " OK \n"); - } - else - DISPLAYLEVEL(5, " \n"); - } - - /* Test compression HC */ - FUZ_DISPLAYTEST; - ret = LZ5_compressHC(block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ5_compressHC() failed"); - HCcompressedSize = ret; - - /* Test compression HC using external state */ - FUZ_DISPLAYTEST; - ret = LZ5_compressHC_withStateHC(stateLZ5HC, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ5_compressHC_withStateHC() failed"); - - /* Test compression using external state */ - FUZ_DISPLAYTEST; - ret = LZ5_compress_withState(stateLZ5, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ5_compress_withState() failed"); - - /* Test compression */ - FUZ_DISPLAYTEST; - ret = LZ5_compress(block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ5_compress() failed"); - compressedSize = ret; - - /* Decompression tests */ - - crcOrig = XXH32(block, blockSize, 0); - - /* Test decoding with output size being exactly what's necessary => must work */ - FUZ_DISPLAYTEST; - ret = LZ5_decompress_fast(compressedBuffer, decodedBuffer, blockSize); - FUZ_CHECKTEST(ret<0, "LZ5_decompress_fast failed despite correct space"); - FUZ_CHECKTEST(ret!=compressedSize, "LZ5_decompress_fast failed : did not fully read compressed data"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_fast corrupted decoded data"); - - /* Test decoding with one byte missing => must fail */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ5_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_fast should have failed, due to Output Size being too small"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ5_decompress_fast overrun specified output buffer"); - - /* Test decoding with one byte too much => must fail */ - FUZ_DISPLAYTEST; - ret = LZ5_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_fast should have failed, due to Output Size being too large"); - - /* Test decoding with output size exactly what's necessary => must work */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ5_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); - FUZ_CHECKTEST(ret<0, "LZ5_decompress_safe failed despite sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ5_decompress_safe did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ5_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_safe corrupted decoded data"); - - // Test decoding with more than enough output size => must work - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - decodedBuffer[blockSize+1] = 0; - ret = LZ5_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); - FUZ_CHECKTEST(ret<0, "LZ5_decompress_safe failed despite amply sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ5_decompress_safe did not regenerate original data"); - //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ5_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? - FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ5_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_safe corrupted decoded data"); - - // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ5_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_safe should have failed, due to Output Size being one byte too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ5_decompress_safe overrun specified output buffer size"); - - // Test decoding with output size being 10 bytes too short => must fail - FUZ_DISPLAYTEST; - if (blockSize>10) - { - decodedBuffer[blockSize-10] = 0; - ret = LZ5_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_safe should have failed, due to Output Size being 10 bytes too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ5_decompress_safe overrun specified output buffer size"); - } - - // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; - ret = LZ5_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); - - // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ5_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_safe should have failed, due to input size being too large"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ5_decompress_safe overrun specified output buffer size"); - - // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; - ret = LZ5_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); - FUZ_CHECKTEST(ret<0, "LZ5_decompress_safe_partial failed despite sufficient space"); - - // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; - ret = LZ5_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); - FUZ_CHECKTEST(ret<0, "LZ5_decompress_safe_partial failed despite sufficient space"); - - /* Test Compression with limited output size */ - - /* Test compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ5_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ5_compress_limitedOutput() failed despite sufficient space"); - - /* Test compression with output size being exactly what's necessary and external state (should work) */ - FUZ_DISPLAYTEST; - ret = LZ5_compress_limitedOutput_withState(stateLZ5, block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ5_compress_limitedOutput_withState() failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ5_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize); - FUZ_CHECKTEST(ret==0, "LZ5_compressHC_limitedOutput() failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ5_compressHC_limitedOutput_withStateHC(stateLZ5HC, block, compressedBuffer, blockSize, HCcompressedSize); - FUZ_CHECKTEST(ret==0, "LZ5_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); - - /* Test compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { - int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= compressedSize) missingBytes = compressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[compressedSize-missingBytes] = 0; - ret = LZ5_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ5_compress_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ5_compress_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) - } - - /* Test HC compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { - int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ5_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ5_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ5_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) - } - - - /********************/ - /* Dictionary tests */ - /********************/ - - /* Compress using dictionary */ - FUZ_DISPLAYTEST; - { - LZ5_stream_t LZ5_stream; - LZ5_resetStream(&LZ5_stream); - LZ5_compress_continue (&LZ5_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */ - blockContinueCompressedSize = LZ5_compress_continue (&LZ5_stream, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ5_compress_continue failed"); - } - - /* Decompress with dictionary as prefix */ - FUZ_DISPLAYTEST; - memcpy(decodedBuffer, dict, dictSize); - ret = LZ5_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ5_decompress_fast_withPrefix64k did not read all compressed block input"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - if (crcCheck!=crcOrig) - { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - - } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - ret = LZ5_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ5_decompress_safe_usingDict did not regenerate original data"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_safe_usingDict corrupted decoded data"); - - /* Compress using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ5_loadDict(&LZ5dict, dict, dictSize); - blockContinueCompressedSize = LZ5_compress_continue(&LZ5dict, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ5_compress_continue failed"); - - FUZ_DISPLAYTEST; - LZ5_loadDict(&LZ5dict, dict, dictSize); - ret = LZ5_compress_limitedOutput_continue(&LZ5dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ5_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); - - FUZ_DISPLAYTEST; - LZ5_loadDict(&LZ5dict, dict, dictSize); - ret = LZ5_compress_limitedOutput_continue(&LZ5dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ5_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ5_compress_limitedOutput_continue should work : enough size available within output buffer"); - - /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ5_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ5_decompress_fast_usingDict did not read all compressed block input"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ5_decompress_fast_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ5_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ5_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ5_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_safe_usingDict corrupted decoded data"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ5_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ5_decompress_fast_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ5_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ5_decompress_safe_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - { - U32 missingBytes = (FUZ_rand(&randState) & 0xF) + 2; - if ((U32)blockSize > missingBytes) - { - decodedBuffer[blockSize-missingBytes] = 0; - ret = LZ5_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ5_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); - FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ5_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); - } - } - - /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ5_resetStreamHC (&LZ5dictHC, FUZ_rand(&randState) & 0x7); - LZ5_loadDictHC(&LZ5dictHC, dict, dictSize); - blockContinueCompressedSize = LZ5_compressHC_continue(&LZ5dictHC, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ5_compressHC_continue failed"); - - FUZ_DISPLAYTEST; - LZ5_loadDictHC(&LZ5dictHC, dict, dictSize); - ret = LZ5_compressHC_limitedOutput_continue(&LZ5dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ5_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); - - FUZ_DISPLAYTEST; - LZ5_loadDictHC(&LZ5dictHC, dict, dictSize); - ret = LZ5_compressHC_limitedOutput_continue(&LZ5dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ5_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ5_compress_limitedOutput_continue should work : enough size available within output buffer"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ5_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ5_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ5_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ5_decompress_safe_usingDict corrupted decoded data"); - - /* ***** End of tests *** */ - /* Fill stats */ - bytes += blockSize; - cbytes += compressedSize; - hcbytes += HCcompressedSize; - ccbytes += blockContinueCompressedSize; - } - - if (nbCycles<=1) nbCycles = cycleNb; /* end by time */ - bytes += !bytes; /* avoid division by 0 */ - printf("\r%7u /%7u - ", cycleNb, nbCycles); - printf("all tests completed successfully \n"); - printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); - printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); - printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); - - /* release memory */ - { -_exit: - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - free(stateLZ5); - free(stateLZ5HC); - return result; - -_output_error: - result = 1; - goto _exit; - } -} - - -#define testInputSize (192 KB) -#define testCompressedSize (128 KB) -#define ringBufferSize (8 KB) - -static void FUZ_unitTests(void) -{ - const unsigned testNb = 0; - const unsigned seed = 0; - const unsigned cycleNb= 0; - char testInput[testInputSize]; - char testCompressed[testCompressedSize]; - char testVerify[testInputSize]; - char ringBuffer[ringBufferSize]; - U32 randState = 1; - - /* Init */ - FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); - - /* 32-bits address space overflow test */ - FUZ_AddressOverflow(); - - /* LZ5 streaming tests */ - { - LZ5_stream_t* statePtr; - LZ5_stream_t streamingState; - U64 crcOrig; - U64 crcNew; - int result; - - /* Allocation test */ - statePtr = LZ5_createStream(); - FUZ_CHECKTEST(statePtr==NULL, "LZ5_createStream() allocation failed"); - LZ5_freeStream(statePtr); - - /* simple compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ5_resetStream(&streamingState); - result = LZ5_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ5_compress_limitedOutput_continue() compression failed"); - - result = LZ5_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ5_decompress_safe() decompression failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ5_decompress_safe() decompression corruption"); - - /* ring buffer test */ - { - XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ5_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - } - - /* LZ5 HC streaming tests */ - { - LZ5_streamHC_t* sp; - LZ5_streamHC_t sHC; - U64 crcOrig; - U64 crcNew; - int result; - - /* Allocation test */ - sp = LZ5_createStreamHC(); - FUZ_CHECKTEST(sp==NULL, "LZ5_createStreamHC() allocation failed"); - LZ5_freeStreamHC(sp); - - /* simple HC compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ5_resetStreamHC(&sHC, 0); - result = LZ5_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ5_compressHC_limitedOutput_continue() compression failed"); - - result = LZ5_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ5_decompress_safe() decompression failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ5_decompress_safe() decompression corruption"); - - /* simple dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ5_resetStreamHC(&sHC, 0); - LZ5_loadDictHC(&sHC, testInput, 64 KB); - result = LZ5_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ5_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ5_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ5_decompress_safe() simple dictionary decompression test failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ5_decompress_safe() simple dictionary decompression test : corruption"); - - /* multiple HC compression test with dictionary */ - { - int result1, result2; - int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ5_resetStreamHC(&sHC, 0); - LZ5_loadDictHC(&sHC, testInput, segSize); - result1 = LZ5_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); - FUZ_CHECKTEST(result1==0, "LZ5_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); - result2 = LZ5_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); - FUZ_CHECKTEST(result2==0, "LZ5_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); - - result = LZ5_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); - FUZ_CHECKTEST(result!=segSize, "LZ5_decompress_safe() dictionary decompression part 1 failed"); - result = LZ5_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); - FUZ_CHECKTEST(result!=segSize, "LZ5_decompress_safe() dictionary decompression part 2 failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ5_decompress_safe() dictionary decompression corruption"); - } - - /* remote dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ5_resetStreamHC(&sHC, 0); - LZ5_loadDictHC(&sHC, testInput, 32 KB); - result = LZ5_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ5_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); - - result = LZ5_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ5_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ5_decompress_safe_usingDict() decompression corruption"); - - /* multiple HC compression with ext. dictionary */ - { - XXH64_state_t crcOrigState; - XXH64_state_t crcNewState; - const char* dict = testInput + 3; - int dictSize = (FUZ_rand(&randState) & 8191); - char* dst = testVerify; - - size_t segStart = dictSize + 7; - int segSize = (FUZ_rand(&randState) & 8191); - int segNb = 1; - - LZ5_resetStreamHC(&sHC, 0); - LZ5_loadDictHC(&sHC, dict, dictSize); - - XXH64_reset(&crcOrigState, 0); - XXH64_reset(&crcNewState, 0); - - while (segStart + segSize < testInputSize) - { - XXH64_update(&crcOrigState, testInput + segStart, segSize); - crcOrig = XXH64_digest(&crcOrigState); - result = LZ5_compressHC_limitedOutput_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ5_compressBound(segSize)); - FUZ_CHECKTEST(result==0, "LZ5_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ5_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); - FUZ_CHECKTEST(result!=segSize, "LZ5_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); - XXH64_update(&crcNewState, dst, segSize); - crcNew = XXH64_digest(&crcNewState); - if (crcOrig!=crcNew) - { - size_t c=0; - while (dst[c] == testInput[segStart+c]) c++; - DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); - } - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ5_decompress_safe_usingDict() part %i corruption", segNb); - - dict = dst; - //dict = testInput + segStart; - dictSize = segSize; - - dst += segSize + 1; - segNb ++; - - segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; - segSize = (FUZ_rand(&randState) & 8191); - } - } - - /* ring buffer test */ - { - XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ5_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - - /* small decoder-side ring buffer test */ - { - XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ5_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 12; - const U32 maxMessageSizeMask = (1< dBufferSize) dNext = 0; - - while (totalMessageSize < 9 MB) - { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - result = LZ5_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ5_compressHC_limitedOutput_continue() compression failed"); - - result = LZ5_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ5_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - if (crcOrig != crcNew) - FUZ_findDiff(testInput + iNext, testVerify + dNext); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ5_decompress_safe() decompression corruption during small decoder-side ring buffer test"); - - /* prepare next message */ - dNext += messageSize; - totalMessageSize += messageSize; - messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; - iNext = (FUZ_rand(&randState) & 65535); - if (dNext > dBufferSize) dNext = 0; - } - } - } - - printf("All unit tests completed successfully \n"); - return; -_output_error: - exit(1); -} - - -static int FUZ_usage(char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); - DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); - DISPLAY( " -s# : Select seed (default:prompt user)\n"); - DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); - DISPLAY( " -v : verbose\n"); - DISPLAY( " -p : pause at the end\n"); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - U32 seed=0; - int seedset=0; - int argNb; - int nbTests = NB_ATTEMPTS; - int testNb = 0; - int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause = 0; - char* programName = argv[0]; - U32 duration = 0; - - /* Check command line */ - for(argNb=1; argNb='0') && (*argument<='9')) - { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests = 0; duration = 0; - for (;;) - { - switch(*argument) - { - case 'm': duration *= 60; argument++; continue; - case 's': - case 'n': argument++; continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; - } - break; - } - break; - - case 's': - argument++; - seed=0; seedset=1; - while ((*argument>='0') && (*argument<='9')) - { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - - case 't': /* select starting test nb */ - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) - { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - - case 'P': /* change probability */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) - { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - default: ; - } - } - } - } - - printf("Starting LZ5 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ5_VERSION); - - if (!seedset) seed = FUZ_GetMilliStart() % 10000; - printf("Seed = %u\n", seed); - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - - if ((seedset==0) && (testNb==0)) FUZ_unitTests(); - - if (nbTests<=0) nbTests=1; - - { - int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (pause) - { - DISPLAY("press enter ... \n"); - (void)getchar(); - } - return result; - } -} diff --git a/programs/lz5cli.c b/programs/lz5cli.c index 6d0bf1e..be6ee98 100644 --- a/programs/lz5cli.c +++ b/programs/lz5cli.c @@ -86,9 +86,6 @@ * Constants ******************************/ #define COMPRESSOR_NAME "LZ5 command line interface" -#ifndef LZ5_VERSION -# define LZ5_VERSION "r128" -#endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ5_VERSION, AUTHOR, __DATE__ #define LZ5_EXTENSION ".lz5" diff --git a/programs/lz5io.c b/programs/lz5io.c index 9aa35f9..6f15840 100644 --- a/programs/lz5io.c +++ b/programs/lz5io.c @@ -99,7 +99,7 @@ #define _8BITS 0xFF #define MAGICNUMBER_SIZE 4 -#define LZ5IO_MAGICNUMBER 0x184D2204 +#define LZ5IO_MAGICNUMBER 0x184D2205 #define LZ5IO_SKIPPABLE0 0x184D2A50 #define LZ5IO_SKIPPABLEMASK 0xFFFFFFF0 #define LEGACY_MAGICNUMBER 0x184C2102