From b120e88c1cdd0f42ec728033452e837c26e39faf Mon Sep 17 00:00:00 2001 From: Alec Miller Date: Tue, 4 Jun 2024 23:52:54 -0700 Subject: [PATCH] kram - use miniz with lodepng, add libCompression Having trouble using this to decode zlib in lodepng. miniz works fine. --- kramv/KramLoader.mm | 8 ++--- libkram/kram/KTXImage.cpp | 22 ++++++++++++- libkram/kram/Kram.cpp | 57 ++++++++++++++++++++++++++------- libkram/kram/KramMmapHelper.cpp | 8 +++-- libkram/kram/KramZipHelper.cpp | 8 +++-- libkram/kram/win_mmap.h | 1 + libkram/lodepng/lodepng.h | 5 +-- 7 files changed, 85 insertions(+), 24 deletions(-) diff --git a/kramv/KramLoader.mm b/kramv/KramLoader.mm index 8e227337..90006a09 100644 --- a/kramv/KramLoader.mm +++ b/kramv/KramLoader.mm @@ -362,7 +362,7 @@ - (void)createStagingBufffer:(uint64_t)dataSize // enough to upload 4k x 4k @ 4 bytes no mips, careful with array and cube // that get too big - // allocate system memory for bufffer, can memcopy to this + // allocate system memory for buffer, can memcpy to this posix_memalign((void **)&_data, getpagesize(), dataSize); // allocate memory for circular staging buffer, only need to memcpy to this @@ -429,7 +429,7 @@ - (void)uploadTexturesIfNeeded:(id)blitEncoder }]; } - // mipgen after possible initial blit above + // mipgen possible after initial blit above if (_mipgenTextures.count > 0) { for (id texture in _mipgenTextures) { // autogen mips will include srgb conversions, so toggling srgb on/off @@ -437,7 +437,7 @@ - (void)uploadTexturesIfNeeded:(id)blitEncoder [blitEncoder generateMipmapsForTexture:texture]; } - // reset the arra + // reset the array [_mipgenTextures removeAllObjects]; } } @@ -497,7 +497,7 @@ inline uint64_t alignOffset(uint64_t offset, uint64_t alignment) // upload mip levels // TODO: about aligning to 4k for base + length - // http://metalkit.org/2017/05/26/working-with-memory-in-metal-part-2.html + // http://metalkit.org/working-with-memory-in-metal/ uint32_t w = image.width; uint32_t h = image.height; diff --git a/libkram/kram/KTXImage.cpp b/libkram/kram/KTXImage.cpp index 363e9b04..aee22870 100644 --- a/libkram/kram/KTXImage.cpp +++ b/libkram/kram/KTXImage.cpp @@ -16,6 +16,15 @@ // for zstd decompress #include "zstd.h" +// Don't trust this see use in Kram.cpp with lodepng +#ifndef USE_LIBCOMPRESSION +#define USE_LIBCOMPRESSION 0 // (KRAM_MAC || KRAM_IOS) +#endif + +#if USE_LIBCOMPRESSION +#include +#endif + namespace kram { // These are props added into the KTX file props data. @@ -1869,13 +1878,24 @@ bool KTXImage::unpackLevel(uint32_t mipNumber, const uint8_t* srcData, uint8_t* case KTX2SupercompressionZlib: { // can use miniz or libCompression +#if USE_LIBCOMPRESSION + // TODO: see if this is faster + char scratchBuffer[compression_decode_scratch_buffer_size(COMPRESSION_ZLIB)]; + + size_t dstDataSizeMiniz = compression_decode_buffer( + (uint8_t*)dstData, dstDataSize, + (const uint8_t*)srcData, srcDataSize, + scratchBuffer, // scratch-buffer that could speed up to pass + COMPRESSION_ZLIB); +#else mz_ulong dstDataSizeMiniz = 0; if (mz_uncompress(dstData, &dstDataSizeMiniz, srcData, srcDataSize) != MZ_OK) { KLOGE("kram", "decode mip zlib failed"); return false; } - if (dstDataSizeMiniz != dstDataSize) { +#endif + if (dstDataSizeMiniz != dstDataSize) { KLOGE("kram", "decode mip zlib size not expected"); return false; } diff --git a/libkram/kram/Kram.cpp b/libkram/kram/Kram.cpp index f31e300b..cee9db7d 100644 --- a/libkram/kram/Kram.cpp +++ b/libkram/kram/Kram.cpp @@ -27,6 +27,16 @@ #include "lodepng.h" #include "miniz.h" +// This doesn't work returns 121 for a 16K decode +// Just open the src directory +#ifndef USE_LIBCOMPRESSION +#define USE_LIBCOMPRESSION 0 // (KRAM_MAC || KRAM_IOS) +#endif + +#if USE_LIBCOMPRESSION +#include +#endif + // one .cpp must supply these new overrides #if USE_EASTL void* __cdecl operator new[](size_t size, const char* name, int flags, unsigned debugFlags, const char* file, int line) @@ -52,9 +62,7 @@ namespace kram { using namespace NAMESPACE_STL; -// lodepng iccp decode is failing when setting this for some reason, find out why -// Must set it with LODEPNG_NO_COMPILE_ZLIB in lodepng.h if true -static bool useMiniZ = false; +static bool useMiniZ = true; template void releaseVector(vector& v) @@ -412,12 +420,36 @@ unsigned LodepngDecompressUsingMiniz( const LodePNGDecompressSettings* settings) { // mz_ulong doesn't line up with size_t on Windows, but does on macOS - mz_ulong dstDataSizeUL = *dstDataSize; - - int result = mz_uncompress(*dstData, &dstDataSizeUL, + KASSERT(*dstDataSize != 0); + +#if USE_LIBCOMPRESSION + // This call can't be replaced since lodepng doesn't pass size + // And it doesn't take a nullable dstData? + char scratchBuffer[compression_decode_scratch_buffer_size(COMPRESSION_ZLIB)]; + size_t bytesDecoded = compression_decode_buffer( + (uint8_t*)*dstData, *dstDataSize, + (const uint8_t*)srcData, srcDataSize, + scratchBuffer, // scratch-buffer that could speed up to pass + COMPRESSION_ZLIB); + + int result = MZ_OK; + if (bytesDecoded != *dstDataSize) { + result = MZ_DATA_ERROR; + *dstDataSize = 0; + } +#else + // This works. + mz_ulong bytesDecoded = *dstDataSize; + int result = mz_uncompress(*dstData, &bytesDecoded, srcData, srcDataSize); - - *dstDataSize = dstDataSizeUL; + + if (result != MZ_OK || bytesDecoded != *dstDataSize) { + *dstDataSize = 0; + } + else { + *dstDataSize = bytesDecoded; + } +#endif return result; } @@ -428,14 +460,17 @@ unsigned LodepngCompressUsingMiniz( const unsigned char* srcData, size_t srcDataSize, const LodePNGCompressSettings* settings) { + // TODO: no setting for compression level in settings? + // TODO: libCompression can only encode zlib to quality 5 + // mz_ulong doesn't line up with size_t on Windows, but does on macOS mz_ulong dstDataSizeUL = *dstDataSize; - int result = mz_compress(*dstData, &dstDataSizeUL, - srcData, srcDataSize); + int result = mz_compress2(*dstData, &dstDataSizeUL, + srcData, srcDataSize, MZ_DEFAULT_COMPRESSION); *dstDataSize = dstDataSizeUL; - + return result; } diff --git a/libkram/kram/KramMmapHelper.cpp b/libkram/kram/KramMmapHelper.cpp index fa3b4a4b..63c4b55b 100644 --- a/libkram/kram/KramMmapHelper.cpp +++ b/libkram/kram/KramMmapHelper.cpp @@ -4,7 +4,9 @@ #include "KramMmapHelper.h" -// here's how to mmmap data, but NSData may have another way +// here's how to mmmap data, but NSData has another way +// +dataWithContentsOfURL:options:error: and NSDataReadingMappedIfSafe or NSDataReadingMappedAlways." + #include #include @@ -12,8 +14,6 @@ #include #include #elif KRAM_WIN -// portable mmap implementation, but only using on Win -// TODO: this indicates that it leaks a CreateFileMapping handle, since it wanted to keep same mmap/munmap api #include "win_mmap.h" #endif @@ -49,6 +49,8 @@ bool MmapHelper::open(const char *filename) } length = sb.st_size; + // Only offset needs padded to pagesize, but here offset is always 0 + // Stop padding out to page size, or do but then don't add to length, or will walk too far in memory // all remaining page data will be zero, but still want length to reflect actual length of file // need Windows equilvent of getpagesize() call before putting this back. This was to use diff --git a/libkram/kram/KramZipHelper.cpp b/libkram/kram/KramZipHelper.cpp index d91b6a3f..bb2a427a 100644 --- a/libkram/kram/KramZipHelper.cpp +++ b/libkram/kram/KramZipHelper.cpp @@ -9,8 +9,9 @@ // test for perf of this compared to one in miniz also see // comments about faster algs. // libcompress can only encode lvl 5, but here it's only decompress. +// Don't trust this. #ifndef USE_LIBCOMPRESSION -#define USE_LIBCOMPRESSION (KRAM_MAC || KRAM_IOS) +#define USE_LIBCOMPRESSION 0 // (KRAM_MAC || KRAM_IOS) #endif #if USE_LIBCOMPRESSION @@ -229,12 +230,13 @@ bool ZipHelper::extract(const ZipEntry& entry, void* buffer, uint64_t bufferSize if (!data) { return false; } - // need to extra data and header + // need to extract data and header + char scratchBuffer[compression_decode_scratch_buffer_size(COMPRESSION_ZLIB)]; uint64_t bytesDecoded = compression_decode_buffer( (uint8_t*)buffer, entry.uncompressedSize, (const uint8_t*)data, entry.compressedSize, - NULL, // scratch-buffer that could speed up to pass + scratchBuffer, // scratch-buffer that could speed up to pass COMPRESSION_ZLIB); bool success = false; diff --git a/libkram/kram/win_mmap.h b/libkram/kram/win_mmap.h index 03a6db8e..fc8cd26c 100644 --- a/libkram/kram/win_mmap.h +++ b/libkram/kram/win_mmap.h @@ -30,6 +30,7 @@ #define MAP_ANON 0x20 #define MAP_FAILED ((void *) -1) +// off_t is 32-bit, which isn't great using myoff_t = int64_t; // TODO: find out which path this takes, want 64-bit mmsp diff --git a/libkram/lodepng/lodepng.h b/libkram/lodepng/lodepng.h index 524bca4f..2ae8a4ec 100644 --- a/libkram/lodepng/lodepng.h +++ b/libkram/lodepng/lodepng.h @@ -37,8 +37,9 @@ extern const char* LODEPNG_VERSION_STRING; // don't need io #define LODEPNG_NO_COMPILE_DISK -// using miniz now, but this was failing using miniz so switched off -//#define LODEPNG_NO_COMPILE_ZLIB +// using miniz or libCompression now, but this was failing using miniz so switched off +// This disables the code that inits the array for zlib_decompress. +// #define LODEPNG_NO_COMPILE_ZLIB // was not doing png encodes, but to strip blocks now need to #define LODEPNG_COMPILE_ENCODER