kram - use miniz with lodepng, add libCompression

Having trouble using this to decode zlib in lodepng.  miniz works fine.

kramv/KramLoader.mm

@@ -362,7 +362,7 @@ - (void)createStagingBufffer:(uint64_t)dataSize
     // enough to upload 4k x 4k @ 4 bytes no mips, careful with array and cube
     // that get too big
 
-    // allocate system memory for bufffer, can memcopy to this
+    // allocate system memory for buffer, can memcpy to this
     posix_memalign((void **)&_data, getpagesize(), dataSize);
 
     // allocate memory for circular staging buffer, only need to memcpy to this
@@ -429,15 +429,15 @@ - (void)uploadTexturesIfNeeded:(id<MTLBlitCommandEncoder>)blitEncoder
         }];
     }
 
-    // mipgen after possible initial blit above
+    // mipgen possible after initial blit above
     if (_mipgenTextures.count > 0) {
         for (id<MTLTexture> texture in _mipgenTextures) {
             // autogen mips will include srgb conversions, so toggling srgb on/off
             // isn't quite correct
             [blitEncoder generateMipmapsForTexture:texture];
         }
 
-        // reset the arra
+        // reset the array
         [_mipgenTextures removeAllObjects];
     }
 }
@@ -497,7 +497,7 @@ inline uint64_t alignOffset(uint64_t offset, uint64_t alignment)
     // upload mip levels
 
     // TODO: about aligning to 4k for base + length
-    // http://metalkit.org/2017/05/26/working-with-memory-in-metal-part-2.html
+    // http://metalkit.org/working-with-memory-in-metal/
 
     uint32_t w = image.width;
     uint32_t h = image.height;

libkram/kram/KTXImage.cpp

@@ -16,6 +16,15 @@
 // for zstd decompress
 #include "zstd.h"
 
+// Don't trust this see use in Kram.cpp with lodepng
+#ifndef USE_LIBCOMPRESSION
+#define USE_LIBCOMPRESSION 0 // (KRAM_MAC || KRAM_IOS)
+#endif
+
+#if USE_LIBCOMPRESSION
+#include <compression.h>
+#endif
+
 namespace kram {
 
 // These are props added into the KTX file props data.
@@ -1869,13 +1878,24 @@ bool KTXImage::unpackLevel(uint32_t mipNumber, const uint8_t* srcData, uint8_t*
 
             case KTX2SupercompressionZlib: {
                 // can use miniz or libCompression
+#if USE_LIBCOMPRESSION
+                // TODO: see if this is faster
+                char scratchBuffer[compression_decode_scratch_buffer_size(COMPRESSION_ZLIB)];
+
+                size_t dstDataSizeMiniz = compression_decode_buffer(
+                    (uint8_t*)dstData, dstDataSize,
+                    (const uint8_t*)srcData, srcDataSize,
+                    scratchBuffer, // scratch-buffer that could speed up to pass
+                    COMPRESSION_ZLIB);
+#else
                 mz_ulong dstDataSizeMiniz = 0;
                 if (mz_uncompress(dstData, &dstDataSizeMiniz,
                                   srcData, srcDataSize) != MZ_OK) {
                     KLOGE("kram", "decode mip zlib failed");
                     return false;
                 }
-                if (dstDataSizeMiniz != dstDataSize) {
+#endif
+              if (dstDataSizeMiniz != dstDataSize) {
                     KLOGE("kram", "decode mip zlib size not expected");
                     return false;
                 }

libkram/kram/Kram.cpp

@@ -27,6 +27,16 @@
 #include "lodepng.h"
 #include "miniz.h"
 
+// This doesn't work returns 121 for a 16K decode
+// Just open the src directory
+#ifndef USE_LIBCOMPRESSION
+#define USE_LIBCOMPRESSION 0 // (KRAM_MAC || KRAM_IOS)
+#endif
+
+#if USE_LIBCOMPRESSION
+#include <compression.h>
+#endif
+
 // one .cpp must supply these new overrides
 #if USE_EASTL
 void* __cdecl operator new[](size_t size, const char* name, int flags, unsigned debugFlags, const char* file, int line)
@@ -52,9 +62,7 @@ namespace kram {
 
 using namespace NAMESPACE_STL;
 
-// lodepng iccp decode is failing when setting this for some reason, find out why
-// Must set it with LODEPNG_NO_COMPILE_ZLIB in lodepng.h if true
-static bool useMiniZ = false;
+static bool useMiniZ = true;
 
 template <typename T>
 void releaseVector(vector<T>& v)
@@ -412,12 +420,36 @@ unsigned LodepngDecompressUsingMiniz(
     const LodePNGDecompressSettings* settings)
 {
     // mz_ulong doesn't line up with size_t on Windows, but does on macOS
-    mz_ulong dstDataSizeUL = *dstDataSize;
-
-    int result = mz_uncompress(*dstData, &dstDataSizeUL,
+    KASSERT(*dstDataSize != 0);
+
+#if USE_LIBCOMPRESSION
+    // This call can't be replaced since lodepng doesn't pass size
+    // And it doesn't take a nullable dstData?
+    char scratchBuffer[compression_decode_scratch_buffer_size(COMPRESSION_ZLIB)];
+    size_t bytesDecoded = compression_decode_buffer(
+         (uint8_t*)*dstData, *dstDataSize,
+         (const uint8_t*)srcData, srcDataSize,
+        scratchBuffer, // scratch-buffer that could speed up to pass
+         COMPRESSION_ZLIB);
+
+    int result = MZ_OK;
+    if (bytesDecoded != *dstDataSize) {
+        result = MZ_DATA_ERROR;
+        *dstDataSize = 0;
+    }
+#else
+    // This works.
+    mz_ulong bytesDecoded = *dstDataSize;
+    int result = mz_uncompress(*dstData, &bytesDecoded,
                                srcData, srcDataSize);
-
-    *dstDataSize = dstDataSizeUL;
+
+    if (result != MZ_OK || bytesDecoded != *dstDataSize) {
+        *dstDataSize = 0;
+    }
+    else {
+        *dstDataSize = bytesDecoded;
+    }
+#endif
 
     return result;
 }
@@ -428,14 +460,17 @@ unsigned LodepngCompressUsingMiniz(
     const unsigned char* srcData, size_t srcDataSize,
     const LodePNGCompressSettings* settings)
 {
+    // TODO: no setting for compression level in settings?
+    // TODO: libCompression can only encode zlib to quality 5
+
     // mz_ulong doesn't line up with size_t on Windows, but does on macOS
     mz_ulong dstDataSizeUL = *dstDataSize;
 
-    int result = mz_compress(*dstData, &dstDataSizeUL,
-                               srcData, srcDataSize);
+    int result = mz_compress2(*dstData, &dstDataSizeUL,
+                               srcData, srcDataSize, MZ_DEFAULT_COMPRESSION);
 
     *dstDataSize = dstDataSizeUL;
-
+    
     return result;
 }
 

libkram/kram/KramMmapHelper.cpp

@@ -4,16 +4,16 @@
 
 #include "KramMmapHelper.h"
 
-// here's how to mmmap data, but NSData may have another way
+// here's how to mmmap data, but NSData has another way
+// +dataWithContentsOfURL:options:error: and NSDataReadingMappedIfSafe or NSDataReadingMappedAlways."
+
 #include <stdio.h>
 #include <sys/stat.h>
 
 #if KRAM_MAC || KRAM_IOS || KRAM_LINUX
 #include <sys/mman.h>
 #include <unistd.h>
 #elif KRAM_WIN
-// portable mmap implementation, but only using on Win
-// TODO: this indicates that it leaks a CreateFileMapping handle, since it wanted to keep same mmap/munmap api
 #include "win_mmap.h"
 #endif
 
@@ -49,6 +49,8 @@ bool MmapHelper::open(const char *filename)
     }
     length = sb.st_size;
 
+    // Only offset needs padded to pagesize, but here offset is always 0
+
     // Stop padding out to page size, or do but then don't add to length, or will walk too far in memory
     // all remaining page data will be zero, but still want length to reflect actual length of file
     // need Windows equilvent of getpagesize() call before putting this back.  This was to use

libkram/kram/KramZipHelper.cpp

@@ -9,8 +9,9 @@
 // test for perf of this compared to one in miniz also see 
 // comments about faster algs.
 // libcompress can only encode lvl 5, but here it's only decompress.
+// Don't trust this.
 #ifndef USE_LIBCOMPRESSION
-#define USE_LIBCOMPRESSION (KRAM_MAC || KRAM_IOS)
+#define USE_LIBCOMPRESSION 0 // (KRAM_MAC || KRAM_IOS)
 #endif
 
 #if USE_LIBCOMPRESSION
@@ -229,12 +230,13 @@ bool ZipHelper::extract(const ZipEntry& entry, void* buffer, uint64_t bufferSize
     if (!data) {
         return false;
     }
-    // need to extra data and header
+    // need to extract data and header
+    char scratchBuffer[compression_decode_scratch_buffer_size(COMPRESSION_ZLIB)];
 
     uint64_t bytesDecoded = compression_decode_buffer(
         (uint8_t*)buffer, entry.uncompressedSize,
         (const uint8_t*)data, entry.compressedSize,
-        NULL, // scratch-buffer that could speed up to pass
+        scratchBuffer, // scratch-buffer that could speed up to pass
         COMPRESSION_ZLIB);
 
     bool success = false;

libkram/kram/win_mmap.h

@@ -30,6 +30,7 @@
 #define MAP_ANON      0x20
 #define MAP_FAILED    ((void *) -1)
 
+// off_t is 32-bit, which isn't great
 using myoff_t = int64_t;
 
 // TODO: find out which path this takes, want 64-bit mmsp

libkram/lodepng/lodepng.h

@@ -37,8 +37,9 @@ extern const char* LODEPNG_VERSION_STRING;
 // don't need io
 #define LODEPNG_NO_COMPILE_DISK
 
-// using miniz now, but this was failing using miniz so switched off
-//#define LODEPNG_NO_COMPILE_ZLIB
+// using miniz or libCompression now, but this was failing using miniz so switched off
+// This disables the code that inits the array for zlib_decompress.
+// #define LODEPNG_NO_COMPILE_ZLIB
 
 // was not doing png encodes, but to strip blocks now need to
 #define LODEPNG_COMPILE_ENCODER