Added tests for compression filters, also failing ones.

fsspec · Aug 29, 2023 · d5c42e9 · d5c42e9
1 parent d3f8419
commit d5c42e9
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 0 deletions.
diff --git a/kerchunk/tests/gen_hdf5.py b/kerchunk/tests/gen_hdf5.py
@@ -0,0 +1,19 @@
+import numpy
+import h5py
+import hdf5plugin
+
+compressors = dict (
+    zstd = hdf5plugin.Zstd(),
+    bitshuffle = hdf5plugin.Bitshuffle(nelems=0, cname='lz4'),
+    lz4 = hdf5plugin.LZ4(nbytes=0),
+    blosc_lz4_bitshuffle = hdf5plugin.Blosc(cname='blosclz', clevel=9, shuffle=hdf5plugin.Blosc.BITSHUFFLE)
+    )
+
+for c in compressors :
+    f = h5py.File(f'hdf5_compression_{c}.h5', 'w')
+    f.create_dataset(
+        'data',
+        data=numpy.arange(100),
+        **compressors[c]
+    )
+    f.close()
diff --git a/kerchunk/tests/hdf5_compression_bitshuffle.h5 b/kerchunk/tests/hdf5_compression_bitshuffle.h5
diff --git a/kerchunk/tests/hdf5_compression_blosc_lz4_bitshuffle.h5 b/kerchunk/tests/hdf5_compression_blosc_lz4_bitshuffle.h5
diff --git a/kerchunk/tests/hdf5_compression_lz4.h5 b/kerchunk/tests/hdf5_compression_lz4.h5
diff --git a/kerchunk/tests/hdf5_compression_zstd.h5 b/kerchunk/tests/hdf5_compression_zstd.h5
diff --git a/kerchunk/tests/test_hdf.py b/kerchunk/tests/test_hdf.py
@@ -288,6 +288,17 @@ def test_compact():
     assert np.allclose(g.ancillary_data.atlas_sdp_gps_epoch[:], 1.19880002e09)
 
 
+def test_compress():
+    import glob
+    files = glob.glob (osp.join(here, "hdf5_compression_*.h5"))
+    for f in files:
+        h = kerchunk.hdf.SingleHdf5ToZarr(f)
+        out = h.translate()
+        m = fsspec.get_mapper("reference://", fo=out)
+        g = zarr.open(m)
+        assert np.mean(g.data) == 49.5
+
+
 def test_embed():
     fn = osp.join(here, "NEONDSTowerTemperatureData.hdf5")
     h = kerchunk.hdf.SingleHdf5ToZarr(fn, vlen_encode="embed")