From 5ae58e40e3552ca57ecda3c22e2e2654513def2d Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 31 Aug 2023 11:09:08 -0400 Subject: [PATCH] lint --- kerchunk/hdf.py | 44 +++++++++++++++++++++++--------------- kerchunk/tests/gen_hdf5.py | 24 ++++++++++----------- kerchunk/tests/test_hdf.py | 3 ++- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/kerchunk/hdf.py b/kerchunk/hdf.py index 35470d3a..23940ad0 100644 --- a/kerchunk/hdf.py +++ b/kerchunk/hdf.py @@ -189,40 +189,50 @@ def _transfer_attrs( f"TypeError transferring attr, skipping:\n {n}@{h5obj.name} = {v} ({type(v)})" ) - def _decode_blosc(properties): # 32001 - blosc_compressors = ('blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd') + def _decode_blosc(properties): # 32001 + blosc_compressors = ("blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd") _1, _2, bytes_per_num, total_bytes, clevel, shuffle, compressor = properties - return dict (id="blosc", blocksize=total_bytes, clevel=clevel, shuffle=shuffle, cname=blosc_compressors[compressor],) - - def _decode_zstd(properties): #32015 - return dict (id='zstd', level=properties[0],) - - - decoders = { "32001" : _decode_blosc, - "32015" : _decode_zstd, - } + return dict( + id="blosc", + blocksize=total_bytes, + clevel=clevel, + shuffle=shuffle, + cname=blosc_compressors[compressor], + ) + + def _decode_zstd(properties): # 32015 + return dict( + id="zstd", + level=properties[0], + ) + + decoders = { + "32001": _decode_blosc, + "32015": _decode_zstd, + } def _decode_filters(self, h5obj: Union[h5py.Dataset, h5py.Group]): if len(h5obj._filters.keys()) > 1: raise RuntimeError( - f"{h5obj.name} uses multiple filters {list (h5obj._filters.keys())}. This is not supported by kerchunk." - ) + f"{h5obj.name} uses multiple filters {list (h5obj._filters.keys())}." + f" This is not supported by kerchunk." + ) for filter_id, properties in h5obj._filters.items(): if not str(filter_id) in self.decoders.keys(): raise RuntimeError( - f"{h5obj.name} uses filter id {filter_id} with properties {properties}, not supported by kerchunk., supported filters are {self.decoders.keys()}" - ) + f"{h5obj.name} uses filter id {filter_id} with properties {properties}," + f" not supported by kerchunk., supported filters are {self.decoders.keys()}" + ) else: return numcodecs.get_codec(self.decoders[filter_id](properties)) - def _translator(self, name: str, h5obj: Union[h5py.Dataset, h5py.Group]): """Produce Zarr metadata for all groups and datasets in the HDF5 file.""" try: # method must not raise exception kwargs = {} if isinstance(h5obj, h5py.Dataset): lggr.debug(f"HDF5 dataset: {h5obj.name}") - lggr.debug (f"HDF5 compression: {h5obj.compression}") + lggr.debug(f"HDF5 compression: {h5obj.compression}") if h5obj.id.get_create_plist().get_layout() == h5py.h5d.COMPACT: # Only do if h5obj.nbytes < self.inline?? kwargs["data"] = h5obj[:] diff --git a/kerchunk/tests/gen_hdf5.py b/kerchunk/tests/gen_hdf5.py index ca76120d..03c73bbb 100644 --- a/kerchunk/tests/gen_hdf5.py +++ b/kerchunk/tests/gen_hdf5.py @@ -2,18 +2,16 @@ import h5py import hdf5plugin -compressors = dict ( - zstd = hdf5plugin.Zstd(), - bitshuffle = hdf5plugin.Bitshuffle(nelems=0, cname='lz4'), - lz4 = hdf5plugin.LZ4(nbytes=0), - blosc_lz4_bitshuffle = hdf5plugin.Blosc(cname='blosclz', clevel=9, shuffle=hdf5plugin.Blosc.BITSHUFFLE) - ) +compressors = dict( + zstd=hdf5plugin.Zstd(), + bitshuffle=hdf5plugin.Bitshuffle(nelems=0, cname="lz4"), + lz4=hdf5plugin.LZ4(nbytes=0), + blosc_lz4_bitshuffle=hdf5plugin.Blosc( + cname="blosclz", clevel=9, shuffle=hdf5plugin.Blosc.BITSHUFFLE + ), +) -for c in compressors : - f = h5py.File(f'hdf5_compression_{c}.h5', 'w') - f.create_dataset( - 'data', - data=numpy.arange(100), - **compressors[c] - ) +for c in compressors: + f = h5py.File(f"hdf5_compression_{c}.h5", "w") + f.create_dataset("data", data=numpy.arange(100), **compressors[c]) f.close() diff --git a/kerchunk/tests/test_hdf.py b/kerchunk/tests/test_hdf.py index 99569529..b37e40b7 100644 --- a/kerchunk/tests/test_hdf.py +++ b/kerchunk/tests/test_hdf.py @@ -290,7 +290,8 @@ def test_compact(): def test_compress(): import glob - files = glob.glob (osp.join(here, "hdf5_compression_*.h5")) + + files = glob.glob(osp.join(here, "hdf5_compression_*.h5")) for f in files: h = kerchunk.hdf.SingleHdf5ToZarr(f) out = h.translate()