Merge branch 'main' into condalock-update

NCAS-CMS · Feb 27, 2024 · d56be8a · d56be8a
2 parents c2f8c75 + babdba7
commit d56be8a
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/activestorage/netcdf_to_zarr.py b/activestorage/netcdf_to_zarr.py
@@ -18,7 +18,7 @@ def gen_json(file_url, varname, outf, storage_type, storage_options):
                                secret=S3_SECRET_KEY,
                                client_kwargs={'endpoint_url': S3_URL},
                                default_fill_cache=False,
-                               default_cache_type="none"
+                               default_cache_type="first"  # best for HDF5
         )
         fs2 = fsspec.filesystem('')
         with fs.open(file_url, 'rb') as s3file:
@@ -32,7 +32,7 @@ def gen_json(file_url, varname, outf, storage_type, storage_options):
     elif storage_type == "s3" and storage_options is not None:
         storage_options = storage_options.copy()
         storage_options['default_fill_cache'] = False
-        storage_options['default_cache_type'] = "none"
+        storage_options['default_cache_type'] = "first"  # best for HDF5
         fs = s3fs.S3FileSystem(**storage_options)
         fs2 = fsspec.filesystem('')
         with fs.open(file_url, 'rb') as s3file: