Add unlim dims for h5netcdf (#1637)

* initial commit adding unlim dims for h5netcdf

* h5netcdf unlimited dimension now roundtrips

Author: Joe Hamman

doc/whats-new.rst

@@ -186,6 +186,9 @@ Enhancements
   functions on data stored as dask arrays (:issue:`1279`).
   By `Joe Hamman <https://github.com/jhamman>`_.
 
+- Support reading and writing unlimited dimensions with h5netcdf (:issue:`1636`).
+  By `Joe Hamman <https://github.com/jhamman>`_.
+
 Bug fixes
 ~~~~~~~~~
 
@@ -260,7 +263,7 @@ Bug fixes
   The previous behavior unintentionally causing additional tests to be skipped
   (:issue:`1531`). By `Joe Hamman <https://github.com/jhamman>`_.
 
-- Fix pynio backend for upcoming release of pynio with python3 support 
+- Fix pynio backend for upcoming release of pynio with python3 support
   (:issue:`1611`). By `Ben Hillman <https://github/brhillman>`_.
 
 .. _whats-new.0.9.6:

xarray/backends/common.py

@@ -232,9 +232,8 @@ def set_necessary_dimensions(self, variable, unlimited_dims=None):
             unlimited_dims = set()
         for d, l in zip(variable.dims, variable.shape):
             if d not in self.dimensions:
-                if d in unlimited_dims:
-                    l = None
-                self.set_dimension(d, l)
+                is_unlimited = d in unlimited_dims
+                self.set_dimension(d, l, is_unlimited)
 
 
 class WritableCFDataStore(AbstractWritableDataStore):

xarray/backends/h5netcdf_.py

@@ -2,7 +2,6 @@
 from __future__ import division
 from __future__ import print_function
 import functools
-import warnings
 
 from .. import Variable
 from ..core import indexing
@@ -106,9 +105,20 @@ def get_dimensions(self):
         with self.ensure_open(autoclose=True):
             return self.ds.dimensions
 
-    def set_dimension(self, name, length):
+    def get_encoding(self):
+        with self.ensure_open(autoclose=True):
+            encoding = {}
+            encoding['unlimited_dims'] = {
+                k for k, v in self.ds.dimensions.items() if v is None}
+        return encoding
+
+    def set_dimension(self, name, length, is_unlimited=False):
         with self.ensure_open(autoclose=False):
-            self.ds.createDimension(name, size=length)
+            if is_unlimited:
+                self.ds.createDimension(name, size=None)
+                self.ds.resize_dimension(name, length)
+            else:
+                self.ds.createDimension(name, size=length)
 
     def set_attribute(self, key, value):
         with self.ensure_open(autoclose=False):
@@ -123,10 +133,6 @@ def prepare_variable(self, name, variable, check_encoding=False,
         if dtype is str:
             dtype = h5py.special_dtype(vlen=unicode_type)
 
-        if unlimited_dims is not None:
-            warnings.warn('h5netcdf does not support unlimited dimensions, '
-                          'got: %s.' % unlimited_dims)
-            unlimited_dims = None
         self.set_necessary_dimensions(variable, unlimited_dims=unlimited_dims)
 
         fill_value = attrs.pop('_FillValue', None)
@@ -141,7 +147,6 @@ def prepare_variable(self, name, variable, check_encoding=False,
                     'chunksizes', 'fletcher32']:
             if key in encoding:
                 kwargs[key] = encoding[key]
-
         nc4_var = self.ds.createVariable(name, dtype, variable.dims,
                                          fill_value=fill_value, **kwargs)
 

xarray/backends/netCDF4_.py

@@ -294,9 +294,10 @@ def get_encoding(self):
                 k for k, v in self.ds.dimensions.items() if v.isunlimited()}
         return encoding
 
-    def set_dimension(self, name, length):
+    def set_dimension(self, name, length, is_unlimited=False):
         with self.ensure_open(autoclose=False):
-            self.ds.createDimension(name, size=length)
+            dim_length = length if not is_unlimited else None
+            self.ds.createDimension(name, size=dim_length)
 
     def set_attribute(self, key, value):
         with self.ensure_open(autoclose=False):

xarray/backends/scipy_.py

@@ -164,12 +164,13 @@ def get_encoding(self):
             k for k, v in self.ds.dimensions.items() if v is None}
         return encoding
 
-    def set_dimension(self, name, length):
+    def set_dimension(self, name, length, is_unlimited=False):
         with self.ensure_open(autoclose=False):
             if name in self.dimensions:
                 raise ValueError('%s does not support modifying dimensions'
                                  % type(self).__name__)
-            self.ds.createDimension(name, length)
+            dim_length = length if not is_unlimited else None
+            self.ds.createDimension(name, dim_length)
 
     def _validate_attr_key(self, key):
         if not is_valid_nc3_name(key):

xarray/tests/test_backends.py

@@ -1171,12 +1171,14 @@ def test_read_byte_attrs_as_unicode(self):
 
     def test_encoding_unlimited_dims(self):
         ds = Dataset({'x': ('y', np.arange(10.0))})
+        with self.roundtrip(ds,
+                            save_kwargs=dict(unlimited_dims=['y'])) as actual:
+            self.assertEqual(actual.encoding['unlimited_dims'], set('y'))
+            self.assertDatasetEqual(ds, actual)
         ds.encoding = {'unlimited_dims': ['y']}
-        with create_tmp_file() as tmp_file:
-            with pytest.warns(UserWarning):
-                ds.to_netcdf(tmp_file, engine='h5netcdf')
-            with pytest.warns(UserWarning):
-                ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y'])
+        with self.roundtrip(ds) as actual:
+            self.assertEqual(actual.encoding['unlimited_dims'], set('y'))
+            self.assertDatasetEqual(ds, actual)
 
 
 # tests pending h5netcdf fix
@@ -1343,7 +1345,7 @@ def test_common_coord_when_datavars_all(self):
                 self.assertEqual(var_shape, coord_shape)
                 self.assertNotEqual(coord_shape1, coord_shape)
                 self.assertNotEqual(coord_shape2, coord_shape)
-      
+
     def test_common_coord_when_datavars_minimal(self):
         opt = 'minimal'