document netcdf-format issues, intersphinx h5netcdf, adapt docstrings, add to whats-new.rst

Author: kmuehlbauer

doc/conf.py

@@ -333,6 +333,7 @@
     "cftime": ("https://unidata.github.io/cftime", None),
     "rasterio": ("https://rasterio.readthedocs.io/en/latest", None),
     "sparse": ("https://sparse.pydata.org/en/latest/", None),
+    "h5netcdf": ("https://h5netcdf.org/", None),
 }
 
 

doc/user-guide/io.rst

@@ -37,16 +37,16 @@ netCDF are based on the even more widely used HDF5 file-format.
 
 __ https://www.unidata.ucar.edu/software/netcdf/
 
+Reading and writing netCDF files with xarray requires :std:doc:`scipy <scipy:index>`, `netCDF4-Python`__ or :std:doc:`h5netcdf <h5netcdf:index>` libraries to be installed (the two latter are able
+to read/write netCDF V4 files and use the compression options described below).
+
 .. tip::
 
     If you aren't familiar with this data format, the `netCDF FAQ`_ is a good
-    place to start.
+    place to start. A comprehensive documentation is available in the Unidata NetCDF Users Guide's `The netCDF File Format`_ section.
 
 .. _netCDF FAQ: https://www.unidata.ucar.edu/software/netcdf/docs/faq.html#What-Is-netCDF
-
-Reading and writing netCDF files with xarray requires scipy or the
-`netCDF4-Python`__ library to be installed (the latter is required to
-read/write netCDF V4 files and use the compression options described below).
+.. _The netCDF File Format: https://docs.unidata.ucar.edu/nug/current/netcdf_introduction.html#netcdf_format
 
 __ https://github.com/Unidata/netcdf4-python
 
@@ -70,6 +70,15 @@ By default, the file is saved as netCDF4 (assuming netCDF4-Python is
 installed). You can control the format and engine used to write the file with
 the ``format`` and ``engine`` arguments.
 
+.. warning::
+
+  There are several `Limitations of NetCDF`_ which can have influence on `Large File Support`_, `Data Types`_, `Dealing with strings`_ or the like depending on the requested ``format`` (see :py:meth:`Dataset.to_netcdf` for details). In general, using ``netcdf4``/``h5netcdf`` engine with ``format="NETCDF4"`` (default) is recommended.
+
+.. _Limitations of NetCDF: https://docs.unidata.ucar.edu/nug/current/netcdf_introduction.html#limitations
+.. _Large File Support: https://docs.unidata.ucar.edu/nug/current/file_structure_and_performance.html#large_file_support
+.. _Data Types: https://docs.unidata.ucar.edu/nug/current/md_types.html
+.. _Dealing with strings: https://unidata.github.io/netcdf4-python/#dealing-with-strings
+
 .. tip::
 
    Using the `h5netcdf <https://github.com/h5netcdf/h5netcdf>`_  package
@@ -540,8 +549,7 @@ Invalid netCDF files
 ~~~~~~~~~~~~~~~~~~~~
 
 The library ``h5netcdf`` allows writing some dtypes (booleans, complex, ...) that aren't
-allowed in netCDF4 (see
-`h5netcdf documentation <https://github.com/shoyer/h5netcdf#invalid-netcdf-files>`_).
+allowed in netCDF4 (see :std:ref:`h5netcdf:invalid netcdf`).
 This feature is available through :py:meth:`DataArray.to_netcdf` and
 :py:meth:`Dataset.to_netcdf` when used with ``engine="h5netcdf"``
 and currently raises a warning unless ``invalid_netcdf=True`` is set:
@@ -923,6 +931,7 @@ installed, xarray can convert a ``DataArray`` into a ``Cube`` using
 :py:meth:`DataArray.to_iris`:
 
 .. ipython:: python
+    :okexcept:
 
     da = xr.DataArray(
         np.random.rand(4, 5),
@@ -937,6 +946,7 @@ Conversely, we can create a new ``DataArray`` object from a ``Cube`` using
 :py:meth:`DataArray.from_iris`:
 
 .. ipython:: python
+    :okexcept:
 
     da_cube = xr.DataArray.from_iris(cube)
     da_cube
@@ -1165,7 +1175,7 @@ rasterio is installed. Here is an example of how to use
 
         Deprecated in favor of rioxarray.
         For information about transitioning, see:
-        `rioxarray getting started docs<https://corteva.github.io/rioxarray/stable/getting_started/getting_started.html>``
+        `rioxarray getting started docs <https://corteva.github.io/rioxarray/stable/getting_started/getting_started.html>`_
 
 .. ipython::
     :verbatim:

doc/whats-new.rst

@@ -37,6 +37,9 @@ Bug fixes
 
 - Fix :py:meth:`xr.polyval` with non-system standard integer coeffs (:pull:`7619`).
   By `Shreyal Gupta <https://github.com/Ravenin7>`_ and `Michael Niklas <https://github.com/headtr1ck>`_.
+- Preserve dtypes (bool, vlen string) within encoding (:pull:`7654`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
+
 
 Documentation
 ~~~~~~~~~~~~~
@@ -6178,7 +6181,7 @@ Enhancements
   The default value for the ``display_width`` option is 80.
 
 .. _h5py: http://www.h5py.org/
-.. _h5netcdf: https://github.com/shoyer/h5netcdf
+.. _h5netcdf: https://github.com/h5netcdf/h5netcdf
 
 Deprecations
 ~~~~~~~~~~~~

xarray/backends/h5netcdf_.py

@@ -351,12 +351,12 @@ class H5netcdfBackendEntrypoint(BackendEntrypoint):
     selected as the default if the "netcdf4" engine is not available.
 
     Additionally it can open valid HDF5 files, see
-    https://h5netcdf.org/#invalid-netcdf-files for more info.
+    :std:ref:`h5netcdf:invalid netcdf` for more info.
     It will not be detected as valid backend for such files, so make
     sure to specify ``engine="h5netcdf"`` in ``open_dataset``.
 
     For more information about the underlying library, visit:
-    https://h5netcdf.org
+    :std:doc:`h5netcdf:index`
 
     See Also
     --------

xarray/backends/netCDF4_.py

@@ -521,7 +521,7 @@ class NetCDF4BackendEntrypoint(BackendEntrypoint):
     as default for these files.
 
     Additionally it can open valid HDF5 files, see
-    https://h5netcdf.org/#invalid-netcdf-files for more info.
+    :std:ref:`h5netcdf:invalid netcdf` for more info.
     It will not be detected as valid backend for such files, so make
     sure to specify ``engine="netcdf4"`` in ``open_dataset``.
 

xarray/core/dataarray.py

@@ -3914,7 +3914,7 @@ def to_netcdf(
         invalid_netcdf: bool, default: False
             Only valid along with ``engine="h5netcdf"``. If True, allow writing
             hdf5 files which are invalid netcdf as described in
-            https://github.com/h5netcdf/h5netcdf.
+            :std:ref:`h5netcdf:invalid netcdf`.
 
         Returns
         -------

xarray/core/dataset.py

@@ -1892,7 +1892,7 @@ def to_netcdf(
         invalid_netcdf: bool, default: False
             Only valid along with ``engine="h5netcdf"``. If True, allow writing
             hdf5 files which are invalid netcdf as described in
-            https://github.com/h5netcdf/h5netcdf.
+            :std:ref:`h5netcdf:invalid netcdf`.
 
         Returns
         -------