Merge branch 'master' into idxmax

Author: max-sixty

doc/whats-new.rst

@@ -38,6 +38,9 @@ New Features
 - Implement :py:meth:`~xarray.DataArray.idxmax`, :py:meth:`~xarray.DataArray.idxmin`,
   :py:meth:`~xarray.Dataset.idxmax`, :py:meth:`~xarray.Dataset.idxmin`.  (:issue:`60`, :pull:`3871`)
   By `Todd Jennings <https://github.com/toddrjen>`_
+- Limited the length of array items with long string reprs to a
+  reasonable width (:pull:`3900`)
+  By `Maximilian Roos <https://github.com/max-sixty>`_
 
 
 Bug fixes

xarray/backends/zarr.py

@@ -10,13 +10,20 @@
 from .common import AbstractWritableDataStore, BackendArray, _encode_variable_name
 
 # need some special secret attributes to tell us the dimensions
-_DIMENSION_KEY = "_ARRAY_DIMENSIONS"
+DIMENSION_KEY = "_ARRAY_DIMENSIONS"
 
 
-# zarr attributes have to be serializable as json
-# many xarray datasets / variables have numpy arrays and values
-# these functions handle encoding / decoding of such items
-def _encode_zarr_attr_value(value):
+def encode_zarr_attr_value(value):
+    """
+    Encode a attribute value as something that can be serialized as json
+
+    Many xarray datasets / variables have numpy arrays and values. This
+    function handles encoding / decoding of such items.
+
+    ndarray -> list
+    scalar array -> scalar
+    other -> other (no change)
+    """
     if isinstance(value, np.ndarray):
         encoded = value.tolist()
     # this checks if it's a scalar number
@@ -170,7 +177,20 @@ def _get_zarr_dims_and_attrs(zarr_obj, dimension_key):
     return dimensions, attributes
 
 
-def _extract_zarr_variable_encoding(variable, raise_on_invalid=False):
+def extract_zarr_variable_encoding(variable, raise_on_invalid=False):
+    """
+    Extract zarr encoding dictionary from xarray Variable
+
+    Parameters
+    ----------
+    variable : xarray.Variable
+    raise_on_invalid : bool, optional
+
+    Returns
+    -------
+    encoding : dict
+        Zarr encoding for `variable`
+    """
     encoding = variable.encoding.copy()
 
     valid_encodings = {"chunks", "compressor", "filters", "cache_metadata"}
@@ -271,7 +291,7 @@ def __init__(self, zarr_group, consolidate_on_close=False):
 
     def open_store_variable(self, name, zarr_array):
         data = indexing.LazilyOuterIndexedArray(ZarrArrayWrapper(name, self))
-        dimensions, attributes = _get_zarr_dims_and_attrs(zarr_array, _DIMENSION_KEY)
+        dimensions, attributes = _get_zarr_dims_and_attrs(zarr_array, DIMENSION_KEY)
         attributes = dict(attributes)
         encoding = {
             "chunks": zarr_array.chunks,
@@ -298,7 +318,7 @@ def get_dimensions(self):
         dimensions = {}
         for k, v in self.ds.arrays():
             try:
-                for d, s in zip(v.attrs[_DIMENSION_KEY], v.shape):
+                for d, s in zip(v.attrs[DIMENSION_KEY], v.shape):
                     if d in dimensions and dimensions[d] != s:
                         raise ValueError(
                             "found conflicting lengths for dimension %s "
@@ -310,7 +330,7 @@ def get_dimensions(self):
                 raise KeyError(
                     "Zarr object is missing the attribute `%s`, "
                     "which is required for xarray to determine "
-                    "variable dimensions." % (_DIMENSION_KEY)
+                    "variable dimensions." % (DIMENSION_KEY)
                 )
         return dimensions
 
@@ -328,7 +348,7 @@ def encode_variable(self, variable):
         return variable
 
     def encode_attribute(self, a):
-        return _encode_zarr_attr_value(a)
+        return encode_zarr_attr_value(a)
 
     def store(
         self,
@@ -433,10 +453,10 @@ def set_variables(self, variables, check_encoding_set, writer, unlimited_dims=No
                     writer.add(v.data, zarr_array, region=tuple(new_region))
             else:
                 # new variable
-                encoding = _extract_zarr_variable_encoding(v, raise_on_invalid=check)
+                encoding = extract_zarr_variable_encoding(v, raise_on_invalid=check)
                 encoded_attrs = {}
                 # the magic for storing the hidden dimension data
-                encoded_attrs[_DIMENSION_KEY] = dims
+                encoded_attrs[DIMENSION_KEY] = dims
                 for k2, v2 in attrs.items():
                     encoded_attrs[k2] = self.encode_attribute(v2)
 

xarray/core/formatting.py

@@ -4,6 +4,7 @@
 import functools
 from datetime import datetime, timedelta
 from itertools import zip_longest
+from typing import Hashable
 
 import numpy as np
 import pandas as pd
@@ -14,7 +15,7 @@
 from .pycompat import dask_array_type, sparse_array_type
 
 
-def pretty_print(x, numchars):
+def pretty_print(x, numchars: int):
     """Given an object `x`, call `str(x)` and format the returned string so
     that it is numchars long, padding with trailing spaces or truncating with
     ellipses as necessary
@@ -163,7 +164,7 @@ def format_items(x):
     return formatted
 
 
-def format_array_flat(array, max_width):
+def format_array_flat(array, max_width: int):
     """Return a formatted string for as many items in the flattened version of
     array that will fit within max_width characters.
     """
@@ -198,11 +199,20 @@ def format_array_flat(array, max_width):
     num_back = count - num_front
     # note that num_back is 0 <--> array.size is 0 or 1
     #                         <--> relevant_back_items is []
-    pprint_str = (
-        " ".join(relevant_front_items[:num_front])
-        + padding
-        + " ".join(relevant_back_items[-num_back:])
+    pprint_str = "".join(
+        [
+            " ".join(relevant_front_items[:num_front]),
+            padding,
+            " ".join(relevant_back_items[-num_back:]),
+        ]
     )
+
+    # As a final check, if it's still too long even with the limit in values,
+    # replace the end with an ellipsis
+    # NB: this will still returns a full 3-character ellipsis when max_width < 3
+    if len(pprint_str) > max_width:
+        pprint_str = pprint_str[: max(max_width - 3, 0)] + "..."
+
     return pprint_str
 
 
@@ -258,10 +268,16 @@ def inline_variable_array_repr(var, max_width):
         return "..."
 
 
-def summarize_variable(name, var, col_width, marker=" ", max_width=None):
+def summarize_variable(
+    name: Hashable, var, col_width: int, marker: str = " ", max_width: int = None
+):
     """Summarize a variable in one line, e.g., for the Dataset.__repr__."""
     if max_width is None:
-        max_width = OPTIONS["display_width"]
+        max_width_options = OPTIONS["display_width"]
+        if not isinstance(max_width_options, int):
+            raise TypeError(f"`max_width` value of `{max_width}` is not a valid int")
+        else:
+            max_width = max_width_options
     first_col = pretty_print(f"  {marker} {name} ", col_width)
     if var.dims:
         dims_str = "({}) ".format(", ".join(map(str, var.dims)))
@@ -295,7 +311,7 @@ def summarize_datavar(name, var, col_width):
     return summarize_variable(name, var.variable, col_width)
 
 
-def summarize_coord(name, var, col_width):
+def summarize_coord(name: Hashable, var, col_width: int):
     is_index = name in var.dims
     marker = "*" if is_index else " "
     if is_index:

xarray/tests/test_backends.py

@@ -4498,3 +4498,50 @@ def test_invalid_netcdf_raises(engine):
     data = create_test_data()
     with raises_regex(ValueError, "unrecognized option 'invalid_netcdf'"):
         data.to_netcdf("foo.nc", engine=engine, invalid_netcdf=True)
+
+
+@requires_zarr
+def test_encode_zarr_attr_value():
+    # array -> list
+    arr = np.array([1, 2, 3])
+    expected = [1, 2, 3]
+    actual = backends.zarr.encode_zarr_attr_value(arr)
+    assert isinstance(actual, list)
+    assert actual == expected
+
+    # scalar array -> scalar
+    sarr = np.array(1)[()]
+    expected = 1
+    actual = backends.zarr.encode_zarr_attr_value(sarr)
+    assert isinstance(actual, int)
+    assert actual == expected
+
+    # string -> string (no change)
+    expected = "foo"
+    actual = backends.zarr.encode_zarr_attr_value(expected)
+    assert isinstance(actual, str)
+    assert actual == expected
+
+
+@requires_zarr
+def test_extract_zarr_variable_encoding():
+
+    var = xr.Variable("x", [1, 2])
+    actual = backends.zarr.extract_zarr_variable_encoding(var)
+    assert "chunks" in actual
+    assert actual["chunks"] is None
+
+    var = xr.Variable("x", [1, 2], encoding={"chunks": (1,)})
+    actual = backends.zarr.extract_zarr_variable_encoding(var)
+    assert actual["chunks"] == (1,)
+
+    # does not raise on invalid
+    var = xr.Variable("x", [1, 2], encoding={"foo": (1,)})
+    actual = backends.zarr.extract_zarr_variable_encoding(var)
+
+    # raises on invalid
+    var = xr.Variable("x", [1, 2], encoding={"foo": (1,)})
+    with raises_regex(ValueError, "unexpected encoding parameters"):
+        actual = backends.zarr.extract_zarr_variable_encoding(
+            var, raise_on_invalid=True
+        )

xarray/tests/test_formatting.py

@@ -115,7 +115,7 @@ def test_format_items(self):
 
     def test_format_array_flat(self):
         actual = formatting.format_array_flat(np.arange(100), 2)
-        expected = "0 ... 99"
+        expected = "..."
         assert expected == actual
 
         actual = formatting.format_array_flat(np.arange(100), 9)
@@ -134,11 +134,13 @@ def test_format_array_flat(self):
         expected = "0 1 2 ... 98 99"
         assert expected == actual
 
+        # NB: Probably not ideal; an alternative would be cutting after the
+        # first ellipsis
         actual = formatting.format_array_flat(np.arange(100.0), 11)
-        expected = "0.0 ... 99.0"
+        expected = "0.0 ... ..."
         assert expected == actual
 
-        actual = formatting.format_array_flat(np.arange(100.0), 1)
+        actual = formatting.format_array_flat(np.arange(100.0), 12)
         expected = "0.0 ... 99.0"
         assert expected == actual
 
@@ -154,16 +156,25 @@ def test_format_array_flat(self):
         expected = ""
         assert expected == actual
 
-        actual = formatting.format_array_flat(np.arange(1), 0)
+        actual = formatting.format_array_flat(np.arange(1), 1)
         expected = "0"
         assert expected == actual
 
-        actual = formatting.format_array_flat(np.arange(2), 0)
+        actual = formatting.format_array_flat(np.arange(2), 3)
         expected = "0 1"
         assert expected == actual
 
-        actual = formatting.format_array_flat(np.arange(4), 0)
-        expected = "0 ... 3"
+        actual = formatting.format_array_flat(np.arange(4), 7)
+        expected = "0 1 2 3"
+        assert expected == actual
+
+        actual = formatting.format_array_flat(np.arange(5), 7)
+        expected = "0 ... 4"
+        assert expected == actual
+
+        long_str = [" ".join(["hello world" for _ in range(100)])]
+        actual = formatting.format_array_flat(np.asarray([long_str]), 21)
+        expected = "'hello world hello..."
         assert expected == actual
 
     def test_pretty_print(self):