Fix validity for soma_data in sparse arrays

single-cell-data · Jan 24, 2025 · 716b705 · 716b705
1 parent a7e79be
commit 716b705
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 27 deletions.
diff --git a/apis/python/src/tiledbsoma/_dense_nd_array.py b/apis/python/src/tiledbsoma/_dense_nd_array.py
@@ -290,9 +290,6 @@ def write(
         """
         _util.check_type("values", values, (pa.Tensor,))
 
-        if np.isnan(values).any():
-            raise SOMAError("soma_data cannot contain nulls")
-
         clib_handle = self._handle._handle
 
         # Compute the coordinates for the dense array.

diff --git a/apis/python/src/tiledbsoma/_sparse_nd_array.py b/apis/python/src/tiledbsoma/_sparse_nd_array.py
@@ -308,7 +308,6 @@ def write(
         Lifecycle:
             Maturing.
         """
-
         write_options: Union[TileDBCreateOptions, TileDBWriteOptions]
         sort_coords = None
         if isinstance(platform_config, TileDBCreateOptions):

diff --git a/apis/python/tests/test_dense_nd_array.py b/apis/python/tests/test_dense_nd_array.py
@@ -565,14 +565,3 @@ def test_pass_configs(tmp_path):
                 "sm.io_concurrency_level": "1",
             }
         )
-
-
-def test_dense_nd_array_null(tmp_path):
-    uri = tmp_path.as_posix()
-
-    data = pa.Tensor.from_numpy(np.array([np.nan]))
-
-    with soma.DenseNDArray.create(uri, type=pa.int64(), shape=(1,)) as a:
-        # Cannot write null values
-        with pytest.raises(soma.SOMAError):
-            a.write(coords=(0,), values=data)
diff --git a/apis/python/tests/test_sparse_nd_array.py b/apis/python/tests/test_sparse_nd_array.py
@@ -2012,10 +2012,18 @@ def test(path, tiledb_config):
 def test_sparse_nd_array_null(tmp_path):
     uri = tmp_path.as_posix()
 
-    arrow_array = pa.array([None], type=pa.float64())
-    values = pa.Table.from_arrays([arrow_array], names=["soma_data"])
+    pydict = {
+        "soma_dim_0": pa.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
+        "soma_data": pa.array(
+            [None, 0, None, 1, 2, None, None, 3, 4, 5], type=pa.float64()
+        ),
+    }
+    table = pa.Table.from_pydict(pydict)
 
-    with soma.SparseNDArray.create(uri, type=pa.int64(), shape=(1,)) as a:
-        # Cannot write null values
-        with pytest.raises(soma.SOMAError):
-            a.write(values=values)
+    with soma.SparseNDArray.create(uri, type=pa.int64(), shape=(10,)) as A:
+        A.write(table[:5])
+        A.write(table[5:])
+
+    with soma.SparseNDArray.open(uri) as A:
+        pdf = A.read().tables().concat()
+        np.testing.assert_array_equal(pdf["soma_data"], table["soma_data"])
diff --git a/libtiledbsoma/src/soma/column_buffer.h b/libtiledbsoma/src/soma/column_buffer.h
@@ -153,11 +153,11 @@ class ColumnBuffer {
                 validity_.assign(num_elems, 1);  // Default all to valid (1)
             }
         } else {
-            if (validity.has_value()) {
-                throw TileDBSOMAError(
-                    "[ColumnBuffer] Validity buffer passed for non-nullable "
-                    "column");
-            }
+            throw TileDBSOMAError(std::format(
+                "[ColumnBuffer] Validity buffer passed for non-nullable "
+                "column '{}'. Saw values: {}",
+                name_,
+                validity_values));
         }
     }
 

diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.cc b/libtiledbsoma/src/soma/soma_sparse_ndarray.cc
@@ -56,7 +56,7 @@ void SOMASparseNDArray::create(
     attr->format = strdup(std::string(format).c_str());
     attr->name = strdup("soma_data");
     attr->n_children = 0;
-    attr->flags = 0;
+    attr->flags = ARROW_FLAG_NULLABLE;
     attr->dictionary = nullptr;
     attr->metadata = nullptr;
     attr->release = &ArrowAdapter::release_schema;