Update ndarray to 0.16 and ndarray-rand to 0.15 (#7358)

<!--
Open the PR up as a draft until you feel it is ready for a proper
review.

Do not make PR:s from your own `main` branch, as that makes it difficult
for reviewers to add their own fixes.

Add any improvements to the branch as new commits to make it easier for
reviewers to follow the progress. All commits will be squashed to a
single commit once the PR is merged into `main`.

Make sure you mention any issues that this PR closes in the description,
as well as any other related issues.

To get an auto-generated PR description you can put "copilot:summary" or
"copilot:walkthrough" anywhere.
-->

### What
Fixes #7157. I also updated ndarray-rand (sorry if that should be
separate - it didn't appear to require any changes).

into_raw_vec() is now into_raw_vec_and_offset().0 (the offset is
ignored)

I wasn't sure whether an order (`RowMajor` vs. `ColumnMajor`) should be
specified in `into_shape_with_order()`. I ran through all the examples
and couldn't find one that actually used this code, so guidance here is
appreciated.

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7358?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7358?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7358)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

Cargo.lock

@@ -3529,22 +3529,24 @@ checksum = "308d96db8debc727c3fd9744aac51751243420e46edf401010908da7f8d5e57c"
 
 [[package]]
 name = "ndarray"
-version = "0.15.6"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32"
+checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
 dependencies = [
  "matrixmultiply",
  "num-complex",
  "num-integer",
  "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
  "rawpointer",
 ]
 
 [[package]]
 name = "ndarray-rand"
-version = "0.14.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65608f937acc725f5b164dcf40f4f0bc5d67dc268ab8a649d3002606718c4588"
+checksum = "f093b3db6fd194718dcdeea6bd8c829417deae904e3fcc7732dabcd4416d25d8"
 dependencies = [
  "ndarray",
  "rand",
@@ -4348,9 +4350,18 @@ checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2"
 
 [[package]]
 name = "portable-atomic"
-version = "1.5.1"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b"
+checksum = "fcdd8420072e66d54a407b3316991fe946ce3ab1083a7f575b2463866624704d"
+dependencies = [
+ "portable-atomic",
+]
 
 [[package]]
 name = "powerfmt"

Cargo.toml

@@ -198,8 +198,8 @@ mime_guess2 = "2.0" # infer MIME type by file extension, and map mime to file ex
 mint = "0.5.9"
 mp4 = "0.14.0"
 natord = "1.0.9"
-ndarray = "0.15"
-ndarray-rand = "0.14"
+ndarray = "0.16"
+ndarray-rand = "0.15"
 never = "0.1"
 nohash-hasher = "0.2"
 notify = "6.0"

crates/store/re_types/src/datatypes/tensor_data_ext.rs

@@ -8,9 +8,6 @@ use crate::archetypes::EncodedImage;
 
 use super::{TensorBuffer, TensorData, TensorDimension};
 
-// Much of the following duplicates code from: `crates/re_components/src/tensor.rs`, which
-// will eventually go away as the Tensor migration is completed.
-
 // ----------------------------------------------------------------------------
 
 impl TensorData {
@@ -169,7 +166,6 @@ macro_rules! tensor_from_ndarray {
             type Error = TensorCastError;
 
             fn try_from(value: ndarray::Array<$type, D>) -> Result<Self, Self::Error> {
-                let value = value.as_standard_layout();
                 let shape = value
                     .shape()
                     .iter()
@@ -178,13 +174,26 @@ macro_rules! tensor_from_ndarray {
                         name: None,
                     })
                     .collect();
-                value
-                    .is_standard_layout()
-                    .then(|| TensorData {
-                        shape,
-                        buffer: TensorBuffer::$variant(value.to_owned().into_raw_vec().into()),
-                    })
-                    .ok_or(TensorCastError::NotContiguousStdOrder)
+
+                let vec = if value.is_standard_layout() {
+                    let (mut vec, offset) = value.into_raw_vec_and_offset();
+                    // into_raw_vec_and_offset() guarantees that the logical element order (.iter()) matches the internal
+                    // storage order in the returned vector if the array is in standard layout.
+                    if let Some(offset) = offset {
+                        vec.drain(..offset);
+                        vec
+                    } else {
+                        debug_assert!(vec.is_empty());
+                        vec
+                    }
+                } else {
+                    value.into_iter().collect::<Vec<_>>()
+                };
+
+                Ok(Self {
+                    shape,
+                    buffer: TensorBuffer::$variant(vec.into()),
+                })
             }
         }
 
@@ -311,13 +320,18 @@ impl<D: ::ndarray::Dimension> TryFrom<::ndarray::Array<half::f16, D>> for Tensor
             })
             .collect();
         if value.is_standard_layout() {
+            let (vec, offset) = value.into_raw_vec_and_offset();
+            // into_raw_vec_and_offset() guarantees that the logical element order (.iter()) matches the internal
+            // storage order in the returned vector if the array is in standard layout.
+            let vec_slice = if let Some(offset) = offset {
+                &vec[offset..]
+            } else {
+                debug_assert!(vec.is_empty());
+                &vec
+            };
             Ok(Self {
                 shape,
-                buffer: TensorBuffer::F16(
-                    bytemuck::cast_slice(value.into_raw_vec().as_slice())
-                        .to_vec()
-                        .into(),
-                ),
+                buffer: TensorBuffer::F16(Vec::from(bytemuck::cast_slice(vec_slice)).into()),
             })
         } else {
             Ok(Self {

crates/store/re_types/tests/types/tensor.rs

@@ -106,7 +106,7 @@ fn convert_tensor_to_ndarray_f32() {
 }
 
 #[test]
-fn convert_ndarray_u8_to_tensor() {
+fn convert_ndarray_f64_to_tensor() {
     let n = ndarray::array![[1., 2., 3.], [4., 5., 6.]];
     let t = TensorData::try_from(n).unwrap();
 
@@ -125,6 +125,97 @@ fn convert_ndarray_slice_to_tensor() {
     assert_eq!(t.shape(), &[TensorDimension::unnamed(2)]);
 }
 
+#[test]
+fn convert_ndarray_to_tensor_both_layouts() {
+    #[rustfmt::skip]
+    let row_major_vec = vec![
+        1, 2, 3,
+        4, 5, 6,
+        7, 8, 9
+    ];
+    #[rustfmt::skip]
+    let col_major_vec = vec![
+        1, 4, 7,
+        2, 5, 8,
+        3, 6, 9
+    ];
+
+    let shape = ndarray::Ix2(3, 3);
+
+    let row_major = ndarray::Array::from_vec(row_major_vec)
+        .into_shape_with_order((shape, ndarray::Order::RowMajor))
+        .unwrap();
+
+    let col_major = ndarray::Array::from_vec(col_major_vec)
+        .into_shape_with_order((shape, ndarray::Order::ColumnMajor))
+        .unwrap();
+
+    assert!(row_major.is_standard_layout());
+    assert!(!col_major.is_standard_layout());
+
+    // make sure that the offset is in fact zero, in case ndarray behavior changes
+    let rm = row_major.clone();
+    let cm = col_major.clone();
+    let (_, rm_offset) = rm.into_raw_vec_and_offset();
+    let (_, cm_offset) = cm.into_raw_vec_and_offset();
+    assert_eq!(rm_offset.unwrap(), 0);
+    assert_eq!(cm_offset.unwrap(), 0);
+
+    let tensor_row_major = TensorData::try_from(row_major).unwrap();
+    let tensor_col_major = TensorData::try_from(col_major).unwrap();
+
+    assert_eq!(tensor_row_major, tensor_col_major);
+}
+
+#[test]
+fn convert_ndarray_to_tensor_both_layouts_nonzero_offset() {
+    #[rustfmt::skip]
+    let row_major_vec = vec![
+        1, 2, 3,
+        4, 5, 6,
+        7, 8, 9
+    ];
+    #[rustfmt::skip]
+    let col_major_vec = vec![
+        1, 4, 7,
+        2, 5, 8,
+        3, 6, 9
+    ];
+
+    let shape = ndarray::Ix2(3, 3);
+
+    let row_major = ndarray::Array::from_vec(row_major_vec)
+        .into_shape_with_order((shape, ndarray::Order::RowMajor))
+        .unwrap();
+    assert!(row_major.is_standard_layout());
+    let row_major_nonzero_offset = row_major.slice_move(ndarray::s![1.., ..]);
+
+    let col_major = ndarray::Array::from_vec(col_major_vec)
+        .into_shape_with_order((shape, ndarray::Order::ColumnMajor))
+        .unwrap();
+    assert!(!col_major.is_standard_layout());
+    let col_major_nonzero_offset = col_major.slice_move(ndarray::s![1.., ..]);
+
+    assert!(row_major_nonzero_offset.is_standard_layout());
+    assert!(!col_major_nonzero_offset.is_standard_layout());
+
+    // make sure that the offset is in fact non-zero, in case ndarray behavior changes
+    let rmno = row_major_nonzero_offset.clone();
+    let cmno = col_major_nonzero_offset.clone();
+    let (_, rm_offset) = rmno.into_raw_vec_and_offset();
+    let (_, cm_offset) = cmno.into_raw_vec_and_offset();
+    assert!(rm_offset.unwrap() > 0);
+    assert!(cm_offset.unwrap() > 0);
+
+    let tensor_row_major_nonzero_offset = TensorData::try_from(row_major_nonzero_offset).unwrap();
+    let tensor_col_major_nonzero_offset = TensorData::try_from(col_major_nonzero_offset).unwrap();
+
+    assert_eq!(
+        tensor_row_major_nonzero_offset,
+        tensor_col_major_nonzero_offset
+    );
+}
+
 #[test]
 fn check_slices() {
     let t = TensorData::new(

crates/viewer/re_space_view_tensor/src/space_view_class.rs

@@ -424,7 +424,7 @@ pub fn selected_tensor_slice<'a, T: Copy>(
         // This is important for above width/height conversion to work since this assumes at least 2 dimensions.
         tensor
             .view()
-            .into_shape(ndarray::IxDyn(&[tensor.len(), 1]))
+            .into_shape_with_order(ndarray::IxDyn(&[tensor.len(), 1]))
             .unwrap()
     } else {
         tensor.view()

docs/snippets/all/archetypes/image_send_columns.rs

@@ -31,7 +31,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Split up the image data into several components referencing the underlying data.
     let image_size_in_bytes = width * height * 3;
-    let blob = rerun::datatypes::Blob::from(images.into_raw_vec());
+    let blob = rerun::datatypes::Blob::from(images.into_raw_vec_and_offset().0);
     let image_column = times
         .iter()
         .map(|&t| {