♻️Refactored into workspaces

Cargo.toml

@@ -1,18 +1,18 @@
-[package]
-name = "carefree-pyo3"
+[workspace]
+members = [
+    "cfpyo3_rs_core",
+    "cfpyo3_rs_py",
+]
+resolver = "2"
+
+[workspace.package]
 version = "0.1.7"
 edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-[lib]
-name = "cfpyo3"
-crate-type = ["cdylib"]
-
-[dependencies]
-itertools = "0.13.0"
-md-5 = "0.10.6"
-num-traits = "0.2.19"
-numpy = "0.21.0"
-paste = "1.0.15"
-pyo3 = { version = "0.21.0", features = ["abi3-py38", "chrono", "extension-module", "multiple-pymethods"] }
-rayon = "1.10.0"
+authors = ["carefree0910 <[email protected]>"]
+description = "a collection of performant utilities"
+homepage = "https://github.com/carefree0910/carefree-pyo3"
+repository = "https://github.com/carefree0910/carefree-pyo3"
+readme = "README.md"
+keywords = ["python", "rust", "pyo3"]
+categories = ["algorithms", "data-structures"]
+license-file = "LICENSE"

cfpyo3_rs_core/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "cfpyo3_rs_core"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
+readme.workspace = true
+keywords.workspace = true
+categories.workspace = true
+license-file.workspace = true
+
+[lib]
+name = "cfpyo3_core"
+crate-type = ["rlib"]
+
+[dependencies]
+itertools = "0.13.0"
+md-5 = "0.10.6"
+num-traits = "0.2.19"
+numpy = "0.21.0"
+rayon = "1.10.0"

cfpyo3_rs_core/src/df/frame.rs

@@ -0,0 +1,12 @@
+use crate::toolkit::array::AFloat;
+
+use super::{ColumnsDtype, IndexDtype};
+use numpy::ndarray::{ArrayView1, ArrayView2};
+
+mod ops;
+
+pub struct DataFrame<'a, T: AFloat> {
+    pub index: ArrayView1<'a, IndexDtype>,
+    pub columns: ArrayView1<'a, ColumnsDtype>,
+    pub data: ArrayView2<'a, T>,
+}

cfpyo3_rs_core/src/df/frame/ops.rs

@@ -0,0 +1,13 @@
+use super::DataFrame;
+use crate::toolkit::array::{corr_axis1, mean_axis1, AFloat};
+use numpy::ndarray::ArrayView2;
+
+impl<'a, T: AFloat> DataFrame<'a, T> {
+    pub fn mean_axis1(&'a self, num_threads: usize) -> Vec<T> {
+        mean_axis1(&self.data.view(), num_threads)
+    }
+
+    pub fn corr_with_axis1(&'a self, other: ArrayView2<T>, num_threads: usize) -> Vec<T> {
+        corr_axis1(&self.data.view(), &other.view(), num_threads)
+    }
+}

cfpyo3_rs_core/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod df;
+pub mod toolkit;

src/toolkit/array.rs → cfpyo3_rs_core/src/toolkit/array.rs

@@ -1,11 +1,15 @@
 use itertools::{enumerate, izip};
 use num_traits::{Float, FromPrimitive};
-use numpy::{
-    ndarray::{ArrayView1, ArrayView2, Axis, ScalarOperand},
-    IntoPyArray, PyArray1,
+use numpy::ndarray::{ArrayView1, ArrayView2, Axis, ScalarOperand};
+use std::{
+    cell::UnsafeCell,
+    fmt::{Debug, Display},
+    iter::zip,
+    mem,
+    ops::{AddAssign, MulAssign, SubAssign},
+    ptr,
+    thread::available_parallelism,
 };
-use pyo3::prelude::*;
-use std::{cell::UnsafeCell, iter::zip, mem, ops::AddAssign, ptr, thread::available_parallelism};
 
 #[derive(Copy, Clone)]
 pub struct UnsafeSlice<'a, T> {
@@ -51,7 +55,7 @@ fn fill_concat<D: Copy>((offsets, arrays, mut out): Task<D>) {
         out.copy_from_slice(offset, arrays[i].as_slice().unwrap());
     });
 }
-fn fast_concat_2d_axis0<D: Copy + Send + Sync>(
+pub fn fast_concat_2d_axis0<D: Copy + Send + Sync>(
     arrays: Vec<ArrayView2<D>>,
     num_rows: Vec<usize>,
     num_columns: usize,
@@ -128,34 +132,34 @@ fn fast_concat_2d_axis0<D: Copy + Send + Sync>(
     }
 }
 
-macro_rules! fast_concat_2d_axis0_impl {
-    ($name:ident, $dtype:ty, $multiplier:expr) => {
-        pub fn $name<'py>(
-            py: Python<'py>,
-            arrays: Vec<ArrayView2<$dtype>>,
-        ) -> Bound<'py, PyArray1<$dtype>> {
-            let num_rows: Vec<usize> = arrays.iter().map(|a| a.shape()[0]).collect();
-            let num_columns = arrays[0].shape()[1];
-            for array in &arrays {
-                if array.shape()[1] != num_columns {
-                    panic!("all arrays should have same number of columns");
-                }
-            }
-            let num_total_rows: usize = num_rows.iter().sum();
-            let mut out: Vec<$dtype> = vec![0.; num_total_rows * num_columns];
-            let out_slice = UnsafeSlice::new(out.as_mut_slice());
-            fast_concat_2d_axis0(arrays, num_rows, num_columns, $multiplier, out_slice);
-            out.into_pyarray_bound(py)
-        }
-    };
+pub trait AFloat:
+    Float
+    + AddAssign
+    + SubAssign
+    + MulAssign
+    + FromPrimitive
+    + ScalarOperand
+    + Send
+    + Sync
+    + Debug
+    + Display
+{
 }
-fast_concat_2d_axis0_impl!(fast_concat_2d_axis0_f32, f32, 1);
-fast_concat_2d_axis0_impl!(fast_concat_2d_axis0_f64, f64, 2);
-
-fn mean<T>(a: ArrayView1<T>) -> T
-where
-    T: Float + AddAssign,
+impl<T> AFloat for T where
+    T: Float
+        + AddAssign
+        + SubAssign
+        + MulAssign
+        + FromPrimitive
+        + ScalarOperand
+        + Send
+        + Sync
+        + Debug
+        + Display
 {
+}
+
+fn mean<T: AFloat>(a: ArrayView1<T>) -> T {
     let mut sum = T::zero();
     let mut num = T::zero();
     for &x in a.iter() {
@@ -172,10 +176,7 @@ where
     }
 }
 
-fn corr<T>(a: ArrayView1<T>, b: ArrayView1<T>) -> T
-where
-    T: Float + AddAssign + FromPrimitive + ScalarOperand,
-{
+fn corr<T: AFloat>(a: ArrayView1<T>, b: ArrayView1<T>) -> T {
     let valid_indices: Vec<usize> = zip(a.iter(), b.iter())
         .enumerate()
         .filter_map(|(i, (&x, &y))| {
@@ -201,10 +202,7 @@ where
     cov / (var1.sqrt() * var2.sqrt())
 }
 
-pub fn mean_axis1<T>(a: &ArrayView2<T>, num_threads: usize) -> Vec<T>
-where
-    T: Float + AddAssign + FromPrimitive + Send + Sync,
-{
+pub fn mean_axis1<T: AFloat>(a: &ArrayView2<T>, num_threads: usize) -> Vec<T> {
     let mut res: Vec<T> = vec![T::zero(); a.nrows()];
     let mut slice = UnsafeSlice::new(res.as_mut_slice());
     let pool = rayon::ThreadPoolBuilder::new()
@@ -219,10 +217,7 @@ where
     res
 }
 
-pub fn corr_axis1<T>(a: &ArrayView2<T>, b: &ArrayView2<T>, num_threads: usize) -> Vec<T>
-where
-    T: Float + AddAssign + FromPrimitive + ScalarOperand + Send + Sync,
-{
+pub fn corr_axis1<T: AFloat>(a: &ArrayView2<T>, b: &ArrayView2<T>, num_threads: usize) -> Vec<T> {
     let mut res: Vec<T> = vec![T::zero(); a.nrows()];
     let mut slice = UnsafeSlice::new(res.as_mut_slice());
     let pool = rayon::ThreadPoolBuilder::new()

src/toolkit/misc.rs → cfpyo3_rs_core/src/toolkit/misc.rs

@@ -1,7 +1,5 @@
 use md5::{Digest, Md5};
-use pyo3::prelude::*;
 
-#[pyfunction]
 pub fn hash_code(code: &str) -> String {
     let mut hasher = Md5::new();
     hasher.update(code.as_bytes());

cfpyo3_rs_core/tests/test_array.rs

@@ -0,0 +1,71 @@
+use cfpyo3_core::toolkit::array;
+use numpy::ndarray::ArrayView2;
+
+fn assert_allclose<T: array::AFloat>(a: &[T], b: &[T]) {
+    let atol = T::from_f64(1e-6).unwrap();
+    let rtol = T::from_f64(1e-6).unwrap();
+    a.iter().zip(b.iter()).for_each(|(&x, &y)| {
+        assert!(
+            (x - y).abs() <= atol + rtol * y.abs(),
+            "not close - x: {:?}, y: {:?}",
+            x,
+            y
+        );
+    });
+}
+
+macro_rules! test_fast_concat_2d_axis0 {
+    ($dtype:ty) => {
+        let array_2d_u = ArrayView2::<$dtype>::from_shape((1, 3), &[1., 2., 3.]).unwrap();
+        let array_2d_l =
+            ArrayView2::<$dtype>::from_shape((2, 3), &[4., 5., 6., 7., 8., 9.]).unwrap();
+        let arrays = vec![array_2d_u, array_2d_l];
+        let mut out: Vec<$dtype> = vec![0.; 3 * 3];
+        let out_slice = array::UnsafeSlice::new(out.as_mut_slice());
+        array::fast_concat_2d_axis0(arrays, vec![1, 2], 3, 1, out_slice);
+        assert_eq!(out.as_slice(), &[1., 2., 3., 4., 5., 6., 7., 8., 9.]);
+    };
+}
+
+macro_rules! test_mean_axis1 {
+    ($dtype:ty) => {
+        let array = ArrayView2::<$dtype>::from_shape((2, 3), &[1., 2., 3., 4., 5., 6.]).unwrap();
+        let out = array::mean_axis1(&array, 1);
+        assert_allclose(out.as_slice(), &[2., 5.]);
+    };
+}
+
+macro_rules! test_corr_axis1 {
+    ($dtype:ty) => {
+        let array = ArrayView2::<$dtype>::from_shape((2, 3), &[1., 2., 3., 4., 5., 6.]).unwrap();
+        let out = array::corr_axis1(&array, &(&array + 1.).view(), 1);
+        assert_allclose(out.as_slice(), &[1., 1.]);
+    };
+}
+
+#[test]
+fn test_fast_concat_2d_axis0_f32() {
+    test_fast_concat_2d_axis0!(f32);
+}
+#[test]
+fn test_fast_concat_2d_axis0_f64() {
+    test_fast_concat_2d_axis0!(f64);
+}
+
+#[test]
+fn test_mean_axis1_f32() {
+    test_mean_axis1!(f32);
+}
+#[test]
+fn test_mean_axis1_f64() {
+    test_mean_axis1!(f64);
+}
+
+#[test]
+fn test_corr_axis1_f32() {
+    test_corr_axis1!(f32);
+}
+#[test]
+fn test_corr_axis1_f64() {
+    test_corr_axis1!(f64);
+}

cfpyo3_rs_py/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "cfpyo3_rs_py"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
+readme.workspace = true
+keywords.workspace = true
+categories.workspace = true
+license-file.workspace = true
+
+[lib]
+name = "cfpyo3"
+crate-type = ["cdylib"]
+
+[dependencies]
+cfpyo3_rs_core = { path = "../cfpyo3_rs_core" }
+numpy = "0.21.0"
+paste = "1.0.15"
+pyo3 = { version = "0.21.0", features = ["abi3-py38", "chrono", "extension-module", "multiple-pymethods"] }

cfpyo3_rs_py/src/df.rs

@@ -0,0 +1 @@
+pub mod frame;

src/df/frame.rs → cfpyo3_rs_py/src/df/frame.rs

@@ -1,4 +1,4 @@
-use super::{ColumnsDtype, IndexDtype};
+use cfpyo3_core::df::{ColumnsDtype, IndexDtype};
 use numpy::{PyArray1, PyArray2};
 use pyo3::prelude::*;
 

src/df/frame/meta.rs → cfpyo3_rs_py/src/df/frame/meta.rs

@@ -1,4 +1,6 @@
-use super::{ColumnsDtype, DataFrameF64, IndexDtype};
+use super::DataFrameF64;
+use cfpyo3_core::df::frame::DataFrame;
+use cfpyo3_core::df::{ColumnsDtype, IndexDtype};
 use numpy::{
     ndarray::{ArrayView1, ArrayView2},
     PyArray1, PyArray2, PyArrayMethods,
@@ -15,6 +17,16 @@ impl DataFrameF64 {
     pub fn get_data_array<'a>(&'a self, py: Python<'a>) -> ArrayView2<'a, f64> {
         unsafe { self.data.bind(py).as_array() }
     }
+    pub fn to_core<'a>(&'a self, py: Python<'a>) -> DataFrame<'a, f64> {
+        let index = self.get_index_array(py);
+        let columns = self.get_columns_array(py);
+        let data = self.get_data_array(py);
+        DataFrame::<f64> {
+            index,
+            columns,
+            data,
+        }
+    }
 }
 
 #[pymethods]