From ba16f0a74d41242a784b822ce20fcdfc3b8f35dc Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:33:03 +0100 Subject: [PATCH] try removing pyo3-polars from repo --- Cargo.toml | 2 - polars_business/polars_business/Cargo.toml | 2 +- pyo3-polars-derive/Cargo.toml | 28 --- pyo3-polars-derive/README.md | 4 - pyo3-polars-derive/src/attr.rs | 50 ----- pyo3-polars-derive/src/keywords.rs | 2 - pyo3-polars-derive/src/lib.rs | 212 --------------------- pyo3-polars-derive/tests/01.rs | 19 -- pyo3-polars-derive/tests/02.rs | 14 -- pyo3-polars-derive/tests/run.rs | 6 - pyo3-polars/Cargo.toml | 27 --- pyo3-polars/src/derive.rs | 30 --- pyo3-polars/src/error.rs | 66 ------- pyo3-polars/src/export.rs | 3 - pyo3-polars/src/ffi/mod.rs | 2 - pyo3-polars/src/ffi/to_py.rs | 24 --- pyo3-polars/src/ffi/to_rust.rs | 27 --- pyo3-polars/src/lib.rs | 203 -------------------- 18 files changed, 1 insertion(+), 720 deletions(-) delete mode 100644 pyo3-polars-derive/Cargo.toml delete mode 100644 pyo3-polars-derive/README.md delete mode 100644 pyo3-polars-derive/src/attr.rs delete mode 100644 pyo3-polars-derive/src/keywords.rs delete mode 100644 pyo3-polars-derive/src/lib.rs delete mode 100644 pyo3-polars-derive/tests/01.rs delete mode 100644 pyo3-polars-derive/tests/02.rs delete mode 100644 pyo3-polars-derive/tests/run.rs delete mode 100644 pyo3-polars/Cargo.toml delete mode 100644 pyo3-polars/src/derive.rs delete mode 100644 pyo3-polars/src/error.rs delete mode 100644 pyo3-polars/src/export.rs delete mode 100644 pyo3-polars/src/ffi/mod.rs delete mode 100644 pyo3-polars/src/ffi/to_py.rs delete mode 100644 pyo3-polars/src/ffi/to_rust.rs delete mode 100644 pyo3-polars/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 433de5b..221d2ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,6 @@ resolver = "2" members = [ "polars_business/polars_business", - "pyo3-polars", - "pyo3-polars-derive", ] [workspace.dependencies] diff --git a/polars_business/polars_business/Cargo.toml b/polars_business/polars_business/Cargo.toml index 220db0f..9b7af44 100644 --- a/polars_business/polars_business/Cargo.toml +++ b/polars_business/polars_business/Cargo.toml @@ -15,7 +15,7 @@ polars-time = { workspace = true, default-features = false } polars-plan = { workspace = true, default-features = false } polars-ops = { workspace = true, default-features = false } pyo3 = { version = "0.20.0", features = ["extension-module"] } -pyo3-polars = { version = "*", path = "../../pyo3-polars", features = ["derive"] } +pyo3-polars = { git = "https://github.com/pola-rs/pyo3-polars", rev = "0a3df66ce626504e418491cb4b1d31bd3ee7ed6c", features = ["derive"] } serde = { version = "1", features = ["derive"] } [target .'cfg(target_os = "linux")'] diff --git a/pyo3-polars-derive/Cargo.toml b/pyo3-polars-derive/Cargo.toml deleted file mode 100644 index 9645a04..0000000 --- a/pyo3-polars-derive/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "pyo3-polars-derive" -version = "0.1.0" -edition = "2021" -license = "MIT" -readme = "README.md" -repository = "https://github.com/pola-rs/pyo3-polars" -description = "Proc-macro's for pyo3-polars" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -proc-macro = true - -[[test]] -name = "tests" -path = "tests/run.rs" - -[dependencies] -polars-core = { workspace = true } -polars-ffi = { workspace = true } -polars-plan = { workspace = true } -proc-macro2 = "1.0" -quote = "1.0" -syn = { version = "2", features = ["full", "extra-traits"] } - -[dev-dependencies] -trybuild = { version = "1", features = ["diff"] } diff --git a/pyo3-polars-derive/README.md b/pyo3-polars-derive/README.md deleted file mode 100644 index 1112723..0000000 --- a/pyo3-polars-derive/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Procedure macros for pyo3-polars - -This crate is not intended for direct usage. Use `pyo3-polars` to get the functionality -of this crate. diff --git a/pyo3-polars-derive/src/attr.rs b/pyo3-polars-derive/src/attr.rs deleted file mode 100644 index a5fecc3..0000000 --- a/pyo3-polars-derive/src/attr.rs +++ /dev/null @@ -1,50 +0,0 @@ -use crate::keywords; -use proc_macro2::Ident; -use std::fmt::Debug; -use syn::parse::{Parse, ParseStream}; -use syn::Token; - -#[derive(Clone, Debug)] -pub struct KeyWordAttribute { - pub kw: K, - pub value: V, -} - -impl Parse for KeyWordAttribute { - fn parse(input: ParseStream) -> syn::Result { - let kw = input.parse()?; - let _: Token![=] = input.parse()?; - let value = input.parse()?; - Ok(KeyWordAttribute { kw, value }) - } -} - -pub type OutputAttribute = KeyWordAttribute; -pub type OutputFuncAttribute = KeyWordAttribute; - -#[derive(Default, Debug)] -pub struct ExprsFunctionOptions { - pub output_dtype: Option, - pub output_type_fn: Option, -} - -impl Parse for ExprsFunctionOptions { - fn parse(input: ParseStream<'_>) -> syn::Result { - let mut options = ExprsFunctionOptions::default(); - - while !input.is_empty() { - let lookahead = input.lookahead1(); - - if lookahead.peek(keywords::output_type) { - let attr = input.parse::()?; - options.output_dtype = Some(attr.value) - } else if lookahead.peek(keywords::type_func) { - let attr = input.parse::()?; - options.output_type_fn = Some(attr.value) - } else { - panic!("didn't recognize attribute") - } - } - Ok(options) - } -} diff --git a/pyo3-polars-derive/src/keywords.rs b/pyo3-polars-derive/src/keywords.rs deleted file mode 100644 index 062baba..0000000 --- a/pyo3-polars-derive/src/keywords.rs +++ /dev/null @@ -1,2 +0,0 @@ -syn::custom_keyword!(output_type); -syn::custom_keyword!(type_func); diff --git a/pyo3-polars-derive/src/lib.rs b/pyo3-polars-derive/src/lib.rs deleted file mode 100644 index 3f4eea2..0000000 --- a/pyo3-polars-derive/src/lib.rs +++ /dev/null @@ -1,212 +0,0 @@ -mod attr; -mod keywords; - -use proc_macro::TokenStream; -use quote::quote; -use std::sync::atomic::{AtomicBool, Ordering}; -use syn::{parse_macro_input, FnArg}; - -static INIT: AtomicBool = AtomicBool::new(false); - -fn insert_error_function() -> proc_macro2::TokenStream { - let is_init = INIT.swap(true, Ordering::Relaxed); - - // Only expose the error retrieval function on the first expression. - if !is_init { - quote!( - pub use pyo3_polars::derive::get_last_error_message; - ) - } else { - proc_macro2::TokenStream::new() - } -} - -fn quote_call_kwargs(ast: &syn::ItemFn, fn_name: &syn::Ident) -> proc_macro2::TokenStream { - quote!( - - let kwargs = std::slice::from_raw_parts(kwargs_ptr, kwargs_len); - - let kwargs = match pyo3_polars::derive::_parse_kwargs(kwargs) { - Ok(value) => value, - Err(err) => { - pyo3_polars::derive::_update_last_error(err); - return; - } - }; - - // define the function - #ast - - // call the function - let result: PolarsResult = #fn_name(&inputs, kwargs); - - ) -} - -fn quote_call_no_kwargs(ast: &syn::ItemFn, fn_name: &syn::Ident) -> proc_macro2::TokenStream { - quote!( - // define the function - #ast - // call the function - let result: PolarsResult = #fn_name(&inputs); - ) -} - -fn quote_process_results() -> proc_macro2::TokenStream { - quote!(match result { - Ok(out) => { - // Update return value. - *return_value = polars_ffi::export_series(&out); - } - Err(err) => { - // Set latest error, but leave return value in empty state. - pyo3_polars::derive::_update_last_error(err); - } - }) -} - -fn create_expression_function(ast: syn::ItemFn) -> proc_macro2::TokenStream { - // count how often the user define a kwargs argument. - let n_kwargs = ast - .sig - .inputs - .iter() - .filter(|fn_arg| { - if let FnArg::Typed(pat) = fn_arg { - if let syn::Pat::Ident(pat) = pat.pat.as_ref() { - pat.ident.to_string() == "kwargs" - } else { - false - } - } else { - true - } - }) - .count(); - - let fn_name = &ast.sig.ident; - let error_msg_fn = insert_error_function(); - - let quote_call = match n_kwargs { - 0 => quote_call_no_kwargs(&ast, fn_name), - 1 => quote_call_kwargs(&ast, fn_name), - _ => unreachable!(), // arguments are unique - }; - let quote_process_result = quote_process_results(); - - quote!( - use pyo3_polars::export::*; - - #error_msg_fn - - // create the outer public function - #[no_mangle] - pub unsafe extern "C" fn #fn_name ( - e: *mut polars_ffi::SeriesExport, - input_len: usize, - kwargs_ptr: *const u8, - kwargs_len: usize, - return_value: *mut polars_ffi::SeriesExport - ) { - let inputs = polars_ffi::import_series_buffer(e, input_len).unwrap(); - - #quote_call - - #quote_process_result - } - ) -} - -fn get_field_name(fn_name: &syn::Ident) -> syn::Ident { - syn::Ident::new(&format!("__polars_field_{}", fn_name), fn_name.span()) -} - -fn get_inputs() -> proc_macro2::TokenStream { - quote!( - let inputs = std::slice::from_raw_parts(field, len); - let inputs = inputs.iter().map(|field| { - let field = polars_core::export::arrow::ffi::import_field_from_c(field).unwrap(); - let out = polars_core::prelude::Field::from(&field); - out - }).collect::>(); - ) -} - -fn create_field_function( - fn_name: &syn::Ident, - dtype_fn_name: &syn::Ident, -) -> proc_macro2::TokenStream { - let map_field_name = get_field_name(fn_name); - let inputs = get_inputs(); - - quote! ( - #[no_mangle] - pub unsafe extern "C" fn #map_field_name( - field: *mut polars_core::export::arrow::ffi::ArrowSchema, - len: usize, - return_value: *mut polars_core::export::arrow::ffi::ArrowSchema, - ) { - #inputs; - - let result = #dtype_fn_name(&inputs); - - match result { - Ok(out) => { - let out = polars_core::export::arrow::ffi::export_field_to_c(&out.to_arrow()); - *return_value = out; - }, - Err(err) => { - // Set latest error, but leave return value in empty state. - pyo3_polars::derive::_update_last_error(err); - } - } - } - ) -} - -fn create_field_function_from_with_dtype( - fn_name: &syn::Ident, - dtype: syn::Ident, -) -> proc_macro2::TokenStream { - let map_field_name = get_field_name(fn_name); - let inputs = get_inputs(); - - quote! ( - #[no_mangle] - pub unsafe extern "C" fn #map_field_name( - field: *mut polars_core::export::arrow::ffi::ArrowSchema, - len: usize, - return_value: *mut polars_core::export::arrow::ffi::ArrowSchema - ) { - #inputs - - let mapper = polars_plan::dsl::FieldsMapper::new(&inputs); - let dtype = polars_core::datatypes::DataType::#dtype; - let out = mapper.with_dtype(dtype).unwrap(); - let out = polars_core::export::arrow::ffi::export_field_to_c(&out.to_arrow()); - *return_value = out; - } - ) -} - -#[proc_macro_attribute] -pub fn polars_expr(attr: TokenStream, input: TokenStream) -> TokenStream { - let ast = parse_macro_input!(input as syn::ItemFn); - - let options = parse_macro_input!(attr as attr::ExprsFunctionOptions); - let expanded_field_fn = if let Some(fn_name) = options.output_type_fn { - create_field_function(&ast.sig.ident, &fn_name) - } else if let Some(dtype) = options.output_dtype { - create_field_function_from_with_dtype(&ast.sig.ident, dtype) - } else { - panic!("didn't understand polars_expr attribute") - }; - - let expanded_expr = create_expression_function(ast); - let expanded = quote!( - #expanded_field_fn - - #expanded_expr - ); - TokenStream::from(expanded) -} diff --git a/pyo3-polars-derive/tests/01.rs b/pyo3-polars-derive/tests/01.rs deleted file mode 100644 index ae079b7..0000000 --- a/pyo3-polars-derive/tests/01.rs +++ /dev/null @@ -1,19 +0,0 @@ -use polars_core::error::PolarsResult; -use polars_core::prelude::{Field, Series}; -use polars_plan::dsl::FieldsMapper; -use pyo3_polars_derive::polars_expr; - -fn horizontal_product_output(input_fields: &[Field]) -> PolarsResult { - FieldsMapper::new(input_fields).map_to_supertype() -} - -#[polars_expr(type_func=horizontal_product_output)] -fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult { - let mut acc = series[0].clone(); - for s in &series[1..] { - acc = &acc * s - } - Ok(acc) -} - -fn main() {} diff --git a/pyo3-polars-derive/tests/02.rs b/pyo3-polars-derive/tests/02.rs deleted file mode 100644 index e3db225..0000000 --- a/pyo3-polars-derive/tests/02.rs +++ /dev/null @@ -1,14 +0,0 @@ -use polars_core::error::PolarsResult; -use polars_core::prelude::Series; -use pyo3_polars_derive::polars_expr; - -#[polars_expr(output_type=Int32)] -fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult { - let mut acc = series[0].clone(); - for s in &series[1..] { - acc = &acc * s - } - Ok(acc) -} - -fn main() {} diff --git a/pyo3-polars-derive/tests/run.rs b/pyo3-polars-derive/tests/run.rs deleted file mode 100644 index dc9bca6..0000000 --- a/pyo3-polars-derive/tests/run.rs +++ /dev/null @@ -1,6 +0,0 @@ -#[test] -fn tests() { - let t = trybuild::TestCases::new(); - t.pass("tests/01.rs"); - t.pass("tests/02.rs"); -} diff --git a/pyo3-polars/Cargo.toml b/pyo3-polars/Cargo.toml deleted file mode 100644 index c9f4a6a..0000000 --- a/pyo3-polars/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "pyo3-polars" -version = "0.7.0" -edition = "2021" -license = "MIT" -readme = "../README.md" -repository = "https://github.com/pola-rs/pyo3-polars" -description = "Expression plugins and PyO3 types for polars" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -ciborium = { version = "0.2.1", optional = true } -polars = { workspace = true, default-features = false } -polars-core = { workspace = true, default-features = false } -polars-ffi = { workspace = true, optional = true } -polars-lazy = { workspace = true, optional = true } -polars-plan = { workspace = true, optional = true } -pyo3 = "0.20.0" -pyo3-polars-derive = { version = "0.1.0", path = "../pyo3-polars-derive", optional = true } -serde = { version = "1", optional = true } -serde-pickle = { version = "1", optional = true } -thiserror = "1" - -[features] -lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "ciborium"] -derive = ["pyo3-polars-derive", "polars-plan", "polars-ffi", "serde-pickle", "serde"] diff --git a/pyo3-polars/src/derive.rs b/pyo3-polars/src/derive.rs deleted file mode 100644 index 405bc28..0000000 --- a/pyo3-polars/src/derive.rs +++ /dev/null @@ -1,30 +0,0 @@ -use polars::prelude::PolarsError; -use polars_core::error::{to_compute_err, PolarsResult}; -pub use pyo3_polars_derive::polars_expr; -use serde::Deserialize; -use std::cell::RefCell; -use std::ffi::CString; - -pub type DefaultKwargs = serde_pickle::Value; - -thread_local! { - static LAST_ERROR: RefCell = RefCell::new(CString::default()); -} - -pub unsafe fn _parse_kwargs<'a, T>(kwargs: &'a [u8]) -> PolarsResult -where - T: Deserialize<'a>, -{ - serde_pickle::from_slice(kwargs, Default::default()).map_err(to_compute_err) -} - -pub fn _update_last_error(err: PolarsError) { - let msg = format!("{}", err); - let msg = CString::new(msg).unwrap(); - LAST_ERROR.with(|prev| *prev.borrow_mut() = msg) -} - -#[no_mangle] -pub unsafe extern "C" fn get_last_error_message() -> *const std::os::raw::c_char { - LAST_ERROR.with(|prev| prev.borrow_mut().as_ptr()) -} diff --git a/pyo3-polars/src/error.rs b/pyo3-polars/src/error.rs deleted file mode 100644 index 9e38cf6..0000000 --- a/pyo3-polars/src/error.rs +++ /dev/null @@ -1,66 +0,0 @@ -use std::fmt::{Debug, Formatter}; - -use polars::prelude::PolarsError; -use pyo3::create_exception; -use pyo3::exceptions::{PyException, PyIOError, PyIndexError, PyRuntimeError, PyValueError}; -use pyo3::prelude::*; -use thiserror::Error; - -#[derive(Error)] -pub enum PyPolarsErr { - #[error(transparent)] - Polars(#[from] PolarsError), - #[error("{0}")] - Other(String), -} - -impl std::convert::From for PyErr { - fn from(err: PyPolarsErr) -> PyErr { - let default = || PyRuntimeError::new_err(format!("{:?}", &err)); - - use PyPolarsErr::*; - match &err { - Polars(err) => match err { - PolarsError::ComputeError(err) => ComputeError::new_err(err.to_string()), - PolarsError::NoData(err) => NoDataError::new_err(err.to_string()), - PolarsError::ShapeMismatch(err) => ShapeError::new_err(err.to_string()), - PolarsError::SchemaMismatch(err) => SchemaError::new_err(err.to_string()), - PolarsError::Io(err) => PyIOError::new_err(err.to_string()), - PolarsError::OutOfBounds(err) => PyIndexError::new_err(err.to_string()), - PolarsError::InvalidOperation(err) => PyValueError::new_err(err.to_string()), - PolarsError::Duplicate(err) => DuplicateError::new_err(err.to_string()), - PolarsError::ColumnNotFound(err) => ColumnNotFound::new_err(err.to_string()), - PolarsError::SchemaFieldNotFound(err) => { - SchemaFieldNotFound::new_err(err.to_string()) - } - PolarsError::StructFieldNotFound(err) => { - StructFieldNotFound::new_err(err.to_string()) - } - PolarsError::StringCacheMismatch(err) => { - StringCacheMismatchError::new_err(err.to_string()) - } - }, - _ => default(), - } - } -} - -impl Debug for PyPolarsErr { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - use PyPolarsErr::*; - match self { - Polars(err) => write!(f, "{:?}", err), - Other(err) => write!(f, "BindingsError: {:?}", err), - } - } -} - -create_exception!(exceptions, ColumnNotFound, PyException); -create_exception!(exceptions, SchemaFieldNotFound, PyException); -create_exception!(exceptions, StructFieldNotFound, PyException); -create_exception!(exceptions, ComputeError, PyException); -create_exception!(exceptions, NoDataError, PyException); -create_exception!(exceptions, ShapeError, PyException); -create_exception!(exceptions, SchemaError, PyException); -create_exception!(exceptions, DuplicateError, PyException); -create_exception!(exceptions, StringCacheMismatchError, PyException); diff --git a/pyo3-polars/src/export.rs b/pyo3-polars/src/export.rs deleted file mode 100644 index 76551b3..0000000 --- a/pyo3-polars/src/export.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub use polars_core; -pub use polars_ffi; -pub use polars_plan; diff --git a/pyo3-polars/src/ffi/mod.rs b/pyo3-polars/src/ffi/mod.rs deleted file mode 100644 index df928c0..0000000 --- a/pyo3-polars/src/ffi/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod to_py; -pub(crate) mod to_rust; diff --git a/pyo3-polars/src/ffi/to_py.rs b/pyo3-polars/src/ffi/to_py.rs deleted file mode 100644 index 9b4fa3a..0000000 --- a/pyo3-polars/src/ffi/to_py.rs +++ /dev/null @@ -1,24 +0,0 @@ -use polars::export::arrow::ffi; -use polars::prelude::{ArrayRef, ArrowField}; -use pyo3::ffi::Py_uintptr_t; -use pyo3::prelude::*; - -/// Arrow array to Python. -pub(crate) fn to_py_array(array: ArrayRef, py: Python, pyarrow: &PyModule) -> PyResult { - let schema = Box::new(ffi::export_field_to_c(&ArrowField::new( - "", - array.data_type().clone(), - true, - ))); - let array = Box::new(ffi::export_array_to_c(array)); - - let schema_ptr: *const ffi::ArrowSchema = &*schema; - let array_ptr: *const ffi::ArrowArray = &*array; - - let array = pyarrow.getattr("Array")?.call_method1( - "_import_from_c", - (array_ptr as Py_uintptr_t, schema_ptr as Py_uintptr_t), - )?; - - Ok(array.to_object(py)) -} diff --git a/pyo3-polars/src/ffi/to_rust.rs b/pyo3-polars/src/ffi/to_rust.rs deleted file mode 100644 index 27f5037..0000000 --- a/pyo3-polars/src/ffi/to_rust.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::error::PyPolarsErr; -use polars::export::arrow::ffi; -use polars::prelude::*; -use pyo3::ffi::Py_uintptr_t; -use pyo3::prelude::*; - -pub fn array_to_rust(obj: &PyAny) -> PyResult { - // prepare a pointer to receive the Array struct - let array = Box::new(ffi::ArrowArray::empty()); - let schema = Box::new(ffi::ArrowSchema::empty()); - - let array_ptr = &*array as *const ffi::ArrowArray; - let schema_ptr = &*schema as *const ffi::ArrowSchema; - - // make the conversion through PyArrow's private API - // this changes the pointer's memory and is thus unsafe. In particular, `_export_to_c` can go out of bounds - obj.call_method1( - "_export_to_c", - (array_ptr as Py_uintptr_t, schema_ptr as Py_uintptr_t), - )?; - - unsafe { - let field = ffi::import_field_from_c(schema.as_ref()).map_err(PyPolarsErr::from)?; - let array = ffi::import_array_from_c(*array, field.data_type).map_err(PyPolarsErr::from)?; - Ok(array) - } -} diff --git a/pyo3-polars/src/lib.rs b/pyo3-polars/src/lib.rs deleted file mode 100644 index 237b796..0000000 --- a/pyo3-polars/src/lib.rs +++ /dev/null @@ -1,203 +0,0 @@ -//! This crate offers a [`PySeries`] and a [`PyDataFrame`] which are simple wrapper around `Series` and `DataFrame`. The -//! advantage of these wrappers is that they can be converted to and from python as they implement `FromPyObject` and `IntoPy`. -//! -//! # Example -//! -//! From `src/lib.rs`. -//! ```rust -//! # use polars::prelude::*; -//! # use pyo3::prelude::*; -//! # use pyo3_polars::PyDataFrame; -//! -//! #[pyfunction] -//! fn my_cool_function(pydf: PyDataFrame) -> PyResult { -//! let df: DataFrame = pydf.into(); -//! let df = { -//! // some work on the dataframe here -//! todo!() -//! }; -//! -//! // wrap the dataframe and it will be automatically converted to a python polars dataframe -//! Ok(PyDataFrame(df)) -//! } -//! -//! /// A Python module implemented in Rust. -//! #[pymodule] -//! fn expression_lib(_py: Python, m: &PyModule) -> PyResult<()> { -//! m.add_function(wrap_pyfunction!(my_cool_function, m)?)?; -//! Ok(()) -//! } -//! ``` -//! -//! Compile your crate with `maturin` and then import from python. -//! -//! From `my_python_file.py`. -//! ```python -//! from expression_lib import my_cool_function -//! -//! df = pl.DataFrame({ -//! "foo": [1, 2, None], -//! "bar": ["a", None, "c"], -//! }) -//! out_df = my_cool_function(df) -//! ``` -#[cfg(feature = "derive")] -pub mod derive; -pub mod error; -#[cfg(feature = "derive")] -pub mod export; -mod ffi; - -use crate::error::PyPolarsErr; -use crate::ffi::to_py::to_py_array; -use polars::prelude::*; -use pyo3::{FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject}; - -#[cfg(feature = "lazy")] -use {polars_lazy::frame::LazyFrame, polars_plan::logical_plan::LogicalPlan}; - -#[repr(transparent)] -#[derive(Debug, Clone)] -/// A wrapper around a [`Series`] that can be converted to and from python with `pyo3`. -pub struct PySeries(pub Series); - -#[repr(transparent)] -#[derive(Debug, Clone)] -/// A wrapper around a [`DataFrame`] that can be converted to and from python with `pyo3`. -pub struct PyDataFrame(pub DataFrame); - -#[cfg(feature = "lazy")] -#[repr(transparent)] -#[derive(Clone)] -/// A wrapper around a [`DataFrame`] that can be converted to and from python with `pyo3`. -/// # Warning -/// If the [`LazyFrame`] contains in memory data, -/// such as a [`DataFrame`] this will be serialized/deserialized. -/// -/// It is recommended to only have `LazyFrame`s that scan data -/// from disk -pub struct PyLazyFrame(pub LazyFrame); - -impl From for DataFrame { - fn from(value: PyDataFrame) -> Self { - value.0 - } -} - -impl From for Series { - fn from(value: PySeries) -> Self { - value.0 - } -} - -#[cfg(feature = "lazy")] -impl From for LazyFrame { - fn from(value: PyLazyFrame) -> Self { - value.0 - } -} - -impl AsRef for PySeries { - fn as_ref(&self) -> &Series { - &self.0 - } -} - -impl AsRef for PyDataFrame { - fn as_ref(&self) -> &DataFrame { - &self.0 - } -} - -#[cfg(feature = "lazy")] -impl AsRef for PyLazyFrame { - fn as_ref(&self) -> &LazyFrame { - &self.0 - } -} - -impl<'a> FromPyObject<'a> for PySeries { - fn extract(ob: &'a PyAny) -> PyResult { - let ob = ob.call_method0("rechunk")?; - - let name = ob.getattr("name")?; - let name = name.str()?.to_str()?; - - let arr = ob.call_method0("to_arrow")?; - let arr = ffi::to_rust::array_to_rust(arr)?; - Ok(PySeries( - Series::try_from((name, arr)).map_err(PyPolarsErr::from)?, - )) - } -} - -impl<'a> FromPyObject<'a> for PyDataFrame { - fn extract(ob: &'a PyAny) -> PyResult { - let series = ob.call_method0("get_columns")?; - let n = ob.getattr("width")?.extract::()?; - let mut columns = Vec::with_capacity(n); - for pyseries in series.iter()? { - let pyseries = pyseries?; - let s = pyseries.extract::()?.0; - columns.push(s); - } - Ok(PyDataFrame(DataFrame::new_no_checks(columns))) - } -} - -#[cfg(feature = "lazy")] -impl<'a> FromPyObject<'a> for PyLazyFrame { - fn extract(ob: &'a PyAny) -> PyResult { - let s = ob.call_method0("__getstate__")?.extract::>()?; - let lp: LogicalPlan = ciborium::de::from_reader(&*s).map_err( - |e| PyPolarsErr::Other( - format!("Error when deserializing LazyFrame. This may be due to mismatched polars versions. {}", e) - ) - )?; - Ok(PyLazyFrame(LazyFrame::from(lp))) - } -} - -impl IntoPy for PySeries { - fn into_py(self, py: Python<'_>) -> PyObject { - let s = self.0.rechunk(); - let name = s.name(); - let arr = s.to_arrow(0); - let pyarrow = py.import("pyarrow").expect("pyarrow not installed"); - let polars = py.import("polars").expect("polars not installed"); - - let arg = to_py_array(arr, py, pyarrow).unwrap(); - let s = polars.call_method1("from_arrow", (arg,)).unwrap(); - let s = s.call_method1("rename", (name,)).unwrap(); - s.to_object(py) - } -} - -impl IntoPy for PyDataFrame { - fn into_py(self, py: Python<'_>) -> PyObject { - let pyseries = self - .0 - .get_columns() - .iter() - .map(|s| PySeries(s.clone()).into_py(py)) - .collect::>(); - - let polars = py.import("polars").expect("polars not installed"); - let df_object = polars.call_method1("DataFrame", (pyseries,)).unwrap(); - df_object.into_py(py) - } -} - -#[cfg(feature = "lazy")] -impl IntoPy for PyLazyFrame { - fn into_py(self, py: Python<'_>) -> PyObject { - let polars = py.import("polars").expect("polars not installed"); - let cls = polars.getattr("LazyFrame").unwrap(); - let instance = cls.call_method1("__new__", (cls,)).unwrap(); - let mut writer: Vec = vec![]; - ciborium::ser::into_writer(&self.0.logical_plan, &mut writer).unwrap(); - - instance.call_method1("__setstate__", (&*writer,)).unwrap(); - instance.into_py(py) - } -}