From 913ca39569a2a16e5641f5eccbe69de13367bb28 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 29 May 2024 08:28:27 -0400 Subject: [PATCH] Add `compute` and `internal` modules (#12) * Add compute and internal modules * implement concat * simplify return * Add to_python methods * Add README file --- arro3-compute/Cargo.lock | 1191 +++++++++++++++++ arro3-compute/Cargo.toml | 27 + .../array.rs => arro3-compute/README.md | 0 arro3-compute/pyproject.toml | 25 + .../python/arro3/compute/__init__.py | 4 + .../python/arro3/compute/_rust.pyi | 0 arro3-compute/python/arro3/compute/py.typed | 0 arro3-compute/src/concat.rs | 12 + arro3-compute/src/lib.rs | 20 + arro3-core/Cargo.lock | 8 + arro3-core/Cargo.toml | 17 +- arro3-core/src/ffi/to_python/mod.rs | 5 - arro3-core/src/ffi/to_python/schema.rs | 31 - arro3-core/src/lib.rs | 25 +- arro3-internal/Cargo.lock | 1177 ++++++++++++++++ arro3-internal/Cargo.toml | 24 + arro3-internal/README.md | 0 {arro3-core => arro3-internal}/src/array.rs | 39 +- {arro3-core => arro3-internal}/src/chunked.rs | 46 +- {arro3-core => arro3-internal}/src/error.rs | 0 .../src/ffi/from_python/array.rs | 0 .../src/ffi/from_python/chunked.rs | 0 .../src/ffi/from_python/ffi_stream.rs | 0 .../src/ffi/from_python/field.rs | 0 .../src/ffi/from_python/mod.rs | 0 .../src/ffi/from_python/record_batch.rs | 0 .../ffi/from_python/record_batch_reader.rs | 0 .../src/ffi/from_python/schema.rs | 0 .../src/ffi/from_python/table.rs | 0 .../src/ffi/from_python/utils.rs | 0 {arro3-core => arro3-internal}/src/ffi/mod.rs | 0 .../src/ffi/to_python/chunked.rs | 0 .../src/ffi/to_python/ffi_stream.rs | 0 arro3-internal/src/ffi/to_python/mod.rs | 2 + {arro3-core => arro3-internal}/src/field.rs | 20 +- .../src/interop/mod.rs | 0 .../src/interop/numpy/mod.rs | 0 .../src/interop/numpy/to_numpy.rs | 0 arro3-internal/src/lib.rs | 10 + .../src/record_batch.rs | 34 +- .../src/record_batch_reader.rs | 30 +- {arro3-core => arro3-internal}/src/schema.rs | 22 +- {arro3-core => arro3-internal}/src/table.rs | 28 +- 43 files changed, 2681 insertions(+), 116 deletions(-) create mode 100644 arro3-compute/Cargo.lock create mode 100644 arro3-compute/Cargo.toml rename arro3-core/src/ffi/to_python/array.rs => arro3-compute/README.md (100%) create mode 100644 arro3-compute/pyproject.toml create mode 100644 arro3-compute/python/arro3/compute/__init__.py rename arro3-core/src/ffi/to_python/stream.rs => arro3-compute/python/arro3/compute/_rust.pyi (100%) create mode 100644 arro3-compute/python/arro3/compute/py.typed create mode 100644 arro3-compute/src/concat.rs create mode 100644 arro3-compute/src/lib.rs delete mode 100644 arro3-core/src/ffi/to_python/mod.rs delete mode 100644 arro3-core/src/ffi/to_python/schema.rs create mode 100644 arro3-internal/Cargo.lock create mode 100644 arro3-internal/Cargo.toml create mode 100644 arro3-internal/README.md rename {arro3-core => arro3-internal}/src/array.rs (72%) rename {arro3-core => arro3-internal}/src/chunked.rs (73%) rename {arro3-core => arro3-internal}/src/error.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/array.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/chunked.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/ffi_stream.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/field.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/mod.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/record_batch.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/record_batch_reader.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/schema.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/table.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/from_python/utils.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/mod.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/to_python/chunked.rs (100%) rename {arro3-core => arro3-internal}/src/ffi/to_python/ffi_stream.rs (100%) create mode 100644 arro3-internal/src/ffi/to_python/mod.rs rename {arro3-core => arro3-internal}/src/field.rs (76%) rename {arro3-core => arro3-internal}/src/interop/mod.rs (100%) rename {arro3-core => arro3-internal}/src/interop/numpy/mod.rs (100%) rename {arro3-core => arro3-internal}/src/interop/numpy/to_numpy.rs (100%) create mode 100644 arro3-internal/src/lib.rs rename {arro3-core => arro3-internal}/src/record_batch.rs (78%) rename {arro3-core => arro3-internal}/src/record_batch_reader.rs (73%) rename {arro3-core => arro3-internal}/src/schema.rs (69%) rename {arro3-core => arro3-internal}/src/table.rs (79%) diff --git a/arro3-compute/Cargo.lock b/arro3-compute/Cargo.lock new file mode 100644 index 0000000..61e30f9 --- /dev/null +++ b/arro3-compute/Cargo.lock @@ -0,0 +1,1191 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arro3-compute" +version = "0.1.0" +dependencies = [ + "arro3-internal", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "arrow-select", + "pyo3", + "thiserror", +] + +[[package]] +name = "arro3-internal" +version = "0.1.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "numpy", + "pyo3", + "thiserror", +] + +[[package]] +name = "arrow" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", +] + +[[package]] +name = "arrow-json" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", + "hashbrown", +] + +[[package]] +name = "arrow-schema" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "arrow-select" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cc" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-targets", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "matrixmultiply" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ndarray" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "rawpointer", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "numpy" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef41cbb417ea83b30525259e30ccef6af39b31c240bda578889494c5392d331" +dependencies = [ + "half", + "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "rustc-hash", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "proc-macro2" +version = "1.0.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "2.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" + +[[package]] +name = "thiserror" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/arro3-compute/Cargo.toml b/arro3-compute/Cargo.toml new file mode 100644 index 0000000..5f7ae5c --- /dev/null +++ b/arro3-compute/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "arro3-compute" +version = "0.1.0" +authors = ["Kyle Barron "] +edition = "2021" +description = "Rust-based compute kernels for Arrow in Python." +readme = "README.md" +repository = "https://github.com/kylebarron/arro3" +license = "MIT OR Apache-2.0" +keywords = ["python", "arrow"] +categories = [] +rust-version = "1.75" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "_rust" +crate-type = ["cdylib"] + +[dependencies] +arrow-array = "51" +arrow-buffer = "51" +arrow-schema = "51" +arrow-select = "51" +arrow = { version = "51", features = ["ffi"] } +pyo3 = { version = "0.20.0", features = ["abi3-py38"] } +thiserror = "1" +arro3-internal = { path = "../arro3-internal" } diff --git a/arro3-core/src/ffi/to_python/array.rs b/arro3-compute/README.md similarity index 100% rename from arro3-core/src/ffi/to_python/array.rs rename to arro3-compute/README.md diff --git a/arro3-compute/pyproject.toml b/arro3-compute/pyproject.toml new file mode 100644 index 0000000..0c1c646 --- /dev/null +++ b/arro3-compute/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["maturin>=1.4.0,<2.0"] +build-backend = "maturin" + +[project] +name = "arro3-compute" +requires-python = ">=3.8" +dependencies = ["arro3-core"] +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[tool.maturin] +features = ["pyo3/extension-module"] +module-name = "arro3.compute._rust" +python-source = "python" + +[tool.poetry] +name = "arro3-compute" +version = "0.0.0" +description = "" +authors = [] +packages = [{ include = "python/arro3/compute" }] diff --git a/arro3-compute/python/arro3/compute/__init__.py b/arro3-compute/python/arro3/compute/__init__.py new file mode 100644 index 0000000..69bf76d --- /dev/null +++ b/arro3-compute/python/arro3/compute/__init__.py @@ -0,0 +1,4 @@ +from ._rust import * +from ._rust import ___version + +__version__: str = ___version() diff --git a/arro3-core/src/ffi/to_python/stream.rs b/arro3-compute/python/arro3/compute/_rust.pyi similarity index 100% rename from arro3-core/src/ffi/to_python/stream.rs rename to arro3-compute/python/arro3/compute/_rust.pyi diff --git a/arro3-compute/python/arro3/compute/py.typed b/arro3-compute/python/arro3/compute/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/arro3-compute/src/concat.rs b/arro3-compute/src/concat.rs new file mode 100644 index 0000000..4bc1871 --- /dev/null +++ b/arro3-compute/src/concat.rs @@ -0,0 +1,12 @@ +use arro3_internal::array::PyArray; +use arro3_internal::chunked::PyChunkedArray; +use arro3_internal::error::PyArrowResult; +use pyo3::prelude::*; + +#[pyfunction] +pub fn concat(input: PyChunkedArray) -> PyArrowResult { + let (chunks, field) = input.into_inner(); + let array_refs = chunks.iter().map(|arr| arr.as_ref()).collect::>(); + let concatted = arrow_select::concat::concat(array_refs.as_slice())?; + Python::with_gil(|py| PyArray::new(concatted, field).to_python(py)) +} diff --git a/arro3-compute/src/lib.rs b/arro3-compute/src/lib.rs new file mode 100644 index 0000000..c86523a --- /dev/null +++ b/arro3-compute/src/lib.rs @@ -0,0 +1,20 @@ +use pyo3::prelude::*; + +pub mod concat; + +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +#[pyfunction] +fn ___version() -> &'static str { + VERSION +} + +/// A Python module implemented in Rust. +#[pymodule] +fn _rust(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(___version))?; + + m.add_wrapped(wrap_pyfunction!(concat::concat))?; + + Ok(()) +} diff --git a/arro3-core/Cargo.lock b/arro3-core/Cargo.lock index ef1ca69..6804ed6 100644 --- a/arro3-core/Cargo.lock +++ b/arro3-core/Cargo.lock @@ -43,6 +43,14 @@ dependencies = [ [[package]] name = "arro3-core" version = "0.1.0" +dependencies = [ + "arro3-internal", + "pyo3", +] + +[[package]] +name = "arro3-internal" +version = "0.1.0" dependencies = [ "arrow", "arrow-array", diff --git a/arro3-core/Cargo.toml b/arro3-core/Cargo.toml index e0bfe00..28bf9cb 100644 --- a/arro3-core/Cargo.toml +++ b/arro3-core/Cargo.toml @@ -3,9 +3,9 @@ name = "arro3-core" version = "0.1.0" authors = ["Kyle Barron "] edition = "2021" -description = "Efficient, vectorized geospatial operations in Python." +description = "Core library for representing Arrow data in Python." readme = "README.md" -repository = "https://github.com/geoarrow/geoarrow-rs" +repository = "https://github.com/kylebarron/arro3" license = "MIT OR Apache-2.0" keywords = ["python", "arrow"] categories = [] @@ -17,10 +17,11 @@ name = "_rust" crate-type = ["cdylib"] [dependencies] -arrow-array = "51" -arrow-buffer = "51" -arrow-schema = "51" -arrow = { version = "51", features = ["ffi"] } +arro3-internal = { path = "../arro3-internal" } +# arrow-array = "51" +# arrow-buffer = "51" +# arrow-schema = "51" +# arrow = { version = "51", features = ["ffi"] } pyo3 = { version = "0.20.0", features = ["abi3-py38"] } -numpy = { version = "0.20", features = ["half"] } -thiserror = "1" +# numpy = { version = "0.20", features = ["half"] } +# thiserror = "1" diff --git a/arro3-core/src/ffi/to_python/mod.rs b/arro3-core/src/ffi/to_python/mod.rs deleted file mode 100644 index 58021d6..0000000 --- a/arro3-core/src/ffi/to_python/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod array; -pub mod chunked; -pub mod ffi_stream; -pub mod schema; -pub mod stream; diff --git a/arro3-core/src/ffi/to_python/schema.rs b/arro3-core/src/ffi/to_python/schema.rs deleted file mode 100644 index ea18a11..0000000 --- a/arro3-core/src/ffi/to_python/schema.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::ffi::CString; - -use arrow::ffi::FFI_ArrowSchema; -use pyo3::types::PyCapsule; -use pyo3::Python; - -use crate::error::PyArrowResult; -use crate::field::PyField; -use crate::schema::PySchema; - -pub trait ToSchemaPyCapsule { - fn to_py_capsule<'py>(&'py self, py: Python<'py>) -> PyArrowResult<&'py PyCapsule>; -} - -impl ToSchemaPyCapsule for PySchema { - fn to_py_capsule<'py>(&'py self, py: Python<'py>) -> PyArrowResult<&'py PyCapsule> { - let ffi_schema = FFI_ArrowSchema::try_from(self.as_ref())?; - let schema_capsule_name = CString::new("arrow_schema").unwrap(); - let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; - Ok(schema_capsule) - } -} - -impl ToSchemaPyCapsule for PyField { - fn to_py_capsule<'py>(&'py self, py: Python<'py>) -> PyArrowResult<&'py PyCapsule> { - let ffi_schema = FFI_ArrowSchema::try_from(self.as_ref())?; - let schema_capsule_name = CString::new("arrow_schema").unwrap(); - let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; - Ok(schema_capsule) - } -} diff --git a/arro3-core/src/lib.rs b/arro3-core/src/lib.rs index f35e46f..4153da0 100644 --- a/arro3-core/src/lib.rs +++ b/arro3-core/src/lib.rs @@ -1,16 +1,5 @@ use pyo3::prelude::*; -pub mod array; -pub mod chunked; -pub mod error; -pub mod ffi; -pub mod field; -pub mod interop; -pub mod record_batch; -pub mod record_batch_reader; -pub mod schema; -pub mod table; - const VERSION: &str = env!("CARGO_PKG_VERSION"); #[pyfunction] @@ -23,13 +12,13 @@ fn ___version() -> &'static str { fn _rust(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(___version))?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/arro3-internal/Cargo.lock b/arro3-internal/Cargo.lock new file mode 100644 index 0000000..30ecc19 --- /dev/null +++ b/arro3-internal/Cargo.lock @@ -0,0 +1,1177 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arro3-internal" +version = "0.1.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "numpy", + "pyo3", + "thiserror", +] + +[[package]] +name = "arrow" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", +] + +[[package]] +name = "arrow-json" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", + "hashbrown", +] + +[[package]] +name = "arrow-schema" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "arrow-select" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cc" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-targets", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "matrixmultiply" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ndarray" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "rawpointer", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "numpy" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef41cbb417ea83b30525259e30ccef6af39b31c240bda578889494c5392d331" +dependencies = [ + "half", + "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "rustc-hash", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "proc-macro2" +version = "1.0.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "2.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" + +[[package]] +name = "thiserror" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/arro3-internal/Cargo.toml b/arro3-internal/Cargo.toml new file mode 100644 index 0000000..bd32404 --- /dev/null +++ b/arro3-internal/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "arro3-internal" +version = "0.1.0" +authors = ["Kyle Barron "] +edition = "2021" +description = "Internal structs for representing Arrow data in Python." +readme = "README.md" +repository = "https://github.com/kylebarron/arro3" +license = "MIT OR Apache-2.0" +keywords = ["python", "arrow"] +categories = [] +rust-version = "1.75" + +[dependencies] +arrow-array = "51" +arrow-buffer = "51" +arrow-schema = "51" +arrow = { version = "51", features = ["ffi"] } +pyo3 = { version = "0.20.0", features = ["abi3-py38"] } +numpy = { version = "0.20", features = ["half"] } +thiserror = "1" + +[lib] +crate-type = ["rlib"] diff --git a/arro3-internal/README.md b/arro3-internal/README.md new file mode 100644 index 0000000..e69de29 diff --git a/arro3-core/src/array.rs b/arro3-internal/src/array.rs similarity index 72% rename from arro3-core/src/array.rs rename to arro3-internal/src/array.rs index 4bcf723..12778dc 100644 --- a/arro3-core/src/array.rs +++ b/arro3-internal/src/array.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use arrow_array::ArrayRef; use arrow_schema::FieldRef; +use pyo3::intern; use pyo3::prelude::*; use pyo3::types::{PyCapsule, PyTuple, PyType}; @@ -21,6 +22,27 @@ impl PyArray { pub fn new(array: ArrayRef, field: FieldRef) -> Self { Self { array, field } } + + pub fn array(&self) -> &ArrayRef { + &self.array + } + + pub fn field(&self) -> &FieldRef { + &self.field + } + + pub fn into_inner(self) -> (ArrayRef, FieldRef) { + (self.array, self.field) + } + + pub fn to_python(&self, py: Python) -> PyArrowResult { + let arro3_mod = py.import(intern!(py, "arro3.core"))?; + let core_obj = arro3_mod.getattr(intern!(py, "Array"))?.call_method1( + intern!(py, "from_arrow_pycapsule"), + self.__arrow_c_array__(py, None)?, + )?; + Ok(core_obj.to_object(py)) + } } #[pymethods] @@ -39,7 +61,11 @@ impl PyArray { /// For example, you can call [`pyarrow.array()`][pyarrow.array] to convert this array /// into a pyarrow array, without copying memory. #[allow(unused_variables)] - pub fn __arrow_c_array__(&self, requested_schema: Option) -> PyArrowResult { + pub fn __arrow_c_array__<'py>( + &'py self, + py: Python<'py>, + requested_schema: Option, + ) -> PyArrowResult<&'py PyTuple> { let field = &self.field; let ffi_schema = FFI_ArrowSchema::try_from(field)?; let ffi_array = FFI_ArrowArray::new(&self.array.to_data()); @@ -47,12 +73,11 @@ impl PyArray { let schema_capsule_name = CString::new("arrow_schema").unwrap(); let array_capsule_name = CString::new("arrow_array").unwrap(); - Python::with_gil(|py| { - let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; - let array_capsule = PyCapsule::new(py, ffi_array, Some(array_capsule_name))?; - let tuple = PyTuple::new(py, vec![schema_capsule, array_capsule]); - Ok(tuple.to_object(py)) - }) + let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; + let array_capsule = PyCapsule::new(py, ffi_array, Some(array_capsule_name))?; + let tuple = PyTuple::new(py, vec![schema_capsule, array_capsule]); + + Ok(tuple) } pub fn __eq__(&self, other: &PyArray) -> bool { diff --git a/arro3-core/src/chunked.rs b/arro3-internal/src/chunked.rs similarity index 73% rename from arro3-core/src/chunked.rs rename to arro3-internal/src/chunked.rs index 30366f6..11efd93 100644 --- a/arro3-core/src/chunked.rs +++ b/arro3-internal/src/chunked.rs @@ -1,12 +1,13 @@ use std::ffi::CString; -use std::sync::Arc; -use arrow_array::Array; +use arrow_array::{Array, ArrayRef}; use arrow_schema::FieldRef; use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyCapsule, PyType}; +use pyo3::types::{PyCapsule, PyTuple, PyType}; +use crate::error::PyArrowResult; use crate::ffi::from_python::ffi_stream::ArrowArrayStreamReader; use crate::ffi::from_python::utils::import_stream_pycapsule; use crate::ffi::to_python::chunked::ArrayIterator; @@ -17,14 +18,37 @@ use crate::interop::numpy::to_numpy::chunked_to_numpy; // their metadata. #[pyclass(module = "arro3.core._rust", name = "ChunkedArray", subclass)] pub struct PyChunkedArray { - chunks: Vec>, + chunks: Vec, field: FieldRef, } impl PyChunkedArray { - pub fn new(chunks: Vec>, field: FieldRef) -> Self { + pub fn new(chunks: Vec, field: FieldRef) -> Self { Self { chunks, field } } + + pub fn chunks(&self) -> &[ArrayRef] { + &self.chunks + } + + pub fn field(&self) -> &FieldRef { + &self.field + } + + pub fn into_inner(self) -> (Vec, FieldRef) { + (self.chunks, self.field) + } + + pub fn to_python(&self, py: Python) -> PyArrowResult { + let arro3_mod = py.import(intern!(py, "arro3.core"))?; + let core_obj = arro3_mod + .getattr(intern!(py, "ChunkedArray"))? + .call_method1( + intern!(py, "from_arrow_pycapsule"), + PyTuple::new(py, vec![self.__arrow_c_stream__(py, None)?]), + )?; + Ok(core_obj.to_object(py)) + } } #[pymethods] @@ -49,18 +73,18 @@ impl PyChunkedArray { /// [`pyarrow.chunked_array()`][pyarrow.chunked_array] to convert this array into a /// pyarrow array, without copying memory. #[allow(unused_variables)] - fn __arrow_c_stream__(&self, requested_schema: Option) -> PyResult { + fn __arrow_c_stream__<'py>( + &'py self, + py: Python<'py>, + requested_schema: Option, + ) -> PyResult<&'py PyCapsule> { let field = self.field.clone(); let chunks = self.chunks.clone(); let array_reader = Box::new(ArrayIterator::new(chunks.into_iter().map(Ok), field)); let ffi_stream = new_stream(array_reader); let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); - - Python::with_gil(|py| { - let stream_capsule = PyCapsule::new(py, ffi_stream, Some(stream_capsule_name))?; - Ok(stream_capsule.to_object(py)) - }) + PyCapsule::new(py, ffi_stream, Some(stream_capsule_name)) } pub fn __eq__(&self, other: &PyChunkedArray) -> bool { diff --git a/arro3-core/src/error.rs b/arro3-internal/src/error.rs similarity index 100% rename from arro3-core/src/error.rs rename to arro3-internal/src/error.rs diff --git a/arro3-core/src/ffi/from_python/array.rs b/arro3-internal/src/ffi/from_python/array.rs similarity index 100% rename from arro3-core/src/ffi/from_python/array.rs rename to arro3-internal/src/ffi/from_python/array.rs diff --git a/arro3-core/src/ffi/from_python/chunked.rs b/arro3-internal/src/ffi/from_python/chunked.rs similarity index 100% rename from arro3-core/src/ffi/from_python/chunked.rs rename to arro3-internal/src/ffi/from_python/chunked.rs diff --git a/arro3-core/src/ffi/from_python/ffi_stream.rs b/arro3-internal/src/ffi/from_python/ffi_stream.rs similarity index 100% rename from arro3-core/src/ffi/from_python/ffi_stream.rs rename to arro3-internal/src/ffi/from_python/ffi_stream.rs diff --git a/arro3-core/src/ffi/from_python/field.rs b/arro3-internal/src/ffi/from_python/field.rs similarity index 100% rename from arro3-core/src/ffi/from_python/field.rs rename to arro3-internal/src/ffi/from_python/field.rs diff --git a/arro3-core/src/ffi/from_python/mod.rs b/arro3-internal/src/ffi/from_python/mod.rs similarity index 100% rename from arro3-core/src/ffi/from_python/mod.rs rename to arro3-internal/src/ffi/from_python/mod.rs diff --git a/arro3-core/src/ffi/from_python/record_batch.rs b/arro3-internal/src/ffi/from_python/record_batch.rs similarity index 100% rename from arro3-core/src/ffi/from_python/record_batch.rs rename to arro3-internal/src/ffi/from_python/record_batch.rs diff --git a/arro3-core/src/ffi/from_python/record_batch_reader.rs b/arro3-internal/src/ffi/from_python/record_batch_reader.rs similarity index 100% rename from arro3-core/src/ffi/from_python/record_batch_reader.rs rename to arro3-internal/src/ffi/from_python/record_batch_reader.rs diff --git a/arro3-core/src/ffi/from_python/schema.rs b/arro3-internal/src/ffi/from_python/schema.rs similarity index 100% rename from arro3-core/src/ffi/from_python/schema.rs rename to arro3-internal/src/ffi/from_python/schema.rs diff --git a/arro3-core/src/ffi/from_python/table.rs b/arro3-internal/src/ffi/from_python/table.rs similarity index 100% rename from arro3-core/src/ffi/from_python/table.rs rename to arro3-internal/src/ffi/from_python/table.rs diff --git a/arro3-core/src/ffi/from_python/utils.rs b/arro3-internal/src/ffi/from_python/utils.rs similarity index 100% rename from arro3-core/src/ffi/from_python/utils.rs rename to arro3-internal/src/ffi/from_python/utils.rs diff --git a/arro3-core/src/ffi/mod.rs b/arro3-internal/src/ffi/mod.rs similarity index 100% rename from arro3-core/src/ffi/mod.rs rename to arro3-internal/src/ffi/mod.rs diff --git a/arro3-core/src/ffi/to_python/chunked.rs b/arro3-internal/src/ffi/to_python/chunked.rs similarity index 100% rename from arro3-core/src/ffi/to_python/chunked.rs rename to arro3-internal/src/ffi/to_python/chunked.rs diff --git a/arro3-core/src/ffi/to_python/ffi_stream.rs b/arro3-internal/src/ffi/to_python/ffi_stream.rs similarity index 100% rename from arro3-core/src/ffi/to_python/ffi_stream.rs rename to arro3-internal/src/ffi/to_python/ffi_stream.rs diff --git a/arro3-internal/src/ffi/to_python/mod.rs b/arro3-internal/src/ffi/to_python/mod.rs new file mode 100644 index 0000000..99698d8 --- /dev/null +++ b/arro3-internal/src/ffi/to_python/mod.rs @@ -0,0 +1,2 @@ +pub mod chunked; +pub mod ffi_stream; diff --git a/arro3-core/src/field.rs b/arro3-internal/src/field.rs similarity index 76% rename from arro3-core/src/field.rs rename to arro3-internal/src/field.rs index bd1f59d..a8cd943 100644 --- a/arro3-core/src/field.rs +++ b/arro3-internal/src/field.rs @@ -4,8 +4,9 @@ use std::sync::Arc; use arrow::ffi::FFI_ArrowSchema; use arrow_schema::{Field, FieldRef}; use pyo3::exceptions::PyTypeError; +use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyCapsule, PyType}; +use pyo3::types::{PyCapsule, PyTuple, PyType}; use crate::error::PyArrowResult; use crate::ffi::from_python::utils::import_schema_pycapsule; @@ -17,6 +18,15 @@ impl PyField { pub fn new(field: FieldRef) -> Self { Self(field) } + + pub fn to_python(&self, py: Python) -> PyArrowResult { + let arro3_mod = py.import(intern!(py, "arro3.core"))?; + let core_obj = arro3_mod.getattr(intern!(py, "Field"))?.call_method1( + intern!(py, "from_arrow_pycapsule"), + PyTuple::new(py, vec![self.__arrow_c_schema__(py)?]), + )?; + Ok(core_obj.to_object(py)) + } } impl From for FieldRef { @@ -46,14 +56,10 @@ impl PyField { /// /// For example, you can call [`pyarrow.field()`][pyarrow.field] to convert this array /// into a pyarrow field, without copying memory. - fn __arrow_c_schema__(&self) -> PyArrowResult { + fn __arrow_c_schema__<'py>(&'py self, py: Python<'py>) -> PyArrowResult<&'py PyCapsule> { let ffi_schema = FFI_ArrowSchema::try_from(self.0.as_ref())?; let schema_capsule_name = CString::new("arrow_schema").unwrap(); - - Python::with_gil(|py| { - let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; - Ok(schema_capsule.to_object(py)) - }) + Ok(PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?) } pub fn __eq__(&self, other: &PyField) -> bool { diff --git a/arro3-core/src/interop/mod.rs b/arro3-internal/src/interop/mod.rs similarity index 100% rename from arro3-core/src/interop/mod.rs rename to arro3-internal/src/interop/mod.rs diff --git a/arro3-core/src/interop/numpy/mod.rs b/arro3-internal/src/interop/numpy/mod.rs similarity index 100% rename from arro3-core/src/interop/numpy/mod.rs rename to arro3-internal/src/interop/numpy/mod.rs diff --git a/arro3-core/src/interop/numpy/to_numpy.rs b/arro3-internal/src/interop/numpy/to_numpy.rs similarity index 100% rename from arro3-core/src/interop/numpy/to_numpy.rs rename to arro3-internal/src/interop/numpy/to_numpy.rs diff --git a/arro3-internal/src/lib.rs b/arro3-internal/src/lib.rs new file mode 100644 index 0000000..a103605 --- /dev/null +++ b/arro3-internal/src/lib.rs @@ -0,0 +1,10 @@ +pub mod array; +pub mod chunked; +pub mod error; +pub mod ffi; +pub mod field; +pub mod interop; +pub mod record_batch; +pub mod record_batch_reader; +pub mod schema; +pub mod table; diff --git a/arro3-core/src/record_batch.rs b/arro3-internal/src/record_batch.rs similarity index 78% rename from arro3-core/src/record_batch.rs rename to arro3-internal/src/record_batch.rs index 63a8889..7f3266d 100644 --- a/arro3-core/src/record_batch.rs +++ b/arro3-internal/src/record_batch.rs @@ -6,6 +6,7 @@ use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use arrow_array::{Array, RecordBatch, StructArray}; use arrow_schema::{DataType, SchemaBuilder}; use pyo3::exceptions::PyValueError; +use pyo3::intern; use pyo3::prelude::*; use pyo3::types::{PyCapsule, PyTuple, PyType}; @@ -20,6 +21,17 @@ impl PyRecordBatch { pub fn new(batch: RecordBatch) -> Self { Self(batch) } + + pub fn to_python(&self, py: Python) -> PyArrowResult { + let arro3_mod = py.import(intern!(py, "arro3.core"))?; + let core_obj = arro3_mod + .getattr(intern!(py, "RecordBatch"))? + .call_method1( + intern!(py, "from_arrow_pycapsule"), + self.__arrow_c_array__(py, None)?, + )?; + Ok(core_obj.to_object(py)) + } } impl From for PyRecordBatch { @@ -34,6 +46,12 @@ impl From for RecordBatch { } } +impl AsRef for PyRecordBatch { + fn as_ref(&self) -> &RecordBatch { + &self.0 + } +} + #[pymethods] impl PyRecordBatch { /// An implementation of the [Arrow PyCapsule @@ -44,7 +62,11 @@ impl PyRecordBatch { /// For example, you can call [`pyarrow.array()`][pyarrow.array] to convert this array /// into a pyarrow array, without copying memory. #[allow(unused_variables)] - pub fn __arrow_c_array__(&self, requested_schema: Option) -> PyArrowResult { + pub fn __arrow_c_array__<'py>( + &'py self, + py: Python<'py>, + requested_schema: Option, + ) -> PyArrowResult<&'py PyTuple> { let schema = self.0.schema(); let array = StructArray::from(self.0.clone()); @@ -53,13 +75,9 @@ impl PyRecordBatch { let schema_capsule_name = CString::new("arrow_schema").unwrap(); let array_capsule_name = CString::new("arrow_array").unwrap(); - - Python::with_gil(|py| { - let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; - let array_capsule = PyCapsule::new(py, ffi_array, Some(array_capsule_name))?; - let tuple = PyTuple::new(py, vec![schema_capsule, array_capsule]); - Ok(tuple.to_object(py)) - }) + let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; + let array_capsule = PyCapsule::new(py, ffi_array, Some(array_capsule_name))?; + Ok(PyTuple::new(py, vec![schema_capsule, array_capsule])) } pub fn __eq__(&self, other: &PyRecordBatch) -> bool { diff --git a/arro3-core/src/record_batch_reader.rs b/arro3-internal/src/record_batch_reader.rs similarity index 73% rename from arro3-core/src/record_batch_reader.rs rename to arro3-internal/src/record_batch_reader.rs index 32509fd..3c6038d 100644 --- a/arro3-core/src/record_batch_reader.rs +++ b/arro3-internal/src/record_batch_reader.rs @@ -3,8 +3,9 @@ use std::ffi::CString; use arrow::ffi_stream::FFI_ArrowArrayStream; use arrow_array::RecordBatchReader; use pyo3::exceptions::{PyIOError, PyValueError}; +use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyCapsule, PyType}; +use pyo3::types::{PyCapsule, PyTuple, PyType}; use crate::error::PyArrowResult; use crate::ffi::from_python::utils::import_stream_pycapsule; @@ -21,6 +22,17 @@ impl PyRecordBatchReader { .ok_or(PyIOError::new_err("Cannot write from closed stream."))?; Ok(stream) } + + pub fn to_python(&mut self, py: Python) -> PyArrowResult { + let arro3_mod = py.import(intern!(py, "arro3.core"))?; + let core_obj = arro3_mod + .getattr(intern!(py, "RecordBatchReader"))? + .call_method1( + intern!(py, "from_arrow_pycapsule"), + PyTuple::new(py, vec![self.__arrow_c_stream__(py, None)?]), + )?; + Ok(core_obj.to_object(py)) + } } #[pymethods] @@ -32,10 +44,12 @@ impl PyRecordBatchReader { /// /// For example, you can call [`pyarrow.table()`][pyarrow.table] to convert this array /// into a pyarrow table, without copying memory. - fn __arrow_c_stream__( - &mut self, - _requested_schema: Option, - ) -> PyArrowResult { + #[allow(unused_variables)] + fn __arrow_c_stream__<'py>( + &'py mut self, + py: Python<'py>, + requested_schema: Option, + ) -> PyResult<&'py PyCapsule> { let reader = self .0 .take() @@ -43,11 +57,7 @@ impl PyRecordBatchReader { let ffi_stream = FFI_ArrowArrayStream::new(reader); let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); - - Python::with_gil(|py| { - let stream_capsule = PyCapsule::new(py, ffi_stream, Some(stream_capsule_name))?; - Ok(stream_capsule.to_object(py)) - }) + PyCapsule::new(py, ffi_stream, Some(stream_capsule_name)) } #[classmethod] diff --git a/arro3-core/src/schema.rs b/arro3-internal/src/schema.rs similarity index 69% rename from arro3-core/src/schema.rs rename to arro3-internal/src/schema.rs index 1745999..951e103 100644 --- a/arro3-core/src/schema.rs +++ b/arro3-internal/src/schema.rs @@ -1,13 +1,15 @@ +use std::ffi::CString; use std::sync::Arc; +use arrow::ffi::FFI_ArrowSchema; use arrow_schema::{Schema, SchemaRef}; use pyo3::exceptions::PyTypeError; +use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyCapsule, PyType}; +use pyo3::types::{PyCapsule, PyTuple, PyType}; use crate::error::PyArrowResult; use crate::ffi::from_python::utils::import_schema_pycapsule; -use crate::ffi::to_python::schema::ToSchemaPyCapsule; #[pyclass(module = "arro3.core._rust", name = "Schema", subclass)] pub struct PySchema(SchemaRef); @@ -16,6 +18,15 @@ impl PySchema { pub fn new(schema: SchemaRef) -> Self { Self(schema) } + + pub fn to_python(&self, py: Python) -> PyArrowResult { + let arro3_mod = py.import(intern!(py, "arro3.core"))?; + let core_obj = arro3_mod.getattr(intern!(py, "Schema"))?.call_method1( + intern!(py, "from_arrow_pycapsule"), + PyTuple::new(py, vec![self.__arrow_c_schema__(py)?]), + )?; + Ok(core_obj.to_object(py)) + } } impl From for SchemaRef { @@ -45,8 +56,11 @@ impl PySchema { /// /// For example, you can call [`pyarrow.schema()`][pyarrow.schema] to convert this array /// into a pyarrow schema, without copying memory. - fn __arrow_c_schema__(&self, py: Python) -> PyArrowResult { - Ok(self.to_py_capsule(py)?.to_object(py)) + fn __arrow_c_schema__<'py>(&'py self, py: Python<'py>) -> PyArrowResult<&'py PyCapsule> { + let ffi_schema = FFI_ArrowSchema::try_from(self.as_ref())?; + let schema_capsule_name = CString::new("arrow_schema").unwrap(); + let schema_capsule = PyCapsule::new(py, ffi_schema, Some(schema_capsule_name))?; + Ok(schema_capsule) } /// Construct this object from existing Arrow data diff --git a/arro3-core/src/table.rs b/arro3-internal/src/table.rs similarity index 79% rename from arro3-core/src/table.rs rename to arro3-internal/src/table.rs index f9de788..5423b8a 100644 --- a/arro3-core/src/table.rs +++ b/arro3-internal/src/table.rs @@ -6,8 +6,9 @@ use arrow_array::RecordBatchReader; use arrow_array::{RecordBatch, RecordBatchIterator}; use arrow_schema::SchemaRef; use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyCapsule, PyType}; +use pyo3::types::{PyCapsule, PyTuple, PyType}; use crate::error::PyArrowResult; use crate::ffi::from_python::utils::import_stream_pycapsule; @@ -24,9 +25,22 @@ impl PyTable { Self { schema, batches } } + pub fn batches(&self) -> &[RecordBatch] { + &self.batches + } + pub fn into_inner(self) -> (Vec, SchemaRef) { (self.batches, self.schema) } + + pub fn to_python(&self, py: Python) -> PyArrowResult { + let arro3_mod = py.import(intern!(py, "arro3.core"))?; + let core_obj = arro3_mod.getattr(intern!(py, "Table"))?.call_method1( + intern!(py, "from_arrow_pycapsule"), + PyTuple::new(py, vec![self.__arrow_c_stream__(py, None)?]), + )?; + Ok(core_obj.to_object(py)) + } } #[pymethods] @@ -39,7 +53,11 @@ impl PyTable { /// For example, you can call [`pyarrow.table()`][pyarrow.table] to convert this array /// into a pyarrow table, without copying memory. #[allow(unused_variables)] - fn __arrow_c_stream__(&self, requested_schema: Option) -> PyArrowResult { + fn __arrow_c_stream__<'py>( + &'py self, + py: Python<'py>, + requested_schema: Option, + ) -> PyResult<&'py PyCapsule> { let batches = self.batches.clone(); let record_batch_reader = Box::new(RecordBatchIterator::new( @@ -49,11 +67,7 @@ impl PyTable { let ffi_stream = FFI_ArrowArrayStream::new(record_batch_reader); let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); - - Python::with_gil(|py| { - let stream_capsule = PyCapsule::new(py, ffi_stream, Some(stream_capsule_name))?; - Ok(stream_capsule.to_object(py)) - }) + PyCapsule::new(py, ffi_stream, Some(stream_capsule_name)) } pub fn __eq__(&self, other: &PyTable) -> bool {