diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8935329c6f..b671a5a65df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,6 +137,7 @@ jobs: run: | cargo update -p indexmap --precise 1.6.2 cargo update -p hashbrown:0.11.2 --precise 0.9.1 + cargo update -p bitflags --precise 1.2.1 - name: Build docs run: cargo doc --no-deps --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}" diff --git a/.github/workflows/guide.yml b/.github/workflows/guide.yml index fce804bd4a4..9f67ea72d46 100644 --- a/.github/workflows/guide.yml +++ b/.github/workflows/guide.yml @@ -50,7 +50,7 @@ jobs: - name: Deploy if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - uses: peaceiris/actions-gh-pages@v3.7.0-8 + uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./gh-pages-build/ @@ -71,7 +71,7 @@ jobs: ln -sfT $TAG_NAME public/latest - name: Deploy - uses: peaceiris/actions-gh-pages@v3.7.0-8 + uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./public/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 08b11089d70..ad4a980e6cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,29 @@ PyO3 versions, please see the [migration guide](https://pyo3.rs/latest/migration The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [0.14.3] - 2021-08-22 + +### Added + +- Add `PyString::data()` to access the raw bytes stored in a Python string. [#1794](https://github.com/PyO3/pyo3/issues/1794) + +### Fixed + +- Raise `AttributeError` to avoid panic when calling `del` on a `#[setter]` defined class property. [#1779](https://github.com/PyO3/pyo3/issues/1779) +- Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) +- Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) +- Reduce lower bound `num-complex` optional dependency to support interop with `rust-numpy` and `ndarray` when building with the MSRV of 1.41 [#1799](https://github.com/PyO3/pyo3/pull/1799) +- Fix memory leak in `Python::run_code`. [#1806](https://github.com/PyO3/pyo3/pull/1806) +- Fix memory leak in `PyModule::from_code`. [#1810](https://github.com/PyO3/pyo3/pull/1810) +- Remove use of `pyo3::` in `pyo3::types::datetime` which broke builds using `-Z avoid-dev-deps` [#1811](https://github.com/PyO3/pyo3/issues/1811) + ## [0.14.2] - 2021-08-09 ### Added - Add `indexmap` feature to add `ToPyObject`, `IntoPy` and `FromPyObject` implementations for `indexmap::IndexMap`. [#1728](https://github.com/PyO3/pyo3/pull/1728) - Add `pyo3_build_config::add_extension_module_link_args()` to use in build scripts to set linker arguments (for macOS). [#1755](https://github.com/PyO3/pyo3/pull/1755) -- Add `Python::with_gil_unchecked()` unsafe variation of `Python::with_gil()` to allow obtaining a `Python` in scenarios where `Python::with_gil()` would fail. +- Add `Python::with_gil_unchecked()` unsafe variation of `Python::with_gil()` to allow obtaining a `Python` in scenarios where `Python::with_gil()` would fail. [#1769](https://github.com/PyO3/pyo3/pull/1769) ### Changed @@ -880,7 +896,8 @@ Yanked - Initial release -[unreleased]: https://github.com/pyo3/pyo3/compare/v0.14.1...HEAD +[unreleased]: https://github.com/pyo3/pyo3/compare/v0.14.3...HEAD +[0.14.3]: https://github.com/pyo3/pyo3/compare/v0.14.2...v0.14.3 [0.14.2]: https://github.com/pyo3/pyo3/compare/v0.14.1...v0.14.2 [0.14.1]: https://github.com/pyo3/pyo3/compare/v0.14.0...v0.14.1 [0.14.0]: https://github.com/pyo3/pyo3/compare/v0.13.2...v0.14.0 diff --git a/Cargo.toml b/Cargo.toml index 3667f11bf2e..386de8df9eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.14.2" +version = "0.14.3" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -21,10 +21,10 @@ inventory = { version = "0.1.4", optional = true } libc = "0.2.62" parking_lot = "0.11.0" num-bigint = { version = "0.4", optional = true } -num-complex = { version = "0.4", optional = true } +num-complex = { version = ">= 0.2, < 0.5", optional = true } # must stay at 0.1.x for Rust 1.41 compatibility paste = { version = "0.1.18", optional = true } -pyo3-macros = { path = "pyo3-macros", version = "=0.14.2", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.14.3", optional = true } unindent = { version = "0.1.4", optional = true } hashbrown = { version = ">= 0.9, < 0.12", optional = true } indexmap = { version = ">= 1.6, < 1.8", optional = true } @@ -42,7 +42,7 @@ pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initi serde_json = "1.0.61" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.14.2" } +pyo3-build-config = { path = "pyo3-build-config", version = "0.14.3" } [features] default = ["macros"] diff --git a/README.md b/README.md index faef74a32f7..ce14db4969a 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies.pyo3] -version = "0.14.2" +version = "0.14.3" features = ["extension-module"] ``` @@ -108,7 +108,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.14.2" +version = "0.14.3" features = ["auto-initialize"] ``` diff --git a/examples/word-count/src/lib.rs b/examples/word-count/src/lib.rs index 1a078c6bdae..96f9f9e2fae 100644 --- a/examples/word-count/src/lib.rs +++ b/examples/word-count/src/lib.rs @@ -1,6 +1,3 @@ -// Source adopted from -// https://github.com/tildeio/helix-website/blob/master/crates/word_count/src/lib.rs - use pyo3::prelude::*; use rayon::prelude::*; @@ -24,28 +21,11 @@ fn search_sequential_allow_threads(py: Python, contents: &str, needle: &str) -> py.allow_threads(|| search_sequential(contents, needle)) } -fn matches(word: &str, needle: &str) -> bool { - let mut needle = needle.chars(); - for ch in word.chars().skip_while(|ch| !ch.is_alphabetic()) { - match needle.next() { - None => { - return !ch.is_alphabetic(); - } - Some(expect) => { - if ch.to_lowercase().next() != Some(expect) { - return false; - } - } - } - } - needle.next().is_none() -} - -/// Count the occurences of needle in line, case insensitive +/// Count the occurrences of needle in line, case insensitive fn count_line(line: &str, needle: &str) -> usize { let mut total = 0; for word in line.split(' ') { - if matches(word, needle) { + if word == needle { total += 1; } } diff --git a/examples/word-count/tests/test_word_count.py b/examples/word-count/tests/test_word_count.py index cb633622deb..5991e4ae1de 100644 --- a/examples/word-count/tests/test_word_count.py +++ b/examples/word-count/tests/test_word_count.py @@ -1,7 +1,6 @@ from concurrent.futures import ThreadPoolExecutor import pytest - import word_count diff --git a/examples/word-count/word_count/__init__.py b/examples/word-count/word_count/__init__.py index 8998f1e4d4b..8ce7a175471 100644 --- a/examples/word-count/word_count/__init__.py +++ b/examples/word-count/word_count/__init__.py @@ -8,11 +8,10 @@ ] -def search_py(contents, needle): +def search_py(contents: str, needle: str) -> int: total = 0 - for line in contents.split(): - words = line.split(" ") - for word in words: + for line in contents.splitlines(): + for word in line.split(" "): if word == needle: total += 1 return total diff --git a/guide/src/class.md b/guide/src/class.md index 3b5d3509cfb..27e662a7a26 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -454,6 +454,10 @@ impl MyClass { In this case, the property `number` is defined and available from Python code as `self.number`. +Attributes defined by `#[setter]` or `#[pyo3(set)]` will always raise `AttributeError` on `del` +operations. Support for defining custom `del` behavior is tracked in +[#1778](https://github.com/PyO3/pyo3/issues/1778). + ## Instance methods To define a Python compatible method, an `impl` block for your struct has to be annotated with the diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index 8daadf38e51..7c92ea4641a 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -1,14 +1,14 @@ # Async / Await -If you are working with a Python library that makes use of async functions or wish to provide +If you are working with a Python library that makes use of async functions or wish to provide Python bindings for an async Rust library, [`pyo3-asyncio`](https://github.com/awestlake87/pyo3-asyncio) -likely has the tools you need. It provides conversions between async functions in both Python and -Rust and was designed with first-class support for popular Rust runtimes such as -[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python -code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing +likely has the tools you need. It provides conversions between async functions in both Python and +Rust and was designed with first-class support for popular Rust runtimes such as +[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python +code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing Python libraries. -In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call +In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call async Python functions with PyO3, how to call async Rust functions from Python, and how to configure your codebase to manage the runtimes of both. @@ -163,13 +163,13 @@ maturin develop && python3 🔗 Found pyo3 bindings 🐍 Found CPython 3.8 at python3 Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.5 (default, Jan 27 2021, 15:41:15) +Python 3.8.5 (default, Jan 27 2021, 15:41:15) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import asyncio >>> >>> from my_async_module import rust_sleep ->>> +>>> >>> async def main(): >>> await rust_sleep() >>> @@ -188,19 +188,21 @@ async def py_sleep(): await asyncio.sleep(1) ``` -**Async functions in Python are simply functions that return a `coroutine` object**. For our purposes, +**Async functions in Python are simply functions that return a `coroutine` object**. For our purposes, we really don't need to know much about these `coroutine` objects. The key factor here is that calling an `async` function is _just like calling a regular function_, the only difference is that we have to do something special with the object that it returns. -Normally in Python, that something special is the `await` keyword, but in order to await this -coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. -That's where `pyo3-asyncio` comes in. -[`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) -performs this conversion for us: +Normally in Python, that something special is the `await` keyword, but in order to await this +coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. +That's where `pyo3-asyncio` comes in. +[`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) +performs this conversion for us. +The following example uses `into_future` to call the `py_sleep` function shown above and then await the +coroutine object returned from the call: -```rust no_run +```rust use pyo3::prelude::*; #[pyo3_asyncio::tokio::main] @@ -209,11 +211,11 @@ async fn main() -> PyResult<()> { // import the module containing the py_sleep function let example = py.import("example")?; - // calling the py_sleep method like a normal function + // calling the py_sleep method like a normal function // returns a coroutine let coroutine = example.call_method0("py_sleep")?; - // convert the coroutine into a Rust future using the + // convert the coroutine into a Rust future using the // tokio runtime pyo3_asyncio::tokio::into_future(coroutine) })?; @@ -225,12 +227,67 @@ async fn main() -> PyResult<()> { } ``` -> If you're interested in learning more about `coroutines` and `awaitables` in general, check out the +Alternatively, the below example shows how to write a `#[pyfunction]` which uses `into_future` to receive and await +a coroutine argument: + +```rust +#[pyfunction] +fn await_coro(coro: &PyAny) -> PyResult<()> { + // convert the coroutine into a Rust future using the + // async_std runtime + let f = pyo3_asyncio::async_std::into_future(coro)?; + + pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { + // await the future + f.await?; + Ok(()) + }) +} +``` + +This could be called from Python as: + +```python +import asyncio + +async def py_sleep(): + asyncio.sleep(1) + +await_coro(py_sleep()) +``` + +If for you wanted to pass a callable function to the `#[pyfunction]` instead, (i.e. the last line becomes `await_coro(py_sleep))`, then the above example needs to be tweaked to first call the callable to get the coroutine: + +```rust +#[pyfunction] +fn await_coro(callable: &PyAny) -> PyResult<()> { + // get the coroutine by calling the callable + let coro = callable.call0()?; + + // convert the coroutine into a Rust future using the + // async_std runtime + let f = pyo3_asyncio::async_std::into_future(coro)?; + + pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { + // await the future + f.await?; + Ok(()) + }) +} +``` + +This can be particularly helpful where you need to repeatedly create and await a coroutine. Trying to await the same coroutine multiple times will raise an error: + +```python +RuntimeError: cannot reuse already awaited coroutine +``` + +> If you're interested in learning more about `coroutines` and `awaitables` in general, check out the > [Python 3 `asyncio` docs](https://docs.python.org/3/library/asyncio-task.html) for more information. ## Awaiting a Rust Future in Python -Here we have the same async function as before written in Rust using the +Here we have the same async function as before written in Rust using the [`async-std`](https://async.rs/) runtime: ```rust @@ -243,12 +300,12 @@ async fn rust_sleep() { Similar to Python, Rust's async functions also return a special object called a `Future`: -```rust compile_fail +```rust let future = rust_sleep(); ``` -We can convert this `Future` object into Python to make it `awaitable`. This tells Python that you -can use the `await` keyword with it. In order to do this, we'll call +We can convert this `Future` object into Python to make it `awaitable`. This tells Python that you +can use the `await` keyword with it. In order to do this, we'll call [`pyo3_asyncio::async_std::future_into_py`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.future_into_py.html): ```rust @@ -284,38 +341,38 @@ doesn't always play well with Rust. Luckily, Rust's event loops are pretty flexible and don't _need_ control over the main thread, so in `pyo3-asyncio`, we decided the best way to handle Rust/Python interop was to just surrender the main -thread to Python and run Rust's event loops in the background. Unfortunately, since most event loop +thread to Python and run Rust's event loops in the background. Unfortunately, since most event loop implementations _prefer_ control over the main thread, this can still make some things awkward. ### PyO3 Asyncio Initialization Because Python needs to control the main thread, we can't use the convenient proc macros from Rust -runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main +runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main thread must block on [`pyo3_asyncio::run_forever`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.run_forever.html) or [`pyo3_asyncio::async_std::run_until_complete`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.run_until_complete.html). Because we have to block on one of those functions, we can't use [`#[async_std::main]`](https://docs.rs/async-std/latest/async_std/attr.main.html) or [`#[tokio::main]`](https://docs.rs/tokio/1.1.0/tokio/attr.main.html) since it's not a good idea to make long blocking calls during an async function. > Internally, these `#[main]` proc macros are expanded to something like this: -> ```rust compile_fail +> ```rust > fn main() { > // your async main fn > async fn _main_impl() { /* ... */ } -> Runtime::new().block_on(_main_impl()); +> Runtime::new().block_on(_main_impl()); > } > ``` > Making a long blocking call inside the `Future` that's being driven by `block_on` prevents that -> thread from doing anything else and can spell trouble for some runtimes (also this will actually -> deadlock a single-threaded runtime!). Many runtimes have some sort of `spawn_blocking` mechanism -> that can avoid this problem, but again that's not something we can use here since we need it to +> thread from doing anything else and can spell trouble for some runtimes (also this will actually +> deadlock a single-threaded runtime!). Many runtimes have some sort of `spawn_blocking` mechanism +> that can avoid this problem, but again that's not something we can use here since we need it to > block on the _main_ thread. -For this reason, `pyo3-asyncio` provides its own set of proc macros to provide you with this -initialization. These macros are intended to mirror the initialization of `async-std` and `tokio` +For this reason, `pyo3-asyncio` provides its own set of proc macros to provide you with this +initialization. These macros are intended to mirror the initialization of `async-std` and `tokio` while also satisfying the Python runtime's needs. Here's a full example of PyO3 initialization with the `async-std` runtime: -```rust no_run +```rust use pyo3::prelude::*; #[pyo3_asyncio::async_std::main] @@ -429,19 +486,19 @@ $ maturin develop && python3 🔗 Found pyo3 bindings 🐍 Found CPython 3.8 at python3 Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.8 (default, Apr 13 2021, 19:58:26) +Python 3.8.8 (default, Apr 13 2021, 19:58:26) [GCC 7.3.0] :: Anaconda, Inc. on linux Type "help", "copyright", "credits" or "license" for more information. >>> import asyncio >>> import uvloop ->>> +>>> >>> import my_async_module ->>> +>>> >>> uvloop.install() ->>> +>>> >>> async def main(): ... await my_async_module.rust_sleep() -... +... >>> asyncio.run(main()) >>> ``` @@ -498,4 +555,4 @@ fn main() -> PyResult<()> { ## Additional Information - Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://docs.rs/pyo3-asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. -- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/testing) \ No newline at end of file +- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing) diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index a94bbe70a94..dc1de0d482c 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.14.2" +version = "0.14.3" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index d9ee19ab98b..7b743b3a420 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -45,14 +45,60 @@ pub fn env_var(var: &str) -> Option { /// strategies are used to populate this type. #[cfg_attr(test, derive(Debug, PartialEq))] pub struct InterpreterConfig { + /// The Python implementation flavor. + /// + /// Serialized to `implementation`. pub implementation: PythonImplementation, + + /// Python `X.Y` version. e.g. `3.9`. + /// + /// Serialized to `version`. pub version: PythonVersion, + + /// Whether link library is shared. + /// + /// Serialized to `shared`. pub shared: bool, + + /// Whether linking against the stable/limited Python 3 API. + /// + /// Serialized to `abi3`. pub abi3: bool, + + /// The name of the link library defining Python. + /// + /// This effectively controls the `cargo:rustc-link-lib=` value to + /// control how libpython is linked. Values should not contain the `lib` + /// prefix. + /// + /// Serialized to `lib_name`. pub lib_name: Option, + + /// The directory containing the Python library to link against. + /// + /// The effectively controls the `cargo:rustc-link-search=native=` value + /// to add an additional library search path for the linker. + /// + /// Serialized to `lib_dir`. pub lib_dir: Option, + + /// Path of host `python` executable. + /// + /// This is a valid executable capable of running on the host/building machine. + /// For configurations derived by invoking a Python interpreter, it was the + /// executable invoked. + /// + /// Serialized to `executable`. pub executable: Option, + + /// Width in bits of pointers on the target machine. + /// + /// Serialized to `pointer_width`. pub pointer_width: Option, + + /// Additional relevant Python build flags / configuration settings. + /// + /// Serialized to `build_flags`. pub build_flags: BuildFlags, } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 6f109c80dfa..ebe108984aa 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.14.2" +version = "0.14.3" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,7 +16,7 @@ edition = "2018" [dependencies] quote = { version = "1", default-features = false } proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "0.14.2" } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.14.3" } [dependencies.syn] version = "1" diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 09fad20596b..603f8df6428 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -133,21 +133,21 @@ impl SelfType { match self { SelfType::Receiver { mutable: false } => { quote! { - let _cell = _py.from_borrowed_ptr::>(_slf); + let _cell = _py.from_borrowed_ptr::<::pyo3::PyCell<#cls>>(_slf); let _ref = _cell.try_borrow()?; let _slf = &_ref; } } SelfType::Receiver { mutable: true } => { quote! { - let _cell = _py.from_borrowed_ptr::>(_slf); + let _cell = _py.from_borrowed_ptr::<::pyo3::PyCell<#cls>>(_slf); let mut _ref = _cell.try_borrow_mut()?; let _slf = &mut _ref; } } SelfType::TryFromPyCell(span) => { quote_spanned! { *span => - let _cell = _py.from_borrowed_ptr::>(_slf); + let _cell = _py.from_borrowed_ptr::<::pyo3::PyCell<#cls>>(_slf); #[allow(clippy::useless_conversion)] // In case _slf is PyCell let _slf = std::convert::TryFrom::try_from(_cell)?; } diff --git a/pyo3-macros-backend/src/proto_method.rs b/pyo3-macros-backend/src/proto_method.rs index 3c14843fd78..8d8569b0834 100644 --- a/pyo3-macros-backend/src/proto_method.rs +++ b/pyo3-macros-backend/src/proto_method.rs @@ -95,7 +95,7 @@ pub(crate) fn impl_method_proto( }; Ok(quote! { - impl<'p> #module::#proto<'p> for #cls { + impl<'p> ::#module::#proto<'p> for #cls { #(#impl_types)* #res_type_def } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 14996df8a2d..de80c2ccf41 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -47,7 +47,7 @@ impl Default for PyClassArgs { freelist: None, name: None, module: None, - base: parse_quote! { pyo3::PyAny }, + base: parse_quote! { ::pyo3::PyAny }, has_dict: false, has_weaklist: false, is_gc: false, @@ -345,22 +345,22 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream { quote! { #[doc(hidden)] pub struct #inventory_cls { - methods: Vec, + methods: ::std::vec::Vec<::pyo3::class::PyMethodDefType>, } - impl pyo3::class::impl_::PyMethodsInventory for #inventory_cls { - fn new(methods: Vec) -> Self { + impl ::pyo3::class::impl_::PyMethodsInventory for #inventory_cls { + fn new(methods: ::std::vec::Vec<::pyo3::class::PyMethodDefType>) -> Self { Self { methods } } - fn get(&'static self) -> &'static [pyo3::class::PyMethodDefType] { + fn get(&'static self) -> &'static [::pyo3::class::PyMethodDefType] { &self.methods } } - impl pyo3::class::impl_::HasMethodsInventory for #cls { + impl ::pyo3::class::impl_::HasMethodsInventory for #cls { type Methods = #inventory_cls; } - pyo3::inventory::collect!(#inventory_cls); + ::pyo3::inventory::collect!(#inventory_cls); } } @@ -380,31 +380,31 @@ fn impl_class( let alloc = attr.freelist.as_ref().map(|freelist| { quote! { - impl pyo3::class::impl_::PyClassWithFreeList for #cls { + impl ::pyo3::class::impl_::PyClassWithFreeList for #cls { #[inline] - fn get_free_list(_py: pyo3::Python<'_>) -> &mut pyo3::impl_::freelist::FreeList<*mut pyo3::ffi::PyObject> { - static mut FREELIST: *mut pyo3::impl_::freelist::FreeList<*mut pyo3::ffi::PyObject> = 0 as *mut _; + fn get_free_list(_py: ::pyo3::Python<'_>) -> &mut ::pyo3::impl_::freelist::FreeList<*mut ::pyo3::ffi::PyObject> { + static mut FREELIST: *mut ::pyo3::impl_::freelist::FreeList<*mut ::pyo3::ffi::PyObject> = 0 as *mut _; unsafe { if FREELIST.is_null() { - FREELIST = Box::into_raw(Box::new( - pyo3::impl_::freelist::FreeList::with_capacity(#freelist))); + FREELIST = ::std::boxed::Box::into_raw(::std::boxed::Box::new( + ::pyo3::impl_::freelist::FreeList::with_capacity(#freelist))); } &mut *FREELIST } } } - impl pyo3::class::impl_::PyClassAllocImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassAllocImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { #[inline] - fn alloc_impl(self) -> Option { - Some(pyo3::class::impl_::alloc_with_freelist::<#cls>) + fn alloc_impl(self) -> ::std::option::Option<::pyo3::ffi::allocfunc> { + ::std::option::Option::Some(::pyo3::class::impl_::alloc_with_freelist::<#cls>) } } - impl pyo3::class::impl_::PyClassFreeImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassFreeImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { #[inline] - fn free_impl(self) -> Option { - Some(pyo3::class::impl_::free_with_freelist::<#cls>) + fn free_impl(self) -> ::std::option::Option<::pyo3::ffi::freefunc> { + ::std::option::Option::Some(::pyo3::class::impl_::free_with_freelist::<#cls>) } } } @@ -414,23 +414,23 @@ fn impl_class( // insert space for weak ref let weakref = if attr.has_weaklist { - quote! { pyo3::pyclass_slots::PyClassWeakRefSlot } + quote! { ::pyo3::pyclass_slots::PyClassWeakRefSlot } } else if attr.has_extends { - quote! { ::WeakRef } + quote! { ::WeakRef } } else { - quote! { pyo3::pyclass_slots::PyClassDummySlot } + quote! { ::pyo3::pyclass_slots::PyClassDummySlot } }; let dict = if attr.has_dict { - quote! { pyo3::pyclass_slots::PyClassDictSlot } + quote! { ::pyo3::pyclass_slots::PyClassDictSlot } } else if attr.has_extends { - quote! { ::Dict } + quote! { ::Dict } } else { - quote! { pyo3::pyclass_slots::PyClassDummySlot } + quote! { ::pyo3::pyclass_slots::PyClassDummySlot } }; let module = if let Some(m) = &attr.module { - quote! { Some(#m) } + quote! { ::std::option::Option::Some(#m) } } else { - quote! { None } + quote! { ::std::option::Option::None } }; // Enforce at compile time that PyGCProtocol is implemented @@ -439,9 +439,9 @@ fn impl_class( let closure_token = syn::Ident::new(&closure_name, Span::call_site()); quote! { fn #closure_token() { - use pyo3::class; + use ::pyo3::class; - fn _assert_implements_protocol<'p, T: pyo3::class::PyGCProtocol<'p>>() {} + fn _assert_implements_protocol<'p, T: ::pyo3::class::PyGCProtocol<'p>>() {} _assert_implements_protocol::<#cls>(); } } @@ -450,12 +450,15 @@ fn impl_class( }; let (impl_inventory, for_each_py_method) = match methods_type { - PyClassMethodsType::Specialization => (None, quote! { visitor(collector.py_methods()); }), + PyClassMethodsType::Specialization => ( + ::std::option::Option::None, + quote! { visitor(collector.py_methods()); }, + ), PyClassMethodsType::Inventory => ( Some(impl_methods_inventory(cls)), quote! { - for inventory in pyo3::inventory::iter::<::Methods>() { - visitor(pyo3::class::impl_::PyMethodsInventory::get(inventory)); + for inventory in ::pyo3::inventory::iter::<::Methods>() { + visitor(::pyo3::class::impl_::PyMethodsInventory::get(inventory)); } }, ), @@ -463,17 +466,17 @@ fn impl_class( let base = &attr.base; let base_nativetype = if attr.has_extends { - quote! { ::BaseNativeType } + quote! { ::BaseNativeType } } else { - quote! { pyo3::PyAny } + quote! { ::pyo3::PyAny } }; // If #cls is not extended type, we allow Self->PyObject conversion let into_pyobject = if !attr.has_extends { quote! { - impl pyo3::IntoPy for #cls { - fn into_py(self, py: pyo3::Python) -> pyo3::PyObject { - pyo3::IntoPy::into_py(pyo3::Py::new(py, self).unwrap(), py) + impl ::pyo3::IntoPy<::pyo3::PyObject> for #cls { + fn into_py(self, py: ::pyo3::Python) -> ::pyo3::PyObject { + ::pyo3::IntoPy::into_py(::pyo3::Py::new(py, self).unwrap(), py) } } } @@ -482,13 +485,13 @@ fn impl_class( }; let thread_checker = if attr.has_unsendable { - quote! { pyo3::class::impl_::ThreadCheckerImpl<#cls> } + quote! { ::pyo3::class::impl_::ThreadCheckerImpl<#cls> } } else if attr.has_extends { quote! { - pyo3::class::impl_::ThreadCheckerInherited<#cls, <#cls as pyo3::class::impl_::PyClassImpl>::BaseType> + ::pyo3::class::impl_::ThreadCheckerInherited<#cls, <#cls as ::pyo3::class::impl_::PyClassImpl>::BaseType> } } else { - quote! { pyo3::class::impl_::ThreadCheckerStub<#cls> } + quote! { ::pyo3::class::impl_::ThreadCheckerStub<#cls> } }; let is_gc = attr.is_gc; @@ -496,54 +499,54 @@ fn impl_class( let is_subclass = attr.has_extends; Ok(quote! { - unsafe impl pyo3::type_object::PyTypeInfo for #cls { - type AsRefTarget = pyo3::PyCell; + unsafe impl ::pyo3::type_object::PyTypeInfo for #cls { + type AsRefTarget = ::pyo3::PyCell; const NAME: &'static str = #cls_name; - const MODULE: Option<&'static str> = #module; + const MODULE: ::std::option::Option<&'static str> = #module; #[inline] - fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject { + fn type_object_raw(py: ::pyo3::Python<'_>) -> *mut ::pyo3::ffi::PyTypeObject { #deprecations - use pyo3::type_object::LazyStaticType; + use ::pyo3::type_object::LazyStaticType; static TYPE_OBJECT: LazyStaticType = LazyStaticType::new(); TYPE_OBJECT.get_or_init::(py) } } - impl pyo3::PyClass for #cls { + impl ::pyo3::PyClass for #cls { type Dict = #dict; type WeakRef = #weakref; type BaseNativeType = #base_nativetype; } - impl<'a> pyo3::derive_utils::ExtractExt<'a> for &'a #cls + impl<'a> ::pyo3::derive_utils::ExtractExt<'a> for &'a #cls { - type Target = pyo3::PyRef<'a, #cls>; + type Target = ::pyo3::PyRef<'a, #cls>; } - impl<'a> pyo3::derive_utils::ExtractExt<'a> for &'a mut #cls + impl<'a> ::pyo3::derive_utils::ExtractExt<'a> for &'a mut #cls { - type Target = pyo3::PyRefMut<'a, #cls>; + type Target = ::pyo3::PyRefMut<'a, #cls>; } #into_pyobject #impl_inventory - impl pyo3::class::impl_::PyClassImpl for #cls { + impl ::pyo3::class::impl_::PyClassImpl for #cls { const DOC: &'static str = #doc; const IS_GC: bool = #is_gc; const IS_BASETYPE: bool = #is_basetype; const IS_SUBCLASS: bool = #is_subclass; - type Layout = pyo3::PyCell; + type Layout = ::pyo3::PyCell; type BaseType = #base; type ThreadChecker = #thread_checker; - fn for_each_method_def(visitor: &mut dyn FnMut(&[pyo3::class::PyMethodDefType])) { - use pyo3::class::impl_::*; + fn for_each_method_def(visitor: &mut dyn ::std::ops::FnMut(&[::pyo3::class::PyMethodDefType])) { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); #for_each_py_method; visitor(collector.py_class_descriptors()); @@ -554,30 +557,30 @@ fn impl_class( visitor(collector.mapping_protocol_methods()); visitor(collector.number_protocol_methods()); } - fn get_new() -> Option { - use pyo3::class::impl_::*; + fn get_new() -> ::std::option::Option<::pyo3::ffi::newfunc> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.new_impl() } - fn get_alloc() -> Option { - use pyo3::class::impl_::*; + fn get_alloc() -> ::std::option::Option<::pyo3::ffi::allocfunc> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.alloc_impl() } - fn get_free() -> Option { - use pyo3::class::impl_::*; + fn get_free() -> ::std::option::Option<::pyo3::ffi::freefunc> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.free_impl() } - fn get_call() -> Option { - use pyo3::class::impl_::*; + fn get_call() -> ::std::option::Option<::pyo3::ffi::PyCFunctionWithKeywords> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.call_impl() } - fn for_each_proto_slot(visitor: &mut dyn FnMut(&[pyo3::ffi::PyType_Slot])) { + fn for_each_proto_slot(visitor: &mut dyn ::std::ops::FnMut(&[::pyo3::ffi::PyType_Slot])) { // Implementation which uses dtolnay specialization to load all slots. - use pyo3::class::impl_::*; + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); visitor(collector.object_protocol_slots()); visitor(collector.number_protocol_slots()); @@ -590,8 +593,8 @@ fn impl_class( visitor(collector.buffer_protocol_slots()); } - fn get_buffer() -> Option<&'static pyo3::class::impl_::PyBufferProcs> { - use pyo3::class::impl_::*; + fn get_buffer() -> ::std::option::Option<&'static ::pyo3::class::impl_::PyBufferProcs> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.buffer_procs() } @@ -645,11 +648,11 @@ fn impl_descriptors( .collect::>()?; Ok(quote! { - impl pyo3::class::impl_::PyClassDescriptors<#cls> - for pyo3::class::impl_::PyClassImplCollector<#cls> + impl ::pyo3::class::impl_::PyClassDescriptors<#cls> + for ::pyo3::class::impl_::PyClassImplCollector<#cls> { - fn py_class_descriptors(self) -> &'static [pyo3::class::methods::PyMethodDefType] { - static METHODS: &[pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; + fn py_class_descriptors(self) -> &'static [::pyo3::class::methods::PyMethodDefType] { + static METHODS: &[::pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; METHODS } } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 82b70fe665a..0991be14998 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -102,7 +102,7 @@ pub fn impl_py_method_def( }; let methoddef = spec.get_methoddef(quote! {{ #wrapper_def #wrapper_ident }}); Ok(quote! { - pyo3::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) + ::pyo3::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) }) } @@ -110,7 +110,7 @@ fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec) -> Result let wrapper_ident = syn::Ident::new("__wrap", Span::call_site()); let wrapper = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; Ok(quote! { - impl pyo3::class::impl_::PyClassNewImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassNewImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { fn new_impl(self) -> Option { Some({ #wrapper @@ -125,7 +125,7 @@ fn impl_py_method_def_call(cls: &syn::Type, spec: &FnSpec) -> Result for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassCallImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { fn call_impl(self) -> Option { Some({ #wrapper @@ -141,13 +141,13 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec) -> TokenStream { let deprecations = &spec.deprecations; let python_name = spec.null_terminated_python_name(); quote! { - pyo3::class::PyMethodDefType::ClassAttribute({ - pyo3::class::PyClassAttributeDef::new( + ::pyo3::class::PyMethodDefType::ClassAttribute({ + ::pyo3::class::PyClassAttributeDef::new( #python_name, - pyo3::class::methods::PyClassAttributeFactory({ - fn __wrap(py: pyo3::Python<'_>) -> pyo3::PyObject { + ::pyo3::class::methods::PyClassAttributeFactory({ + fn __wrap(py: ::pyo3::Python<'_>) -> ::pyo3::PyObject { #deprecations - pyo3::IntoPy::into_py(#cls::#name(), py) + ::pyo3::IntoPy::into_py(#cls::#name(), py) } __wrap }) @@ -206,22 +206,26 @@ pub fn impl_py_setter_def(cls: &syn::Type, property_type: PropertyType) -> Resul PropertyType::Function { self_type, .. } => self_type.receiver(cls), }; Ok(quote! { - pyo3::class::PyMethodDefType::Setter({ + ::pyo3::class::PyMethodDefType::Setter({ #deprecations - pyo3::class::PySetterDef::new( + ::pyo3::class::PySetterDef::new( #python_name, - pyo3::class::methods::PySetter({ + ::pyo3::class::methods::PySetter({ unsafe extern "C" fn __wrap( - _slf: *mut pyo3::ffi::PyObject, - _value: *mut pyo3::ffi::PyObject, - _: *mut std::os::raw::c_void - ) -> std::os::raw::c_int { - pyo3::callback::handle_panic(|_py| { + _slf: *mut ::pyo3::ffi::PyObject, + _value: *mut ::pyo3::ffi::PyObject, + _: *mut ::std::os::raw::c_void + ) -> ::std::os::raw::c_int { + ::pyo3::callback::handle_panic(|_py| { #slf - let _value = _py.from_borrowed_ptr::(_value); - let _val = pyo3::FromPyObject::extract(_value)?; + let _value = _py + .from_borrowed_ptr_or_opt(_value) + .ok_or_else(|| { + ::pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") + })?; + let _val = ::pyo3::FromPyObject::extract(_value)?; - pyo3::callback::convert(_py, #setter_impl) + ::pyo3::callback::convert(_py, #setter_impl) }) } __wrap @@ -262,12 +266,13 @@ pub fn impl_py_getter_def(cls: &syn::Type, property_type: PropertyType) -> Resul .. } => { // named struct field - quote!(_slf.#ident.clone()) + //quote!(_slf.#ident.clone()) + quote!(::std::clone::Clone::clone(&(_slf.#ident))) } PropertyType::Descriptor { field_index, .. } => { // tuple struct field let index = syn::Index::from(field_index); - quote!(_slf.#index.clone()) + quote!(::std::clone::Clone::clone(&(_slf.#index))) } PropertyType::Function { spec, .. } => impl_call_getter(cls, spec)?, }; @@ -277,18 +282,18 @@ pub fn impl_py_getter_def(cls: &syn::Type, property_type: PropertyType) -> Resul PropertyType::Function { self_type, .. } => self_type.receiver(cls), }; Ok(quote! { - pyo3::class::PyMethodDefType::Getter({ + ::pyo3::class::PyMethodDefType::Getter({ #deprecations - pyo3::class::PyGetterDef::new( + ::pyo3::class::PyGetterDef::new( #python_name, - pyo3::class::methods::PyGetter({ + ::pyo3::class::methods::PyGetter({ unsafe extern "C" fn __wrap( - _slf: *mut pyo3::ffi::PyObject, - _: *mut std::os::raw::c_void - ) -> *mut pyo3::ffi::PyObject { - pyo3::callback::handle_panic(|_py| { + _slf: *mut ::pyo3::ffi::PyObject, + _: *mut ::std::os::raw::c_void + ) -> *mut ::pyo3::ffi::PyObject { + ::pyo3::callback::handle_panic(|_py| { #slf - pyo3::callback::convert(_py, #getter_impl) + ::pyo3::callback::convert(_py, #getter_impl) }) } __wrap diff --git a/pyo3-macros-backend/src/pyproto.rs b/pyo3-macros-backend/src/pyproto.rs index 8080622916f..a3943d6ae76 100644 --- a/pyo3-macros-backend/src/pyproto.rs +++ b/pyo3-macros-backend/src/pyproto.rs @@ -68,7 +68,7 @@ fn impl_proto_impl( let flags = if m.can_coexist { // We need METH_COEXIST here to prevent __add__ from overriding __radd__ - Some(quote!(pyo3::ffi::METH_COEXIST)) + Some(quote!(::pyo3::ffi::METH_COEXIST)) } else { None }; @@ -106,11 +106,11 @@ fn impl_normal_methods( let methods_trait = proto.methods_trait(); let methods_trait_methods = proto.methods_trait_methods(); quote! { - impl pyo3::class::impl_::#methods_trait<#ty> - for pyo3::class::impl_::PyClassImplCollector<#ty> + impl ::pyo3::class::impl_::#methods_trait<#ty> + for ::pyo3::class::impl_::PyClassImplCollector<#ty> { - fn #methods_trait_methods(self) -> &'static [pyo3::class::methods::PyMethodDefType] { - static METHODS: &[pyo3::class::methods::PyMethodDefType] = + fn #methods_trait_methods(self) -> &'static [::pyo3::class::methods::PyMethodDefType] { + static METHODS: &[::pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; METHODS } @@ -139,14 +139,14 @@ fn impl_proto_methods( if build_config.version <= PY39 && proto.name == "Buffer" { maybe_buffer_methods = Some(quote! { - impl pyo3::class::impl_::PyBufferProtocolProcs<#ty> - for pyo3::class::impl_::PyClassImplCollector<#ty> + impl ::pyo3::class::impl_::PyBufferProtocolProcs<#ty> + for ::pyo3::class::impl_::PyClassImplCollector<#ty> { fn buffer_procs( self - ) -> Option<&'static pyo3::class::impl_::PyBufferProcs> { - static PROCS: pyo3::class::impl_::PyBufferProcs - = pyo3::class::impl_::PyBufferProcs { + ) -> Option<&'static ::pyo3::class::impl_::PyBufferProcs> { + static PROCS: ::pyo3::class::impl_::PyBufferProcs + = ::pyo3::class::impl_::PyBufferProcs { bf_getbuffer: Some(pyo3::class::buffer::getbuffer::<#ty>), bf_releasebuffer: Some(pyo3::class::buffer::releasebuffer::<#ty>), }; @@ -162,9 +162,9 @@ fn impl_proto_methods( let slot = syn::Ident::new(def.slot, Span::call_site()); let slot_impl = syn::Ident::new(def.slot_impl, Span::call_site()); quote! {{ - pyo3::ffi::PyType_Slot { - slot: pyo3::ffi::#slot, - pfunc: #module::#slot_impl::<#ty> as _ + ::pyo3::ffi::PyType_Slot { + slot: ::pyo3::ffi::#slot, + pfunc: ::#module::#slot_impl::<#ty> as _ } }} }) @@ -177,10 +177,10 @@ fn impl_proto_methods( quote! { #maybe_buffer_methods - impl pyo3::class::impl_::#slots_trait<#ty> - for pyo3::class::impl_::PyClassImplCollector<#ty> + impl ::pyo3::class::impl_::#slots_trait<#ty> + for ::pyo3::class::impl_::PyClassImplCollector<#ty> { - fn #slots_trait_slots(self) -> &'static [pyo3::ffi::PyType_Slot] { + fn #slots_trait_slots(self) -> &'static [::pyo3::ffi::PyType_Slot] { &[#(#tokens),*] } } diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index e6d2e0cfe33..a14589977ff 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.14.2" +version = "0.14.3" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,4 +16,4 @@ proc-macro = true [dependencies] quote = "1" syn = { version = "1", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.14.2" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.14.3" } diff --git a/src/ffi/cpython/ceval.rs b/src/ffi/cpython/ceval.rs index 13173b1b146..29affbd72bb 100644 --- a/src/ffi/cpython/ceval.rs +++ b/src/ffi/cpython/ceval.rs @@ -1,5 +1,4 @@ -use crate::ffi::object::{freefunc, PyObject}; -use crate::ffi::pystate::Py_tracefunc; +use crate::ffi::{freefunc, PyObject, Py_tracefunc}; use std::os::raw::c_int; extern "C" { diff --git a/src/ffi/cpython/initconfig.rs b/src/ffi/cpython/initconfig.rs index ee6fc75b53d..95d28b02bf3 100644 --- a/src/ffi/cpython/initconfig.rs +++ b/src/ffi/cpython/initconfig.rs @@ -4,6 +4,7 @@ use crate::ffi::Py_ssize_t; use libc::wchar_t; use std::os::raw::{c_char, c_int, c_ulong}; +#[repr(C)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum _PyStatus_TYPE { _PyStatus_TYPE_OK = 0, @@ -14,6 +15,7 @@ pub enum _PyStatus_TYPE { #[repr(C)] #[derive(Copy, Clone)] pub struct PyStatus { + pub _type: _PyStatus_TYPE, pub func: *const c_char, pub err_msg: *const c_char, pub exitcode: c_int, diff --git a/src/ffi/cpython/mod.rs b/src/ffi/cpython/mod.rs index 7e8c9c6aec0..b1701d1794e 100644 --- a/src/ffi/cpython/mod.rs +++ b/src/ffi/cpython/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod abstract_; // skipped bytearrayobject.h #[cfg(not(PyPy))] pub(crate) mod bytesobject; +#[cfg(not(PyPy))] pub(crate) mod ceval; pub(crate) mod code; pub(crate) mod compile; @@ -18,13 +19,13 @@ pub(crate) mod object; pub(crate) mod pydebug; #[cfg(all(Py_3_8, not(PyPy)))] pub(crate) mod pylifecycle; - -#[cfg(all(Py_3_8, not(PyPy)))] pub(crate) mod pystate; +pub(crate) mod unicodeobject; pub use self::abstract_::*; #[cfg(not(PyPy))] pub use self::bytesobject::*; +#[cfg(not(PyPy))] pub use self::ceval::*; pub use self::code::*; pub use self::compile::*; @@ -39,6 +40,5 @@ pub use self::object::*; pub use self::pydebug::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::pylifecycle::*; - -#[cfg(all(Py_3_8, not(PyPy)))] pub use self::pystate::*; +pub use self::unicodeobject::*; diff --git a/src/ffi/cpython/pystate.rs b/src/ffi/cpython/pystate.rs index 38491baeb31..7d2f9abf6a6 100644 --- a/src/ffi/cpython/pystate.rs +++ b/src/ffi/cpython/pystate.rs @@ -1,7 +1,19 @@ -use crate::ffi::pystate::{PyInterpreterState, PyThreadState}; +#[cfg(not(PyPy))] +use crate::ffi::PyThreadState; +use crate::ffi::{PyFrameObject, PyInterpreterState, PyObject}; use std::os::raw::c_int; -// Py_tracefunc is defined in ffi::pystate +// skipped _PyInterpreterState_RequiresIDRef +// skipped _PyInterpreterState_RequireIDRef + +// skipped _PyInterpreterState_GetMainModule + +pub type Py_tracefunc = extern "C" fn( + obj: *mut PyObject, + frame: *mut PyFrameObject, + what: c_int, + arg: *mut PyObject, +) -> c_int; pub const PyTrace_CALL: c_int = 0; pub const PyTrace_EXCEPTION: c_int = 1; @@ -12,13 +24,38 @@ pub const PyTrace_C_EXCEPTION: c_int = 5; pub const PyTrace_C_RETURN: c_int = 6; pub const PyTrace_OPCODE: c_int = 7; +// skipped PyTraceInfo +// skipped CFrame +// skipped _PyErr_StackItem +// skipped _PyStackChunk +// skipped _ts (aka PyThreadState) + extern "C" { - // PyGILState_Check is defined in ffi::pystate + // skipped _PyThreadState_Prealloc + // skipped _PyThreadState_UncheckedGet + // skipped _PyThreadState_GetDict + + #[cfg_attr(PyPy, link_name = "PyPyGILState_Check")] + pub fn PyGILState_Check() -> c_int; + + // skipped _PyGILState_GetInterpreterStateUnsafe + // skipped _PyThread_CurrentFrames + // skipped _PyThread_CurrentExceptions + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_Main() -> *mut PyInterpreterState; + #[cfg_attr(PyPy, link_name = "PyPyInterpreterState_Head")] pub fn PyInterpreterState_Head() -> *mut PyInterpreterState; + #[cfg_attr(PyPy, link_name = "PyPyInterpreterState_Next")] pub fn PyInterpreterState_Next(interp: *mut PyInterpreterState) -> *mut PyInterpreterState; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_ThreadHead(interp: *mut PyInterpreterState) -> *mut PyThreadState; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyThreadState_Next(tstate: *mut PyThreadState) -> *mut PyThreadState; + // skipped PyThreadState_DeleteCurrent } #[cfg(Py_3_9)] @@ -44,3 +81,17 @@ extern "C" { eval_frame: _PyFrameEvalFunction, ); } + +// skipped _PyInterpreterState_GetConfig +// skipped _PyInterpreterState_GetConfigCopy +// skipped _PyInterpreterState_SetConfig +// skipped _Py_GetConfig + +// skipped _PyCrossInterpreterData +// skipped _PyObject_GetCrossInterpreterData +// skipped _PyCrossInterpreterData_NewObject +// skipped _PyCrossInterpreterData_Release +// skipped _PyObject_CheckCrossInterpreterData +// skipped crossinterpdatafunc +// skipped _PyCrossInterpreterData_RegisterClass +// skipped _PyCrossInterpreterData_Lookup diff --git a/src/ffi/cpython/unicodeobject.rs b/src/ffi/cpython/unicodeobject.rs new file mode 100644 index 00000000000..7c61243513b --- /dev/null +++ b/src/ffi/cpython/unicodeobject.rs @@ -0,0 +1,605 @@ +use crate::ffi::{ + PyObject, PyUnicode_Check, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_hash_t, Py_ssize_t, +}; +use libc::wchar_t; +use std::os::raw::{c_char, c_int, c_uint, c_void}; + +// skipped Py_UNICODE_ISSPACE() +// skipped Py_UNICODE_ISLOWER() +// skipped Py_UNICODE_ISUPPER() +// skipped Py_UNICODE_ISTITLE() +// skipped Py_UNICODE_ISLINEBREAK +// skipped Py_UNICODE_TOLOWER +// skipped Py_UNICODE_TOUPPER +// skipped Py_UNICODE_TOTITLE +// skipped Py_UNICODE_ISDECIMAL +// skipped Py_UNICODE_ISDIGIT +// skipped Py_UNICODE_ISNUMERIC +// skipped Py_UNICODE_ISPRINTABLE +// skipped Py_UNICODE_TODECIMAL +// skipped Py_UNICODE_TODIGIT +// skipped Py_UNICODE_TONUMERIC +// skipped Py_UNICODE_ISALPHA +// skipped Py_UNICODE_ISALNUM +// skipped Py_UNICODE_COPY +// skipped Py_UNICODE_FILL +// skipped Py_UNICODE_IS_SURROGATE +// skipped Py_UNICODE_IS_HIGH_SURROGATE +// skipped Py_UNICODE_IS_LOW_SURROGATE +// skipped Py_UNICODE_JOIN_SURROGATES +// skipped Py_UNICODE_HIGH_SURROGATE +// skipped Py_UNICODE_LOW_SURROGATE + +#[repr(C)] +pub struct PyASCIIObject { + pub ob_base: PyObject, + pub length: Py_ssize_t, + pub hash: Py_hash_t, + /// A bit field with various properties. + /// + /// Rust doesn't expose bitfields. So we have accessor functions for + /// retrieving values. + /// + /// unsigned int interned:2; // SSTATE_* constants. + /// unsigned int kind:3; // PyUnicode_*_KIND constants. + /// unsigned int compact:1; + /// unsigned int ascii:1; + /// unsigned int ready:1; + /// unsigned int :24; + pub state: u32, + pub wstr: *mut wchar_t, +} + +impl PyASCIIObject { + #[inline] + pub fn interned(&self) -> c_uint { + self.state & 3 + } + + #[inline] + pub fn kind(&self) -> c_uint { + (self.state >> 2) & 7 + } + + #[inline] + pub fn compact(&self) -> c_uint { + (self.state >> 5) & 1 + } + + #[inline] + pub fn ascii(&self) -> c_uint { + (self.state >> 6) & 1 + } + + #[inline] + pub fn ready(&self) -> c_uint { + (self.state >> 7) & 1 + } +} + +#[repr(C)] +pub struct PyCompactUnicodeObject { + pub _base: PyASCIIObject, + pub utf8_length: Py_ssize_t, + pub utf8: *mut c_char, + pub wstr_length: Py_ssize_t, +} + +#[repr(C)] +pub union PyUnicodeObjectData { + any: *mut c_void, + latin1: *mut Py_UCS1, + ucs2: *mut Py_UCS2, + ucs4: *mut Py_UCS4, +} + +#[repr(C)] +pub struct PyUnicodeObject { + pub _base: PyCompactUnicodeObject, + pub data: PyUnicodeObjectData, +} + +extern "C" { + #[cfg(not(PyPy))] + pub fn _PyUnicode_CheckConsistency(op: *mut PyObject, check_content: c_int) -> c_int; +} + +// skipped PyUnicode_GET_SIZE +// skipped PyUnicode_GET_DATA_SIZE +// skipped PyUnicode_AS_UNICODE +// skipped PyUnicode_AS_DATA + +pub const SSTATE_NOT_INTERNED: c_uint = 0; +pub const SSTATE_INTERNED_MORTAL: c_uint = 1; +pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; + +#[inline] +pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { + debug_assert!(PyUnicode_Check(op) != 0); + debug_assert!(PyUnicode_IS_READY(op) != 0); + + (*(op as *mut PyASCIIObject)).ascii() +} + +#[inline] +pub unsafe fn PyUnicode_IS_COMPACT(op: *mut PyObject) -> c_uint { + (*(op as *mut PyASCIIObject)).compact() +} + +#[inline] +pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { + if (*(op as *mut PyASCIIObject)).ascii() != 0 && PyUnicode_IS_COMPACT(op) != 0 { + 1 + } else { + 0 + } +} + +#[cfg(not(Py_3_12))] +#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] +pub const PyUnicode_WCHAR_KIND: c_uint = 0; + +pub const PyUnicode_1BYTE_KIND: c_uint = 1; +pub const PyUnicode_2BYTE_KIND: c_uint = 2; +pub const PyUnicode_4BYTE_KIND: c_uint = 4; + +#[inline] +pub unsafe fn PyUnicode_1BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS1 { + PyUnicode_DATA(op) as *mut Py_UCS1 +} + +#[inline] +pub unsafe fn PyUnicode_2BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS2 { + PyUnicode_DATA(op) as *mut Py_UCS2 +} + +#[inline] +pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { + PyUnicode_DATA(op) as *mut Py_UCS4 +} + +#[inline] +pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { + debug_assert!(PyUnicode_Check(op) != 0); + debug_assert!(PyUnicode_IS_READY(op) != 0); + + (*(op as *mut PyASCIIObject)).kind() +} + +#[inline] +pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { + if PyUnicode_IS_ASCII(op) != 0 { + (op as *mut PyASCIIObject).offset(1) as *mut c_void + } else { + (op as *mut PyCompactUnicodeObject).offset(1) as *mut c_void + } +} + +#[inline] +pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { + debug_assert!(!(*(op as *mut PyUnicodeObject)).data.any.is_null()); + + (*(op as *mut PyUnicodeObject)).data.any +} + +#[inline] +pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { + debug_assert!(PyUnicode_Check(op) != 0); + + if PyUnicode_IS_COMPACT(op) != 0 { + _PyUnicode_COMPACT_DATA(op) + } else { + _PyUnicode_NONCOMPACT_DATA(op) + } +} + +// skipped PyUnicode_WRITE +// skipped PyUnicode_READ +// skipped PyUnicode_READ_CHAR + +#[inline] +pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { + debug_assert!(PyUnicode_Check(op) != 0); + debug_assert!(PyUnicode_IS_READY(op) != 0); + + (*(op as *mut PyASCIIObject)).length +} + +#[inline] +pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { + (*(op as *mut PyASCIIObject)).ready() +} + +#[cfg(not(Py_3_12))] +#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] +#[inline] +pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { + debug_assert!(PyUnicode_Check(op) != 0); + + if PyUnicode_IS_READY(op) != 0 { + 0 + } else { + _PyUnicode_Ready(op) + } +} + +// skipped PyUnicode_MAX_CHAR_VALUE +// skipped _PyUnicode_get_wstr_length +// skipped PyUnicode_WSTR_LENGTH + +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_New")] + pub fn PyUnicode_New(size: Py_ssize_t, maxchar: Py_UCS4) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "_PyPyUnicode_Ready")] + pub fn _PyUnicode_Ready(unicode: *mut PyObject) -> c_int; + + // skipped _PyUnicode_Copy + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] + pub fn PyUnicode_CopyCharacters( + to: *mut PyObject, + to_start: Py_ssize_t, + from: *mut PyObject, + from_start: Py_ssize_t, + how_many: Py_ssize_t, + ) -> Py_ssize_t; + + // skipped _PyUnicode_FastCopyCharacters + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] + pub fn PyUnicode_Fill( + unicode: *mut PyObject, + start: Py_ssize_t, + length: Py_ssize_t, + fill_char: Py_UCS4, + ) -> Py_ssize_t; + + // skipped _PyUnicode_FastFill + + #[cfg(not(Py_3_12))] + #[deprecated] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromUnicode")] + pub fn PyUnicode_FromUnicode(u: *const Py_UNICODE, size: Py_ssize_t) -> *mut PyObject; + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromKindAndData")] + pub fn PyUnicode_FromKindAndData( + kind: c_int, + buffer: *const c_void, + size: Py_ssize_t, + ) -> *mut PyObject; + + // skipped _PyUnicode_FromASCII + // skipped _PyUnicode_FindMaxChar + + #[cfg(not(Py_3_12))] + #[deprecated] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicode")] + pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut Py_UNICODE; + + // skipped _PyUnicode_AsUnicode + + #[cfg(not(Py_3_12))] + #[deprecated] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicodeAndSize")] + pub fn PyUnicode_AsUnicodeAndSize( + unicode: *mut PyObject, + size: *mut Py_ssize_t, + ) -> *mut Py_UNICODE; + + // skipped PyUnicode_GetMax +} + +// skipped _PyUnicodeWriter +// skipped _PyUnicodeWriter_Init +// skipped _PyUnicodeWriter_Prepare +// skipped _PyUnicodeWriter_PrepareInternal +// skipped _PyUnicodeWriter_PrepareKind +// skipped _PyUnicodeWriter_PrepareKindInternal +// skipped _PyUnicodeWriter_WriteChar +// skipped _PyUnicodeWriter_WriteStr +// skipped _PyUnicodeWriter_WriteSubstring +// skipped _PyUnicodeWriter_WriteASCIIString +// skipped _PyUnicodeWriter_WriteLatin1String +// skipped _PyUnicodeWriter_Finish +// skipped _PyUnicodeWriter_Dealloc +// skipped _PyUnicode_FormatAdvancedWriter + +extern "C" { + #[cfg(Py_3_7)] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] + pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *const c_char; + + #[cfg(not(Py_3_7))] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] + pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *mut c_char; + + // skipped _PyUnicode_AsStringAndSize + + #[cfg(Py_3_7)] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] + pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *const c_char; + + #[cfg(not(Py_3_7))] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] + pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *mut c_char; + + // skipped _PyUnicode_AsString + + pub fn PyUnicode_Encode( + s: *const Py_UNICODE, + size: Py_ssize_t, + encoding: *const c_char, + errors: *const c_char, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeUTF7( + data: *const Py_UNICODE, + length: Py_ssize_t, + base64SetO: c_int, + base64WhiteSpace: c_int, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeUTF7 + // skipped _PyUnicode_AsUTF8String + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeUTF8")] + pub fn PyUnicode_EncodeUTF8( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeUTF32( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + byteorder: c_int, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeUTF32 + + pub fn PyUnicode_EncodeUTF16( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + byteorder: c_int, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeUTF16 + // skipped _PyUnicode_DecodeUnicodeEscape + + pub fn PyUnicode_EncodeUnicodeEscape( + data: *const Py_UNICODE, + length: Py_ssize_t, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeRawUnicodeEscape( + data: *const Py_UNICODE, + length: Py_ssize_t, + ) -> *mut PyObject; + + // skipped _PyUnicode_AsLatin1String + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeLatin1")] + pub fn PyUnicode_EncodeLatin1( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped _PyUnicode_AsASCIIString + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeASCII")] + pub fn PyUnicode_EncodeASCII( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeCharmap( + data: *const Py_UNICODE, + length: Py_ssize_t, + mapping: *mut PyObject, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeCharmap + + pub fn PyUnicode_TranslateCharmap( + data: *const Py_UNICODE, + length: Py_ssize_t, + table: *mut PyObject, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped PyUnicode_EncodeMBCS + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeDecimal")] + pub fn PyUnicode_EncodeDecimal( + s: *mut Py_UNICODE, + length: Py_ssize_t, + output: *mut c_char, + errors: *const c_char, + ) -> c_int; + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_TransformDecimalToASCII")] + pub fn PyUnicode_TransformDecimalToASCII( + s: *mut Py_UNICODE, + length: Py_ssize_t, + ) -> *mut PyObject; + + // skipped _PyUnicode_TransformDecimalAndSpaceToASCII +} + +// skipped _PyUnicode_JoinArray +// skipped _PyUnicode_EqualToASCIIId +// skipped _PyUnicode_EqualToASCIIString +// skipped _PyUnicode_XStrip +// skipped _PyUnicode_InsertThousandsGrouping + +// skipped _Py_ascii_whitespace + +// skipped _PyUnicode_IsLowercase +// skipped _PyUnicode_IsUppercase +// skipped _PyUnicode_IsTitlecase +// skipped _PyUnicode_IsXidStart +// skipped _PyUnicode_IsXidContinue +// skipped _PyUnicode_IsWhitespace +// skipped _PyUnicode_IsLinebreak +// skipped _PyUnicode_ToLowercase +// skipped _PyUnicode_ToUppercase +// skipped _PyUnicode_ToTitlecase +// skipped _PyUnicode_ToLowerFull +// skipped _PyUnicode_ToTitleFull +// skipped _PyUnicode_ToUpperFull +// skipped _PyUnicode_ToFoldedFull +// skipped _PyUnicode_IsCaseIgnorable +// skipped _PyUnicode_IsCased +// skipped _PyUnicode_ToDecimalDigit +// skipped _PyUnicode_ToDigit +// skipped _PyUnicode_ToNumeric +// skipped _PyUnicode_IsDecimalDigit +// skipped _PyUnicode_IsDigit +// skipped _PyUnicode_IsNumeric +// skipped _PyUnicode_IsPrintable +// skipped _PyUnicode_IsAlpha +// skipped Py_UNICODE_strlen +// skipped Py_UNICODE_strcpy +// skipped Py_UNICODE_strcat +// skipped Py_UNICODE_strncpy +// skipped Py_UNICODE_strcmp +// skipped Py_UNICODE_strncmp +// skipped Py_UNICODE_strchr +// skipped Py_UNICODE_strrchr +// skipped _PyUnicode_FormatLong +// skipped PyUnicode_AsUnicodeCopy +// skipped _PyUnicode_FromId +// skipped _PyUnicode_EQ +// skipped _PyUnicode_ScanIdentifier + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::PyString; + use crate::{AsPyPointer, Python}; + + #[test] + fn ascii_object_bitfield() { + let ob_base: PyObject = unsafe { std::mem::zeroed() }; + + let mut o = PyASCIIObject { + ob_base, + length: 0, + hash: 0, + state: 0, + wstr: std::ptr::null_mut() as *mut wchar_t, + }; + + assert_eq!(o.interned(), 0); + assert_eq!(o.kind(), 0); + assert_eq!(o.compact(), 0); + assert_eq!(o.ascii(), 0); + assert_eq!(o.ready(), 0); + + for i in 0..4 { + o.state = i; + assert_eq!(o.interned(), i); + } + + for i in 0..8 { + o.state = i << 2; + assert_eq!(o.kind(), i); + } + + o.state = 1 << 5; + assert_eq!(o.compact(), 1); + + o.state = 1 << 6; + assert_eq!(o.ascii(), 1); + + o.state = 1 << 7; + assert_eq!(o.ready(), 1); + } + + #[test] + #[cfg_attr(Py_3_10, allow(deprecated))] + fn ascii() { + Python::with_gil(|py| { + // This test relies on implementation details of PyString. + let s = PyString::new(py, "hello, world"); + let ptr = s.as_ptr(); + + unsafe { + let ascii_ptr = ptr as *mut PyASCIIObject; + let ascii = ascii_ptr.as_ref().unwrap(); + + assert_eq!(ascii.interned(), 0); + assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND); + assert_eq!(ascii.compact(), 1); + assert_eq!(ascii.ascii(), 1); + assert_eq!(ascii.ready(), 1); + + assert_eq!(PyUnicode_IS_ASCII(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 1); + + assert!(!PyUnicode_1BYTE_DATA(ptr).is_null()); + // 2 and 4 byte macros return nonsense for this string instance. + assert_eq!(PyUnicode_KIND(ptr), PyUnicode_1BYTE_KIND); + + assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); + // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. + assert!(!PyUnicode_DATA(ptr).is_null()); + + assert_eq!(PyUnicode_GET_LENGTH(ptr), s.len().unwrap() as _); + assert_eq!(PyUnicode_IS_READY(ptr), 1); + + // This has potential to mutate object. But it should be a no-op since + // we're already ready. + assert_eq!(PyUnicode_READY(ptr), 0); + } + }) + } + + #[test] + #[cfg_attr(Py_3_10, allow(deprecated))] + fn ucs4() { + Python::with_gil(|py| { + let s = "哈哈🐈"; + let py_string = PyString::new(py, s); + let ptr = py_string.as_ptr(); + + unsafe { + let ascii_ptr = ptr as *mut PyASCIIObject; + let ascii = ascii_ptr.as_ref().unwrap(); + + assert_eq!(ascii.interned(), 0); + assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND); + assert_eq!(ascii.compact(), 1); + assert_eq!(ascii.ascii(), 0); + assert_eq!(ascii.ready(), 1); + + assert_eq!(PyUnicode_IS_ASCII(ptr), 0); + assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 0); + + assert!(!PyUnicode_4BYTE_DATA(ptr).is_null()); + assert_eq!(PyUnicode_KIND(ptr), PyUnicode_4BYTE_KIND); + + assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); + // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. + assert!(!PyUnicode_DATA(ptr).is_null()); + + assert_eq!(PyUnicode_GET_LENGTH(ptr), py_string.len().unwrap() as _); + assert_eq!(PyUnicode_IS_READY(ptr), 1); + + // This has potential to mutate object. But it should be a no-op since + // we're already ready. + assert_eq!(PyUnicode_READY(ptr), 0); + } + }) + } +} diff --git a/src/ffi/pystate.rs b/src/ffi/pystate.rs index 81631851e6b..e26619e7ec8 100644 --- a/src/ffi/pystate.rs +++ b/src/ffi/pystate.rs @@ -1,26 +1,49 @@ +#[cfg(not(PyPy))] use crate::ffi::moduleobject::PyModuleDef; use crate::ffi::object::PyObject; -use crate::ffi::PyFrameObject; -use std::os::raw::{c_int, c_long}; +use std::os::raw::c_int; +#[cfg(not(PyPy))] +use std::os::raw::c_long; pub const MAX_CO_EXTRA_USERS: c_int = 255; +opaque_struct!(PyThreadState); opaque_struct!(PyInterpreterState); -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyThreadState { - pub ob_base: PyObject, - pub interp: *mut PyInterpreterState, -} - extern "C" { + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_New() -> *mut PyInterpreterState; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_Clear(arg1: *mut PyInterpreterState); + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_Delete(arg1: *mut PyInterpreterState); - //fn _PyState_AddModule(arg1: *mut PyObject, - // arg2: *mut PyModuleDef) -> c_int; + + #[cfg(all(Py_3_9, not(PyPy)))] + #[cfg_attr(docsrs, doc(all(Py_3_9, not(PyPy))))] + pub fn PyInterpreterState_Get() -> *mut PyInterpreterState; + + #[cfg(all(Py_3_8, not(PyPy)))] + #[cfg_attr(docsrs, doc(all(Py_3_8, not(PyPy))))] + pub fn PyInterpreterState_GetDict() -> *mut PyObject; + + #[cfg(all(Py_3_7, not(PyPy)))] + #[cfg_attr(docsrs, doc(all(Py_3_7, not(PyPy))))] + pub fn PyInterpreterState_GetID() -> i64; + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] + pub fn PyState_AddModule(arg1: *mut PyObject, arg2: *mut PyModuleDef) -> c_int; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] + pub fn PyState_RemoveModule(arg1: *mut PyModuleDef) -> c_int; + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyState_FindModule(arg1: *mut PyModuleDef) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyThreadState_New")] pub fn PyThreadState_New(arg1: *mut PyInterpreterState) -> *mut PyThreadState; //fn _PyThreadState_Prealloc(arg1: *mut PyInterpreterState) @@ -39,6 +62,8 @@ extern "C" { pub fn PyThreadState_Swap(arg1: *mut PyThreadState) -> *mut PyThreadState; #[cfg_attr(PyPy, link_name = "PyPyThreadState_GetDict")] pub fn PyThreadState_GetDict() -> *mut PyObject; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyThreadState_SetAsyncExc(arg1: c_long, arg2: *mut PyObject) -> c_int; } @@ -54,18 +79,12 @@ extern "C" { pub fn PyGILState_Ensure() -> PyGILState_STATE; #[cfg_attr(PyPy, link_name = "PyPyGILState_Release")] pub fn PyGILState_Release(arg1: PyGILState_STATE); + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyGILState_GetThisThreadState() -> *mut PyThreadState; - pub fn PyGILState_Check() -> c_int; } #[inline] pub unsafe fn PyThreadState_GET() -> *mut PyThreadState { PyThreadState_Get() } - -pub type Py_tracefunc = extern "C" fn( - obj: *mut PyObject, - frame: *mut PyFrameObject, - what: c_int, - arg: *mut PyObject, -) -> c_int; diff --git a/src/ffi/unicodeobject.rs b/src/ffi/unicodeobject.rs index de51d816012..9f215cf8299 100644 --- a/src/ffi/unicodeobject.rs +++ b/src/ffi/unicodeobject.rs @@ -40,40 +40,11 @@ pub unsafe fn PyUnicode_CheckExact(op: *mut PyObject) -> c_int { pub const Py_UNICODE_REPLACEMENT_CHARACTER: Py_UCS4 = 0xFFFD; extern "C" { - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_New(size: Py_ssize_t, maxchar: Py_UCS4) -> *mut PyObject; - - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_CopyCharacters( - to: *mut PyObject, - to_start: Py_ssize_t, - from: *mut PyObject, - from_start: Py_ssize_t, - how_many: Py_ssize_t, - ) -> Py_ssize_t; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_Fill( - unicode: *mut PyObject, - start: Py_ssize_t, - length: Py_ssize_t, - fill_char: Py_UCS4, - ) -> Py_ssize_t; - #[cfg(all(not(Py_LIMITED_API), not(Py_3_12)))] - #[deprecated] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromUnicode")] - pub fn PyUnicode_FromUnicode(u: *const Py_UNICODE, size: Py_ssize_t) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromStringAndSize")] pub fn PyUnicode_FromStringAndSize(u: *const c_char, size: Py_ssize_t) -> *mut PyObject; pub fn PyUnicode_FromString(u: *const c_char) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_FromKindAndData( - kind: c_int, - buffer: *const c_void, - size: Py_ssize_t, - ) -> *mut PyObject; - pub fn PyUnicode_Substring( str: *mut PyObject, start: Py_ssize_t, @@ -86,17 +57,6 @@ extern "C" { copy_null: c_int, ) -> *mut Py_UCS4; pub fn PyUnicode_AsUCS4Copy(unicode: *mut PyObject) -> *mut Py_UCS4; - #[cfg(all(not(Py_LIMITED_API), not(Py_3_12)))] - #[deprecated] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicode")] - pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut Py_UNICODE; - #[cfg(all(not(Py_LIMITED_API), not(Py_3_12)))] - #[deprecated] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicodeAndSize")] - pub fn PyUnicode_AsUnicodeAndSize( - unicode: *mut PyObject, - size: *mut Py_ssize_t, - ) -> *mut Py_UNICODE; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetLength")] pub fn PyUnicode_GetLength(unicode: *mut PyObject) -> Py_ssize_t; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetSize")] @@ -143,20 +103,6 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromOrdinal")] pub fn PyUnicode_FromOrdinal(ordinal: c_int) -> *mut PyObject; pub fn PyUnicode_ClearFreeList() -> c_int; - #[cfg(any(not(Py_LIMITED_API), Py_3_10))] - #[cfg(Py_3_7)] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] - pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *const c_char; - #[cfg(not(Py_3_7))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] - pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *mut c_char; - #[cfg(not(Py_LIMITED_API))] - #[cfg(Py_3_7)] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] - pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *const c_char; - #[cfg(not(Py_3_7))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] - pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *mut c_char; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetDefaultEncoding")] pub fn PyUnicode_GetDefaultEncoding() -> *const c_char; #[cfg_attr(PyPy, link_name = "PyPyUnicode_Decode")] @@ -176,13 +122,6 @@ extern "C" { encoding: *const c_char, errors: *const c_char, ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_Encode( - s: *const Py_UNICODE, - size: Py_ssize_t, - encoding: *const c_char, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsEncodedObject")] pub fn PyUnicode_AsEncodedObject( unicode: *mut PyObject, @@ -212,14 +151,6 @@ extern "C" { errors: *const c_char, consumed: *mut Py_ssize_t, ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUTF7( - data: *const Py_UNICODE, - length: Py_ssize_t, - base64SetO: c_int, - base64WhiteSpace: c_int, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeUTF8")] pub fn PyUnicode_DecodeUTF8( string: *const c_char, @@ -234,13 +165,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8String")] pub fn PyUnicode_AsUTF8String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeUTF8")] - pub fn PyUnicode_EncodeUTF8( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeUTF32")] pub fn PyUnicode_DecodeUTF32( string: *const c_char, @@ -257,13 +181,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF32String")] pub fn PyUnicode_AsUTF32String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUTF32( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - byteorder: c_int, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeUTF16")] pub fn PyUnicode_DecodeUTF16( string: *const c_char, @@ -280,13 +197,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF16String")] pub fn PyUnicode_AsUTF16String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUTF16( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - byteorder: c_int, - ) -> *mut PyObject; pub fn PyUnicode_DecodeUnicodeEscape( string: *const c_char, length: Py_ssize_t, @@ -294,22 +204,12 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicodeEscapeString")] pub fn PyUnicode_AsUnicodeEscapeString(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUnicodeEscape( - data: *const Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; pub fn PyUnicode_DecodeRawUnicodeEscape( string: *const c_char, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_AsRawUnicodeEscapeString(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeRawUnicodeEscape( - data: *const Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeLatin1")] pub fn PyUnicode_DecodeLatin1( string: *const c_char, @@ -318,13 +218,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsLatin1String")] pub fn PyUnicode_AsLatin1String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeLatin1")] - pub fn PyUnicode_EncodeLatin1( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeASCII")] pub fn PyUnicode_DecodeASCII( string: *const c_char, @@ -333,13 +226,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsASCIIString")] pub fn PyUnicode_AsASCIIString(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeASCII")] - pub fn PyUnicode_EncodeASCII( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - ) -> *mut PyObject; pub fn PyUnicode_DecodeCharmap( string: *const c_char, length: Py_ssize_t, @@ -350,35 +236,6 @@ extern "C" { unicode: *mut PyObject, mapping: *mut PyObject, ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeCharmap( - data: *const Py_UNICODE, - length: Py_ssize_t, - mapping: *mut PyObject, - errors: *const c_char, - ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_TranslateCharmap( - data: *const Py_UNICODE, - length: Py_ssize_t, - table: *mut PyObject, - errors: *const c_char, - ) -> *mut PyObject; - - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeDecimal")] - pub fn PyUnicode_EncodeDecimal( - s: *mut Py_UNICODE, - length: Py_ssize_t, - output: *mut c_char, - errors: *const c_char, - ) -> c_int; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_TransformDecimalToASCII")] - pub fn PyUnicode_TransformDecimalToASCII( - s: *mut Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; pub fn PyUnicode_DecodeLocaleAndSize( str: *const c_char, len: Py_ssize_t, diff --git a/src/lib.rs b/src/lib.rs index 8089d503119..56b0154b241 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -545,4 +545,7 @@ pub mod doc_test { doctest!("guide/src/trait_bounds.md", guide_trait_bounds_md); doctest!("guide/src/types.md", guide_types_md); doctest!("guide/src/faq.md", faq); + + // deliberate choice not to test guide/ecosystem because those pages depend on external crates + // such as pyo3_asyncio. } diff --git a/src/python.rs b/src/python.rs index b6757820693..5d5abd78c50 100644 --- a/src/python.rs +++ b/src/python.rs @@ -396,6 +396,7 @@ impl<'p> Python<'p> { return Err(PyErr::fetch(self)); } let res_ptr = ffi::PyEval_EvalCode(code_obj, globals, locals); + ffi::Py_DECREF(code_obj); self.from_owned_ptr_or_err(res_ptr) } diff --git a/src/types/datetime.rs b/src/types/datetime.rs index cad4b79d2a9..990d82ee2e2 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -465,8 +465,8 @@ mod tests { #[cfg(not(PyPy))] #[test] fn test_new_with_fold() { - pyo3::Python::with_gil(|py| { - use pyo3::types::{PyDateTime, PyTimeAccess}; + crate::Python::with_gil(|py| { + use crate::types::{PyDateTime, PyTimeAccess}; let a = PyDateTime::new_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, false); let b = PyDateTime::new_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, true); diff --git a/src/types/mod.rs b/src/types/mod.rs index f8bc24cf782..e635f440453 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -23,6 +23,8 @@ pub use self::num::PyLong as PyInt; pub use self::sequence::PySequence; pub use self::set::{PyFrozenSet, PySet}; pub use self::slice::{PySlice, PySliceIndices}; +#[cfg(not(Py_LIMITED_API))] +pub use self::string::PyStringData; pub use self::string::{PyString, PyString as PyUnicode}; pub use self::tuple::PyTuple; pub use self::typeobject::PyType; diff --git a/src/types/module.rs b/src/types/module.rs index 483b8a1ac06..aa08fa2b010 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -127,6 +127,7 @@ impl PyModule { } let mptr = ffi::PyImport_ExecCodeModuleEx(module.as_ptr(), cptr, filename.as_ptr()); + ffi::Py_DECREF(cptr); if mptr.is_null() { return Err(PyErr::fetch(py)); } diff --git a/src/types/string.rs b/src/types/string.rs index 7da450bb279..9ac1a149839 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -1,14 +1,133 @@ // Copyright (c) 2017-present PyO3 Project and Contributors +#[cfg(not(Py_LIMITED_API))] +use crate::exceptions::PyUnicodeDecodeError; use crate::types::PyBytes; use crate::{ ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyNativeType, PyObject, PyResult, PyTryFrom, Python, ToPyObject, }; use std::borrow::Cow; +#[cfg(not(Py_LIMITED_API))] +use std::ffi::CStr; use std::os::raw::c_char; use std::str; +/// Represents raw data backing a Python `str`. +/// +/// Python internally stores strings in various representations. This enumeration +/// represents those variations. +#[cfg(not(Py_LIMITED_API))] +#[cfg_attr(docsrs, doc(cfg(not(Py_LIMITED_API))))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum PyStringData<'a> { + /// UCS1 representation. + Ucs1(&'a [u8]), + + /// UCS2 representation. + Ucs2(&'a [u16]), + + /// UCS4 representation. + Ucs4(&'a [u32]), +} + +#[cfg(not(Py_LIMITED_API))] +impl<'a> PyStringData<'a> { + /// Obtain the raw bytes backing this instance as a [u8] slice. + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::Ucs1(s) => s, + Self::Ucs2(s) => unsafe { + std::slice::from_raw_parts( + s.as_ptr() as *const u8, + s.len() * self.value_width_bytes(), + ) + }, + Self::Ucs4(s) => unsafe { + std::slice::from_raw_parts( + s.as_ptr() as *const u8, + s.len() * self.value_width_bytes(), + ) + }, + } + } + + /// Size in bytes of each value/item in the underlying slice. + #[inline] + pub fn value_width_bytes(&self) -> usize { + match self { + Self::Ucs1(_) => 1, + Self::Ucs2(_) => 2, + Self::Ucs4(_) => 4, + } + } + + /// Convert the raw data to a Rust string. + /// + /// For UCS-1 / UTF-8, returns a borrow into the original slice. For UCS-2 and UCS-4, + /// returns an owned string. + /// + /// Returns [PyUnicodeDecodeError] if the string data isn't valid in its purported + /// storage format. This should only occur for strings that were created via Python + /// C APIs that skip input validation (like `PyUnicode_FromKindAndData`) and should + /// never occur for strings that were created from Python code. + pub fn to_string(self, py: Python) -> PyResult> { + match self { + Self::Ucs1(data) => match str::from_utf8(data) { + Ok(s) => Ok(Cow::Borrowed(s)), + Err(e) => Err(crate::PyErr::from_instance(PyUnicodeDecodeError::new_utf8( + py, data, e, + )?)), + }, + Self::Ucs2(data) => match String::from_utf16(data) { + Ok(s) => Ok(Cow::Owned(s)), + Err(e) => { + let mut message = e.to_string().as_bytes().to_vec(); + message.push(0); + + Err(crate::PyErr::from_instance(PyUnicodeDecodeError::new( + py, + CStr::from_bytes_with_nul(b"utf-16\0").unwrap(), + self.as_bytes(), + 0..self.as_bytes().len(), + CStr::from_bytes_with_nul(&message).unwrap(), + )?)) + } + }, + Self::Ucs4(data) => match data.iter().map(|&c| std::char::from_u32(c)).collect() { + Some(s) => Ok(Cow::Owned(s)), + None => Err(crate::PyErr::from_instance(PyUnicodeDecodeError::new( + py, + CStr::from_bytes_with_nul(b"utf-32\0").unwrap(), + self.as_bytes(), + 0..self.as_bytes().len(), + CStr::from_bytes_with_nul(b"error converting utf-32\0").unwrap(), + )?)), + }, + } + } + + /// Convert the raw data to a Rust string, possibly with data loss. + /// + /// Invalid code points will be replaced with `U+FFFD REPLACEMENT CHARACTER`. + /// + /// Returns a borrow into original data, when possible, or owned data otherwise. + /// + /// The return value of this function should only disagree with [Self::to_string] + /// when that method would error. + pub fn to_string_lossy(self) -> Cow<'a, str> { + match self { + Self::Ucs1(data) => String::from_utf8_lossy(data), + Self::Ucs2(data) => Cow::Owned(String::from_utf16_lossy(data)), + Self::Ucs4(data) => Cow::Owned( + data.iter() + .map(|&c| std::char::from_u32(c).unwrap_or('\u{FFFD}')) + .collect(), + ), + } + } +} + /// Represents a Python `string` (a Unicode string object). /// /// This type is immutable. @@ -49,8 +168,8 @@ impl PyString { pub fn to_str(&self) -> PyResult<&str> { let utf8_slice = { cfg_if::cfg_if! { - if #[cfg(any(not(Py_LIMITED_API), Py_3_10))] { - // PyUnicode_AsUTF8AndSize only available on limited API from Python 3.10 and up. + if #[cfg(not(Py_LIMITED_API))] { + // PyUnicode_AsUTF8AndSize only available on limited API. let mut size: ffi::Py_ssize_t = 0; let data = unsafe { ffi::PyUnicode_AsUTF8AndSize(self.as_ptr(), &mut size) }; if data.is_null() { @@ -89,6 +208,45 @@ impl PyString { } } } + + /// Obtains the raw data backing the Python string. + /// + /// If the Python string object was created through legacy APIs, its internal + /// storage format will be canonicalized before data is returned. + #[cfg(not(Py_LIMITED_API))] + #[cfg_attr(docsrs, doc(cfg(not(Py_LIMITED_API))))] + pub fn data(&self) -> PyResult> { + let ptr = self.as_ptr(); + + if cfg!(not(Py_3_12)) { + #[allow(deprecated)] + let ready = unsafe { ffi::PyUnicode_READY(ptr) }; + if ready != 0 { + // Exception was created on failure. + return Err(crate::PyErr::fetch(self.py())); + } + } + + // The string should be in its canonical form after calling `PyUnicode_READY()`. + // And non-canonical form not possible after Python 3.12. So it should be safe + // to call these APIs. + let length = unsafe { ffi::PyUnicode_GET_LENGTH(ptr) } as usize; + let raw_data = unsafe { ffi::PyUnicode_DATA(ptr) }; + let kind = unsafe { ffi::PyUnicode_KIND(ptr) }; + + match kind { + ffi::PyUnicode_1BYTE_KIND => Ok(PyStringData::Ucs1(unsafe { + std::slice::from_raw_parts(raw_data as *const u8, length) + })), + ffi::PyUnicode_2BYTE_KIND => Ok(PyStringData::Ucs2(unsafe { + std::slice::from_raw_parts(raw_data as *const u16, length) + })), + ffi::PyUnicode_4BYTE_KIND => Ok(PyStringData::Ucs4(unsafe { + std::slice::from_raw_parts(raw_data as *const u32, length) + })), + _ => unreachable!(), + } + } } /// Converts a Rust `str` to a Python object. @@ -193,8 +351,16 @@ impl FromPyObject<'_> for char { #[cfg(test)] mod tests { use super::PyString; + #[cfg(not(Py_LIMITED_API))] + use super::PyStringData; + #[cfg(not(Py_LIMITED_API))] + use crate::exceptions::PyUnicodeDecodeError; + #[cfg(not(Py_LIMITED_API))] + use crate::type_object::PyTypeObject; use crate::Python; use crate::{FromPyObject, PyObject, PyTryFrom, ToPyObject}; + #[cfg(not(Py_LIMITED_API))] + use std::borrow::Cow; #[test] fn test_non_bmp() { @@ -297,4 +463,124 @@ mod tests { assert_eq!(format!("{}", s), "Hello\n"); }) } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs1() { + Python::with_gil(|py| { + let s = PyString::new(py, "hello, world"); + let data = s.data().unwrap(); + + assert_eq!(data, PyStringData::Ucs1(b"hello, world")); + assert_eq!(data.to_string(py).unwrap(), Cow::Borrowed("hello, world")); + assert_eq!(data.to_string_lossy(), Cow::Borrowed("hello, world")); + }) + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs1_invalid() { + Python::with_gil(|py| { + // 0xfe is not allowed in UTF-8. + let buffer = b"f\xfe\0"; + let ptr = unsafe { + crate::ffi::PyUnicode_FromKindAndData( + crate::ffi::PyUnicode_1BYTE_KIND as _, + buffer.as_ptr() as *const _, + 2, + ) + }; + assert!(!ptr.is_null()); + let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let data = s.data().unwrap(); + assert_eq!(data, PyStringData::Ucs1(b"f\xfe")); + let err = data.to_string(py).unwrap_err(); + assert_eq!(err.ptype(py), PyUnicodeDecodeError::type_object(py)); + assert!(err + .to_string() + .contains("'utf-8' codec can't decode byte 0xfe in position 1")); + assert_eq!(data.to_string_lossy(), Cow::Borrowed("f�")); + }); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs2() { + Python::with_gil(|py| { + let s = py.eval("'foo\\ud800'", None, None).unwrap(); + let py_string = s.cast_as::().unwrap(); + let data = py_string.data().unwrap(); + + assert_eq!(data, PyStringData::Ucs2(&[102, 111, 111, 0xd800])); + assert_eq!( + data.to_string_lossy(), + Cow::Owned::("foo�".to_string()) + ); + }) + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs2_invalid() { + Python::with_gil(|py| { + // U+FF22 (valid) & U+d800 (never valid) + let buffer = b"\x22\xff\x00\xd8\x00\x00"; + let ptr = unsafe { + crate::ffi::PyUnicode_FromKindAndData( + crate::ffi::PyUnicode_2BYTE_KIND as _, + buffer.as_ptr() as *const _, + 2, + ) + }; + assert!(!ptr.is_null()); + let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let data = s.data().unwrap(); + assert_eq!(data, PyStringData::Ucs2(&[0xff22, 0xd800])); + let err = data.to_string(py).unwrap_err(); + assert_eq!(err.ptype(py), PyUnicodeDecodeError::type_object(py)); + assert!(err + .to_string() + .contains("'utf-16' codec can't decode bytes in position 0-3")); + assert_eq!(data.to_string_lossy(), Cow::Owned::("Ｂ�".into())); + }); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs4() { + Python::with_gil(|py| { + let s = "哈哈🐈"; + let py_string = PyString::new(py, s); + let data = py_string.data().unwrap(); + + assert_eq!(data, PyStringData::Ucs4(&[21704, 21704, 128008])); + assert_eq!(data.to_string_lossy(), Cow::Owned::(s.to_string())); + }) + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs4_invalid() { + Python::with_gil(|py| { + // U+20000 (valid) & U+d800 (never valid) + let buffer = b"\x00\x00\x02\x00\x00\xd8\x00\x00\x00\x00\x00\x00"; + let ptr = unsafe { + crate::ffi::PyUnicode_FromKindAndData( + crate::ffi::PyUnicode_4BYTE_KIND as _, + buffer.as_ptr() as *const _, + 2, + ) + }; + assert!(!ptr.is_null()); + let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let data = s.data().unwrap(); + assert_eq!(data, PyStringData::Ucs4(&[0x20000, 0xd800])); + let err = data.to_string(py).unwrap_err(); + assert_eq!(err.ptype(py), PyUnicodeDecodeError::type_object(py)); + assert!(err + .to_string() + .contains("'utf-32' codec can't decode bytes in position 0-7")); + assert_eq!(data.to_string_lossy(), Cow::Owned::("𠀀�".into())); + }); + } } diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index 53e99803bb7..38b9761ae8e 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -54,6 +54,8 @@ fn class_with_properties() { py_run!(py, inst, "inst.DATA = 20"); py_run!(py, inst, "assert inst.get_num() == 20 == inst.DATA"); + py_expect_exception!(py, inst, "del inst.DATA", PyAttributeError); + py_run!(py, inst, "assert inst.get_num() == inst.unwrapped == 20"); py_run!(py, inst, "inst.unwrapped = 42"); py_run!(py, inst, "assert inst.get_num() == inst.unwrapped == 42"); diff --git a/tests/test_proc_macro_hygiene.rs b/tests/test_proc_macro_hygiene.rs new file mode 100644 index 00000000000..5634128589c --- /dev/null +++ b/tests/test_proc_macro_hygiene.rs @@ -0,0 +1,44 @@ +#![no_implicit_prelude] + +#[::pyo3::proc_macro::pyclass] +#[derive(::std::clone::Clone)] +pub struct Foo; + +#[::pyo3::proc_macro::pyclass] +pub struct Foo2; + +#[::pyo3::proc_macro::pyclass( + name = "ActuallyBar", + freelist = 8, + weakref, + unsendable, + gc, + subclass, + extends = ::pyo3::types::PyAny, + module = "Spam" +)] +pub struct Bar { + #[pyo3(get, set)] + a: u8, + #[pyo3(get, set)] + b: Foo, + #[pyo3(get, set)] + c: ::std::option::Option<::pyo3::Py>, +} + +#[::pyo3::proc_macro::pyproto] +impl ::pyo3::class::gc::PyGCProtocol for Bar { + fn __traverse__( + &self, + visit: ::pyo3::class::gc::PyVisit, + ) -> ::std::result::Result<(), ::pyo3::class::gc::PyTraverseError> { + if let ::std::option::Option::Some(obj) = &self.c { + visit.call(obj)? + } + ::std::result::Result::Ok(()) + } + + fn __clear__(&mut self) { + self.c = ::std::option::Option::None; + } +}