From bfdebfe75eb9d60717b8b03fff8dcdf76f51c347 Mon Sep 17 00:00:00 2001 From: Owen Shepherd Date: Thu, 1 May 2025 10:52:46 +0100 Subject: [PATCH] Fix no-onig no-wasm builds Now, you can build with: ``` --no-default-features --features=fancy-regex ``` Which previously didn't work. You had to enable the `unstable_wasm` flag. I think using `fancy_regex` without wasm is a valid use-case, as I've seen extremely slow build times using `onig`. See: https://github.com/huggingface/tokenizers/issues/1730 Onig also breaks, sometimes, with compiler updates. See: https://github.com/huggingface/tokenizers/pull/1771 Closes https://github.com/huggingface/tokenizers/issues/1729 --- tokenizers/src/utils/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tokenizers/src/utils/mod.rs b/tokenizers/src/utils/mod.rs index deda862eb..75d88734e 100644 --- a/tokenizers/src/utils/mod.rs +++ b/tokenizers/src/utils/mod.rs @@ -2,15 +2,21 @@ pub(crate) mod cache; #[cfg(feature = "http")] pub(crate) mod from_pretrained; -#[cfg(feature = "unstable_wasm")] +#[cfg(feature = "fancy-regex")] mod fancy; -#[cfg(feature = "unstable_wasm")] +#[cfg(feature = "fancy-regex")] pub use fancy::SysRegex; -#[cfg(not(feature = "unstable_wasm"))] +#[cfg(feature = "onig")] mod onig; -#[cfg(not(feature = "unstable_wasm"))] +#[cfg(feature = "onig")] pub use crate::utils::onig::SysRegex; +#[cfg(all(feature = "onig", feature = "fancy-regex"))] +compile_error!("Features `onig` and `fancy-regex` are mutually exclusive"); + +#[cfg(not(any(feature = "onig", feature = "fancy-regex")))] +compile_error!("One of the `onig`, or `fancy-regex` features must be enabled"); + pub mod iter; pub mod padding; pub mod parallelism;