Merge branch 'huggingface:main' into feature-sso

Author: MeetThePatel

bindings/python/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
-name = 'tokenizers'
-requires-python = '>=3.9'
+name = "tokenizers"
+requires-python = ">=3.9"
 authors = [
-  { name = 'Nicolas Patry', email = '[email protected]' },
-  { name = 'Anthony Moi', email = '[email protected]' },
+  { name = "Nicolas Patry", email = "[email protected]" },
+  { name = "Anthony Moi", email = "[email protected]" },
 ]
 classifiers = [
   "Development Status :: 5 - Production/Stable",
@@ -13,20 +13,21 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.7",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3 :: Only",
   "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 keywords = ["NLP", "tokenizer", "BPE", "transformer", "deep learning"]
-dynamic = ['description', 'license', 'readme', 'version']
+dynamic = ["description", "license", "readme", "version"]
 dependencies = ["huggingface_hub>=0.16.4,<1.0"]
 
 [project.urls]
-Homepage = 'https://github.com/huggingface/tokenizers'
-Source = 'https://github.com/huggingface/tokenizers'
+Homepage = "https://github.com/huggingface/tokenizers"
+Source = "https://github.com/huggingface/tokenizers"
 
 
 [project.optional-dependencies]
@@ -42,12 +43,12 @@ build-backend = "maturin"
 [tool.maturin]
 python-source = "py_src"
 module-name = "tokenizers.tokenizers"
-bindings = 'pyo3'
+bindings = "pyo3"
 features = ["pyo3/extension-module"]
 
 [tool.black]
 line-length = 119
-target-version = ['py35']
+target-version = ["py35"]
 
 [tool.ruff]
 line-length = 119

tokenizers/Cargo.toml

@@ -69,7 +69,7 @@ itertools = "0.13"
 log = "0.4"
 derive_builder = "0.20"
 spm_precompiled = "0.1.3"
-hf-hub = { version = "0.3.2", optional = true }
+hf-hub = { version = "0.4.1", features = ["ureq"], default-features = false, optional = true }
 aho-corasick = "1.1"
 paste = "1.0.14"
 macro_rules_attribute = "0.2.0"
@@ -87,6 +87,7 @@ esaxx_fast = ["esaxx-rs/cpp"]
 progressbar = ["indicatif"]
 http = ["hf-hub"]
 unstable_wasm = ["fancy-regex", "getrandom/js"]
+rustls-tls = ["hf-hub?/rustls-tls"]
 
 [dev-dependencies]
 criterion = "0.5"