From c890af81080b14dc5266b0627b45fc51417b8d30 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Dec 2024 08:46:18 -0500
Subject: [PATCH] Bump tokenizers from 0.20.4 to 0.21.0 (#2574)

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.20.4 to 0.21.0.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.20.4...v0.21.0)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Cargo.lock                              | 4 ++--
 examples/text-classification/Cargo.toml | 2 +-
 examples/text-generation/Cargo.toml     | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 57a06c90b7..e364ad174f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7653,9 +7653,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokenizers"
-version = "0.20.4"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b08cc37428a476fc9e20ac850132a513a2e1ce32b6a31addf2b74fa7033b905"
+checksum = "9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae"
 dependencies = [
  "aho-corasick",
  "derive_builder",
diff --git a/examples/text-classification/Cargo.toml b/examples/text-classification/Cargo.toml
index e6e66e2cf9..4ec5d7c89a 100644
--- a/examples/text-classification/Cargo.toml
+++ b/examples/text-classification/Cargo.toml
@@ -34,7 +34,7 @@ burn = { path = "../../crates/burn", features = [
 ], default-features = false }
 
 # Tokenizer
-tokenizers = { version = "0.20.3", default-features = false, features = [
+tokenizers = { version = "0.21.0", default-features = false, features = [
   "onig",
   "http",
 ] }
diff --git a/examples/text-generation/Cargo.toml b/examples/text-generation/Cargo.toml
index 49bf7f37ce..8fe9bd976c 100644
--- a/examples/text-generation/Cargo.toml
+++ b/examples/text-generation/Cargo.toml
@@ -15,7 +15,7 @@ f16 = []
 burn = {path = "../../crates/burn", features=["train", "tch"]}
 
 # Tokenizer
-tokenizers = {version = "0.20.3", default-features = false, features = [
+tokenizers = {version = "0.21.0", default-features = false, features = [
   "onig",
   "http",
 ]}