diff --git a/Cargo.lock b/Cargo.lock index 57a06c90b7..e364ad174f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7653,9 +7653,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b08cc37428a476fc9e20ac850132a513a2e1ce32b6a31addf2b74fa7033b905" +checksum = "9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae" dependencies = [ "aho-corasick", "derive_builder", diff --git a/examples/text-classification/Cargo.toml b/examples/text-classification/Cargo.toml index e6e66e2cf9..4ec5d7c89a 100644 --- a/examples/text-classification/Cargo.toml +++ b/examples/text-classification/Cargo.toml @@ -34,7 +34,7 @@ burn = { path = "../../crates/burn", features = [ ], default-features = false } # Tokenizer -tokenizers = { version = "0.20.3", default-features = false, features = [ +tokenizers = { version = "0.21.0", default-features = false, features = [ "onig", "http", ] } diff --git a/examples/text-generation/Cargo.toml b/examples/text-generation/Cargo.toml index 49bf7f37ce..8fe9bd976c 100644 --- a/examples/text-generation/Cargo.toml +++ b/examples/text-generation/Cargo.toml @@ -15,7 +15,7 @@ f16 = [] burn = {path = "../../crates/burn", features=["train", "tch"]} # Tokenizer -tokenizers = {version = "0.20.3", default-features = false, features = [ +tokenizers = {version = "0.21.0", default-features = false, features = [ "onig", "http", ]}