FasterTokenizer->FastTokenizer (PaddlePaddle#3719)

README_cn.md

@@ -234,7 +234,7 @@ PaddleNLP针对信息抽取、语义检索、智能问答、情感分析等高
 
 ### 高性能分布式训练与推理
 
-#### ⚡ FasterTokenizer：高性能文本处理库
+#### ⚡ FastTokenizer：高性能文本处理库
 
 <div align="center">
     <img src="https://user-images.githubusercontent.com/11793384/168407921-b4395b1d-44bd-41a0-8c58-923ba2b703ef.png" width="400">
@@ -244,7 +244,7 @@ PaddleNLP针对信息抽取、语义检索、智能问答、情感分析等高
 AutoTokenizer.from_pretrained("ernie-3.0-medium-zh", use_faster=True)
 ```
 
-为了实现更极致的模型部署性能，安装FastTokenizers后只需在`AutoTokenizer` API上打开 `use_faster=True`选项，即可调用C++实现的高性能分词算子，轻松获得超Python百余倍的文本处理加速，更多使用说明可参考[FasterTokenizer文档](./faster_tokenizer)。
+为了实现更极致的模型部署性能，安装FastTokenizers后只需在`AutoTokenizer` API上打开 `use_faster=True`选项，即可调用C++实现的高性能分词算子，轻松获得超Python百余倍的文本处理加速，更多使用说明可参考[FastTokenizer文档](./fast_tokenizer)。
 
 #### ⚡️ FasterGeneration：高性能生成加速库
 

faster_tokenizer/CMakeLists.txt → fast_tokenizer/CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.10)
 
 project(tokenizers LANGUAGES CXX C VERSION 1.0)
 
-option(WITH_TESTING     "Compile PaddleNLP faster_tokenizer with unit testing"        OFF)
-option(WITH_PYTHON      "Compile PaddleNLP faster_tokenizer with python interpreter"   ON)
+option(WITH_TESTING     "Compile PaddleNLP fast_tokenizer with unit testing"        OFF)
+option(WITH_PYTHON      "Compile PaddleNLP fast_tokenizer with python interpreter"   ON)
 add_definitions(-DFASTERTOKENIZER_LIB)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
@@ -137,7 +137,7 @@ include_directories(${TOKENIZERS_INSTALL_INCLUDE_DIR})
 include(generic)
 include(third_party)
 
-add_subdirectory(faster_tokenizer)
+add_subdirectory(fast_tokenizer)
 
 if(WITH_PYTHON)
 
@@ -155,18 +155,18 @@ add_custom_target(build_tokenizers_bdist_wheel ALL
     DEPENDS copy_python_tokenizers)
 endif()
 
-else(WITH_PYTHON) # Pack faster_tokenizer cpp lib
+else(WITH_PYTHON) # Pack fast_tokenizer cpp lib
 
-set(CPP_PACKAGE_DIR ${CMAKE_BINARY_DIR}/cpp/faster_tokenizer)
+set(CPP_PACKAGE_DIR ${CMAKE_BINARY_DIR}/cpp/fast_tokenizer)
 add_custom_target(build_cpp_package_dir ALL
     COMMAND ${CMAKE_COMMAND} -E make_directory ${CPP_PACKAGE_DIR}/lib ${CPP_PACKAGE_DIR}/include ${CPP_PACKAGE_DIR}/third_party/include ${CPP_PACKAGE_DIR}/third_party/lib
     DEPENDS core_tokenizers)
 
 # copy cmake
-file(COPY ${PROJECT_SOURCE_DIR}/FasterTokenizer.cmake DESTINATION ${CPP_PACKAGE_DIR}/)
+file(COPY ${PROJECT_SOURCE_DIR}/FastTokenizer.cmake DESTINATION ${CPP_PACKAGE_DIR}/)
 
 # copy headers
-file(COPY ${PROJECT_SOURCE_DIR}/faster_tokenizer/ DESTINATION ${CPP_PACKAGE_DIR}/include/faster_tokenizer/
+file(COPY ${PROJECT_SOURCE_DIR}/fast_tokenizer/ DESTINATION ${CPP_PACKAGE_DIR}/include/fast_tokenizer/
     FILES_MATCHING PATTERN "*.h"
     PATTERN "test" EXCLUDE
     PATTERN "demo" EXCLUDE
@@ -181,7 +181,7 @@ add_custom_target(copy_third_party_headers ALL
 
 # copy library
 set(TOKENIZER_CORE_NAME "core_tokenizers")
-set(TOKENIZER_CORE_PATH ${CMAKE_BINARY_DIR}/faster_tokenizer)
+set(TOKENIZER_CORE_PATH ${CMAKE_BINARY_DIR}/fast_tokenizer)
 if (WIN32)
     set(ICU_DLL_DIR ${CMAKE_BINARY_DIR}/third_party/icu/src/extern_icu/icu4c/bin64)
     set(ICU_LIB_DIR ${CMAKE_BINARY_DIR}/third_party/icu/src/extern_icu/icu4c/lib64)

fast_tokenizer/README.md

@@ -0,0 +1,105 @@
+# FastTokenizer
+
+------------------------------------------------------------------------------------------
+
+<p align="center">
+    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
+    <a href=""><img src="https://img.shields.io/badge/python-3.6.2+-aff.svg"></a>
+    <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/PaddleNLP?color=9ea"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/PaddleNLP?color=3af"></a>
+    <a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
+</p>
+FastTokenizer是一款简单易用、功能强大的跨平台高性能文本预处理库，集成业界多个常用的Tokenizer实现，支持不同NLP场景下的文本预处理功能，如文本分类、阅读理解，序列标注等。结合PaddleNLP Tokenizer模块，为用户在训练、推理阶段提供高效通用的文本预处理能力。
+
+## 特性
+
+- 高性能。由于底层采用C++实现，所以其性能远高于目前常规Python实现的Tokenizer。在文本分类任务上，FastTokenizer对比Python版本Tokenizer加速比最高可达20倍。
+- 跨平台。FastTokenizer可在不同的系统平台上使用，目前已支持Windows x64，Linux x64以及MacOS 10.14+平台上使用。
+- 多编程语言支持。FastTokenizer提供在C++、Python语言上开发的能力。
+- 灵活性强。用户可以通过指定不同的FastTokenizer组件定制满足需求的Tokenizer。
+
+## 快速开始
+
+下面将介绍Python版本FastTokenizer的使用方式，C++版本的使用方式可参考[FastTokenizer C++ Demo](./fast_tokenizer/demo/README.md)。
+
+### 前置依赖
+
+- Windows 64位系统
+- Linux x64系统
+- MacOS 10.14+系统（m1芯片的MacOS，需要使用x86_64版本的Anaconda作为python环境方可安装使用）
+- Python 3.6 ~ 3.9
+
+### 安装FastTokenizer
+
+```python
+pip install fast_tokenizer
+```
+
+### FastTokenizer使用示例
+
+- 准备词表
+
+```shell
+# Linux或者Mac用户可直接执行以下命令下载测试的词表，Windows 用户可在浏览器上下载到本地。
+wget https://bj.bcebos.com/paddlenlp/models/transformers/ernie/vocab.txt
+```
+
+- 切词示例
+
+FastTokenizer库内置NLP任务常用的Tokenizer，如ErnieFastTokenizer。下面将展示FastTokenizer的简单用法。
+
+```python
+from fast_tokenizer import ErnieFastTokenizer, models
+# 1. 加载词表
+vocab = models.WordPiece.read_file("ernie_vocab.txt")
+# 2. 实例化ErnieFastTokenizer对象
+fast_tokenizer = ErnieFastTokenizer(vocab)
+# 3. 切词
+output = fast_tokenizer.encode("我爱中国")
+# 4. 输出结果
+print("ids: ", output.ids)
+print("type_ids: ", output.type_ids)
+print("tokens: ", output.tokens)
+print("offsets: ", output.offsets)
+print("attention_mask: ", output.attention_mask)
+```
+
+### FastTokenizer在PaddleNLP Tokenizer模块加速示例
+
+PaddleNLP Tokenizer模块可简单地应用在模型训练以及推理部署的文本预处理阶段，并通过`AutoTokenizer.from_pretrained`方式实例化相应的Tokenizer。其中`AutoTokenizer`默认加载得到的Tokenizer是常规Python实现的Tokenizer，其性能会低于C++实现的FastTokenizer。为了提升PaddleNLP Tokenizer模块性能，目前PaddleNLP Tokenizer模块已经支持使用FastTokenizer作为Tokenizer的后端加速切词阶段。在现有的Tokenizer加载接口中，仅需添加`use_fast=True`这一关键词参数，其余代码保持不变，即可加载Fast版本的Tokenizer，代码示例如下：
+
+```python
+from paddlenlp.transformers import AutoTokenizer
+
+# 默认加载Python版本的Tokenizer
+tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
+# 打开use_fast开关，可加载Fast版本Tokenizer
+fast_tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh', use_fast=True)
+
+text1 = tokenizer('自然语言处理')
+text2 = fast_tokenizer('自然语言处理')
+
+print(text1)
+print(text2)
+```
+
+目前PaddleNLP已支持BERT、ERNIE、TinyBERT以及ERNIE-M 4种Tokenizer的Fast版本，其余模型的Tokenizer暂不支持Fast版本。
+
+## FAQ
+
+Q：我在AutoTokenizer.from_pretrained接口上已经打开`use_fast=True`开关，为什么文本预处理阶段性能上好像没有任何变化？
+
+A：在有三种情况下，打开`use_fast=True`开关可能无法提升性能：
+  1. 没有安装fast_tokenizer。若在没有安装fast_tokenizer库的情况下打开`use_fast`开关，PaddleNLP会给出以下warning："Can't find the fast_tokenizer package, please ensure install fast_tokenizer correctly. "。
+
+  2. 加载的Tokenizer类型暂不支持Fast版本。目前支持4种Tokenizer的Fast版本，分别是BERT、ERNIE、TinyBERT以及ERNIE-M Tokenizer。若加载不支持Fast版本的Tokenizer情况下打开`use_fast`开关，PaddleNLP会给出以下warning："The tokenizer XXX doesn't have the fast version. Please check the map paddlenlp.transformers.auto.tokenizer.FASTER_TOKENIZER_MAPPING_NAMES to see which fast tokenizers are currently supported."
+
+  3. 待切词文本长度过短（如文本平均长度小于5）。这种情况下切词开销可能不是整个文本预处理的性能瓶颈，导致在使用FastTokenizer后仍无法提升整体性能。
+
+## 相关文档
+
+[FastTokenizer编译指南](docs/compile/README.md)

faster_tokenizer/docs/compile/README.md → fast_tokenizer/docs/compile/README.md

@@ -1,11 +1,11 @@
-# FasterTokenizer编译指南
+# FastTokenizer编译指南
 
-本文档说明编译FasterTokenizer C++库、Python库两种编译过程，根据编译的平台参考如下文档
+本文档说明编译FastTokenizer C++库、Python库两种编译过程，根据编译的平台参考如下文档
 
 - [Linux & Mac 编译](./how_to_build_linux_and_mac.md)
 - [Windows编译](./how_to_build_windows.md)
 
-FasterTokenizer使用CMake编译，其中编译过程中，各平台上编译选项如下表所示
+FastTokenizer使用CMake编译，其中编译过程中，各平台上编译选项如下表所示
 
 | 选项 | 作用 | 备注 |
 |:---- | :--- | :--- |

faster_tokenizer/docs/compile/how_to_build_linux_and_mac.md → fast_tokenizer/docs/compile/how_to_build_linux_and_mac.md

@@ -9,7 +9,7 @@
 
 ```bash
 git clone https://github.com/PaddlePaddle/PaddleNLP.git
-cd PaddleNLP/faster_tokenizer
+cd PaddleNLP/fast_tokenizer
 mkdir build & cd build
 cmake .. -DWITH_PYTHON=OFF -DWITH_TESTING=OFF -DCMAKE_BUILD_TYPE=Release
 make -j8
@@ -21,7 +21,7 @@ make -j8
 
 ```bash
 git clone https://github.com/PaddlePaddle/PaddleNLP.git
-cd PaddleNLP/faster_tokenizer
+cd PaddleNLP/fast_tokenizer
 mkdir build & cd build
 # 设置Python环境
 export LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH}

faster_tokenizer/docs/compile/how_to_build_windows.md → fast_tokenizer/docs/compile/how_to_build_windows.md

@@ -13,7 +13,7 @@
 
 ```bash
 git clone https://github.com/PaddlePaddle/PaddleNLP.git
-cd PaddleNLP/faster_tokenizer
+cd PaddleNLP/fast_tokenizer
 mkdir build & cd build
 cmake .. -G "Ninja" -DWITH_PYTHON=OFF -DWITH_TESTING=OFF -DCMAKE_BUILD_TYPE=Release
 ninja -j8
@@ -25,7 +25,7 @@ ninja -j8
 
 ```bash
 git clone https://github.com/PaddlePaddle/PaddleNLP.git
-cd PaddleNLP/faster_tokenizer
+cd PaddleNLP/fast_tokenizer
 mkdir build & cd build
 # 需要指定Python库
 cmake .. -G "Ninja" -DWITH_PYTHON=ON ^

faster_tokenizer/faster_tokenizer/CMakeLists.txt → fast_tokenizer/fast_tokenizer/CMakeLists.txt

@@ -13,7 +13,7 @@ endif()
 if (WITH_PYTHON)
 add_subdirectory(pybind)
 cc_library(core_tokenizers SHARED
-           SRCS pybind/pybind.cc tokenizers/ernie_faster_tokenizer.cc
+           SRCS pybind/pybind.cc tokenizers/ernie_fast_tokenizer.cc
            DEPS pybind python pybind_normalizers pybind_utils
                 pybind_pretokenizers pybind_models pybind_decoders
                 pybind_postprocessors pybind_tokenizers pybind_exception
@@ -33,7 +33,7 @@ endif()
 else(WITH_PYTHON)
 # add_subdirectory(tokenizers)
 cc_library(core_tokenizers SHARED
-           SRCS tokenizers/ernie_faster_tokenizer.cc
+           SRCS tokenizers/ernie_fast_tokenizer.cc
            DEPS normalizers pretokenizers models decoders
                 postprocessors core added_vocabulary tokenizer json)
 

faster_tokenizer/faster_tokenizer/core/added_vocabulary.cc → fast_tokenizer/fast_tokenizer/core/added_vocabulary.cc

@@ -12,15 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "faster_tokenizer/core/added_vocabulary.h"
-#include "faster_tokenizer/models/model.h"
-#include "faster_tokenizer/normalizers/normalizer.h"
-#include "faster_tokenizer/pretokenizers/pretokenizer.h"
+#include "fast_tokenizer/core/added_vocabulary.h"
+#include "fast_tokenizer/models/model.h"
+#include "fast_tokenizer/normalizers/normalizer.h"
+#include "fast_tokenizer/pretokenizers/pretokenizer.h"
 #include "glog/logging.h"
 #include "re2/re2.h"
 
 namespace paddlenlp {
-namespace faster_tokenizer {
+namespace fast_tokenizer {
 namespace core {
 
 inline bool StartWithWord(const std::string& sequence) {
@@ -420,5 +420,5 @@ void to_json(nlohmann::json& j, const AddedVocabulary& added_vocab) {
 }
 
 }  // namespace core
-}  // namespace faster_tokenizer
+}  // namespace fast_tokenizer
 }  // namespace paddlenlp

faster_tokenizer/faster_tokenizer/core/added_vocabulary.h → fast_tokenizer/fast_tokenizer/core/added_vocabulary.h

@@ -18,15 +18,15 @@ limitations under the License. */
 #include <string>
 #include <unordered_set>
 
-#include "faster_tokenizer/core/base.h"
+#include "fast_tokenizer/core/base.h"
 #include "nlohmann/json.hpp"
 
 namespace re2 {
 class RE2;
 }  // namespace re2
 
 namespace paddlenlp {
-namespace faster_tokenizer {
+namespace fast_tokenizer {
 
 namespace normalizers {
 class Normalizer;
@@ -139,15 +139,15 @@ class FASTERTOKENIZER_DECL AddedVocabulary {
 };
 
 }  // namespace core
-}  // namespace faster_tokenizer
+}  // namespace fast_tokenizer
 }  // namespace paddlenlp
 
 namespace std {
 template <>
-class hash<paddlenlp::faster_tokenizer::core::AddedToken> {
+class hash<paddlenlp::fast_tokenizer::core::AddedToken> {
 public:
   size_t operator()(
-      const paddlenlp::faster_tokenizer::core::AddedToken& added_token) const {
+      const paddlenlp::fast_tokenizer::core::AddedToken& added_token) const {
     return std::hash<std::string>()(added_token.GetContent());
   }
 };

faster_tokenizer/faster_tokenizer/core/base.h → fast_tokenizer/fast_tokenizer/core/base.h

@@ -22,7 +22,7 @@ limitations under the License. */
 #include <vector>
 
 #include "nlohmann/json.hpp"
-#include "faster_tokenizer/utils/utils.h"
+#include "fast_tokenizer/utils/utils.h"
 
 namespace std {
 template <>
@@ -36,7 +36,7 @@ struct hash<std::pair<uint32_t, uint32_t>> {
 }
 
 namespace paddlenlp {
-namespace faster_tokenizer {
+namespace fast_tokenizer {
 namespace core {
 
 enum FASTERTOKENIZER_DECL OffsetType { CHAR, BYTE };
@@ -359,5 +359,5 @@ struct FASTERTOKENIZER_DECL BPEWord {
 };
 
 }  // namespace core
-}  // namespace faster_tokenizer
+}  // namespace fast_tokenizer
 }  // namespace paddlenlp

faster_tokenizer/faster_tokenizer/core/encoding.cc → fast_tokenizer/fast_tokenizer/core/encoding.cc

@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "faster_tokenizer/core/encoding.h"
+#include "fast_tokenizer/core/encoding.h"
 #include <algorithm>
 #include <cassert>
 #include <climits>
@@ -24,7 +24,7 @@ limitations under the License. */
 #endif
 
 namespace paddlenlp {
-namespace faster_tokenizer {
+namespace fast_tokenizer {
 namespace core {
 
 Encoding::Encoding(const std::vector<uint32_t>& ids,
@@ -693,5 +693,5 @@ void RunMultiThread(std::function<void(size_t, size_t)> func,
 }
 
 }  // namespace core
-}  // namespace faster_tokenizer
+}  // namespace fast_tokenizer
 }  // namespace paddlenlp

faster_tokenizer/faster_tokenizer/core/encoding.h → fast_tokenizer/fast_tokenizer/core/encoding.h

@@ -18,8 +18,8 @@ limitations under the License. */
 #include <string>
 #include <unordered_map>
 #include <vector>
-#include "faster_tokenizer/core/base.h"
-#include "faster_tokenizer/utils/utils.h"
+#include "fast_tokenizer/core/base.h"
+#include "fast_tokenizer/utils/utils.h"
 
 #include <math.h>
 #include <stdlib.h>
@@ -28,7 +28,7 @@ limitations under the License. */
 using namespace std;
 
 namespace paddlenlp {
-namespace faster_tokenizer {
+namespace fast_tokenizer {
 namespace core {
 
 class FASTERTOKENIZER_DECL Encoding {
@@ -133,5 +133,5 @@ int FASTERTOKENIZER_DECL GetThreadNum(size_t batch_size);
 void FASTERTOKENIZER_DECL
 RunMultiThread(std::function<void(size_t, size_t)> func, size_t batch_size);
 }  // namespace core
-}  // namespace faster_tokenizer
+}  // namespace fast_tokenizer
 }  // namespace paddlenlp