Skip to content

Commit a3fd734

Browse files
tqchenjinhongyiiMasterJH5574spectrometerHBHjunrushao
committed
Initial checkin of tokenizer cpp
Co-authored-by: Hongyi Jin <jinhongyi02@gmail.com> Co-authored-by: Ruihang Lai <ruihangl@cs.cmu.edu> Co-authored-by: Bohan Hou <spectrometerh@gmail.com> Co-authored-by: Junru Shao <junrushao@apache.org> Co-authored-by: Zihao Ye <zhye@cs.washington.edu>
1 parent 1294dd4 commit a3fd734

22 files changed

Lines changed: 823 additions & 1 deletion

‎.clang-format‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Run the following command to reformat a file:
2+
# clang-format -i -style=Google <file>
3+
# Or use clang-format-diff to only reformat the changed lines:
4+
# https://clang.llvm.org/docs/ClangFormat.html
5+
BasedOnStyle: Google
6+
DerivePointerAlignment: false
7+
ColumnLimit: 100
8+
PointerAlignment: Left

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,6 @@
3030
*.exe
3131
*.out
3232
*.app
33+
Cargo.lock
34+
package-lock.json
35+
rust/target

‎.gitmodules‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "sentencepiece"]
2+
path = sentencepiece
3+
url = https://github.com/google/sentencepiece

‎CMakeLists.txt‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
cmake_minimum_required(VERSION 3.26)
2+
project(tokenizers_cpp C CXX)
3+
4+
# update to contain more rust flags
5+
set(TOKENIZERS_CPP_RUST_FLAGS "")
6+
set(TOKENIZERS_CPP_CARGO_TARGET "")
7+
8+
# extra link libraries
9+
set(TOKENIZERS_CPP_LINK_LIBS "")
10+
set(CARGO_EXTRA_ENVS "")
11+
message(STATUS "system-name" ${CMAKE_SYSTEM_NAME})
12+
13+
if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
14+
set(TOKENIZERS_CPP_CARGO_TARGET wasm32-unknown-emscripten)
15+
elseif (CMAKE_SYSTEM_NAME STREQUAL "iOS")
16+
set(TOKENIZERS_CPP_CARGO_TARGET aarch64-apple-ios)
17+
# add extra dependency needed for rust tokenizer in iOS
18+
find_library(FOUNDATION_LIB Foundation)
19+
find_library(SECURITY_LIB Security)
20+
list(APPEND TOKENIZERS_CPP_LINK_LIBS ${FOUNDATION_LIB} ${SECURITY_LIB})
21+
elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
22+
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
23+
set(TOKENIZERS_CPP_CARGO_TARGET aarch64-apple-darwin)
24+
endif()
25+
elseif (CMAKE_SYSTEM_NAME STREQUAL "Android")
26+
set(TOKENIZERS_CPP_CARGO_TARGET aarch64-linux-android)
27+
set(CARGO_EXTRA_ENVS
28+
AR_aarch64_linux_android=${ANDROID_TOOLCHAIN_ROOT}/bin/llvm-ar
29+
CC_aarch64_linux_android=${ANDROID_TOOLCHAIN_ROOT}/bin/aarch64-linux-android${ANDROID_NATIVE_API_LEVEL}-clang
30+
CXX_aarch64_linux_android=${ANDROID_TOOLCHAIN_ROOT}/bin/aarch64-linux-android${ANDROID_NATIVE_API_LEVEL}-clang++
31+
)
32+
elseif (CMAKE_SYSTEM_NAME STREQUAL "Windows")
33+
set(TOKENIZERS_CPP_CARGO_TARGET x86_64-pc-windows-msvc)
34+
endif()
35+
36+
if(WIN32)
37+
list(APPEND TOKENIZERS_CPP_LINK_LIBS
38+
wsock32 ws2_32 Bcrypt
39+
iphlpapi userenv psapi
40+
)
41+
endif()
42+
43+
set(TOKENIZERS_CPP_CARGO_FLAGS "")
44+
set(TOKENIZERS_CPP_CARGO_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR})
45+
set(TOKENIZERS_CPP_CARGO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
46+
47+
if (NOT TOKENIZERS_CPP_CARGO_TARGET STREQUAL "")
48+
list(APPEND TOKENIZERS_CPP_CARGO_FLAGS --target ${TOKENIZERS_CPP_CARGO_TARGET})
49+
set(TOKENIZERS_CPP_CARGO_BINARY_DIR
50+
"${TOKENIZERS_CPP_CARGO_BINARY_DIR}/${TOKENIZERS_CPP_CARGO_TARGET}")
51+
endif()
52+
53+
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
54+
set(TOKENIZERS_CPP_CARGO_BINARY_DIR "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/debug")
55+
else ()
56+
list(APPEND TOKENIZERS_CPP_CARGO_FLAGS --release)
57+
set(TOKENIZERS_CPP_CARGO_BINARY_DIR "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/release")
58+
endif ()
59+
60+
get_filename_component(TOKENIZERS_CPP_ROOT ${CMAKE_CURRENT_LIST_FILE} DIRECTORY)
61+
set(TOKENIZERS_CPP_CARGO_SOURCE_PATH ${TOKENIZERS_CPP_ROOT}/rust)
62+
63+
64+
if(MSVC)
65+
set(TOKENIZERS_RUST_LIB "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/tokenizers_c.lib")
66+
else()
67+
set(TOKENIZERS_RUST_LIB "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/libtokenizers_c.a")
68+
endif()
69+
set(TOKENIZERS_CPP_INCLUDE ${TOKENIZERS_CPP_ROOT}/include)
70+
71+
# NOTE: need to use cmake -E env to be portable in win
72+
add_custom_command(
73+
OUTPUT ${TOKENIZERS_RUST_LIB}
74+
COMMAND
75+
${CMAKE_COMMAND} -E env
76+
CARGO_TARGET_DIR=${TOKENIZERS_CPP_CARGO_TARGET_DIR}
77+
${CARGO_EXTRA_ENVS}
78+
RUSTFLAGS="${TOKENIZERS_CPP_RUST_FLAGS}"
79+
cargo build ${TOKENIZERS_CPP_CARGO_FLAGS}
80+
WORKING_DIRECTORY ${TOKENIZERS_CPP_CARGO_SOURCE_PATH}
81+
POST_BUILD COMMAND
82+
${CMAKE_COMMAND} -E copy
83+
${TOKENIZERS_RUST_LIB} "${CMAKE_CURRENT_BINARY_DIR}"
84+
)
85+
86+
set(
87+
TOKENIZER_CPP_SRCS
88+
src/sentencepiece_tokenizer.cc
89+
src/huggingface_tokenizer.cc
90+
)
91+
add_library(tokenizer_cpp_objs OBJECT ${TOKENIZER_CPP_SRCS})
92+
target_include_directories(tokenizer_cpp_objs PRIVATE sentencepiece/src)
93+
target_include_directories(tokenizer_cpp_objs PUBLIC ${TOKENIZERS_CPP_INCLUDE})
94+
95+
# sentencepiece config
96+
option(SPM_ENABLE_SHARED "override sentence piece config" OFF)
97+
option(SPM_ENABLE_TCMALLOC "" OFF)
98+
# provide macro if it does not exist in cmake system
99+
# it is OK to skip those since we do not provide these apps in the ios
100+
# instead just link to the sentencepiece directly
101+
if (CMAKE_SYSTEM_NAME STREQUAL "iOS")
102+
macro (set_xcode_property TARGET XCODE_PROPERTY XCODE_VALUE)
103+
set_property (TARGET ${TARGET} PROPERTY
104+
XCODE_ATTRIBUTE_${XCODE_PROPERTY} ${XCODE_VALUE})
105+
endmacro (set_xcode_property)
106+
endif()
107+
add_subdirectory(sentencepiece sentencepiece EXCLUDE_FROM_ALL)
108+
109+
add_library(tokenizers_c INTERFACE ${TOKENIZERS_RUST_LIB})
110+
target_link_libraries(tokenizers_c INTERFACE ${TOKENIZERS_RUST_LIB})
111+
112+
add_library(tokenizers_cpp STATIC $<TARGET_OBJECTS:tokenizer_cpp_objs>)
113+
target_link_libraries(tokenizers_cpp PRIVATE tokenizers_c sentencepiece-static ${TOKENIZERS_CPP_LINK_LIBS})
114+
target_include_directories(tokenizers_cpp PUBLIC ${TOKENIZERS_CPP_INCLUDE})

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
# tokenizers-cpp
2-
Universal tokenizer binding to HF and sentencepiece
2+
3+
Cross platform universal tokenizer binding to HF and sentencepiece

‎include/tokenizers_cpp.h‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*!
2+
* Copyright (c) 2023 by Contributors
3+
* \file tokenizers_cpp.h
4+
* \brief A C++ binding to common set of tokenizers
5+
*/
6+
#ifndef TOKENIZERS_CPP_H_
7+
#define TOKENIZERS_CPP_H_
8+
9+
#include <memory>
10+
#include <string>
11+
#include <vector>
12+
13+
namespace tokenizers {
14+
15+
/*!
16+
* \brief a universal tokenizer that loads
17+
* either HF's tokenizer or sentence piece,
18+
* depending on the constructor
19+
*/
20+
class Tokenizer {
21+
public:
22+
/*! \brief virtual destructor */
23+
virtual ~Tokenizer() {}
24+
25+
/*!
26+
* \brief Encode text into ids.
27+
* \param text The input text.
28+
* \returns The encoded token ids.
29+
*/
30+
virtual std::vector<int32_t> Encode(const std::string& text) = 0;
31+
32+
/*!
33+
* \brief Decode token ids into text.
34+
* \param text The token ids.
35+
* \returns The decoded text.
36+
*/
37+
virtual std::string Decode(const std::vector<int32_t>& ids) = 0;
38+
39+
//---------------------------------------------------
40+
// Factory functions from byte-blobs
41+
// These factory function takes in in-memory blobs
42+
// so the library can be independent from filesystem
43+
//---------------------------------------------------
44+
/*!
45+
* \brief Create HF tokenizer from a single in-memory json blob.
46+
*
47+
* \param json_blob The json blob.
48+
* \return The created tokenzier.
49+
*/
50+
static std::unique_ptr<Tokenizer> FromBlobJSON(const std::string& json_blob);
51+
/*!
52+
* \brief Create BPE tokenizer
53+
*
54+
* \param vocab_blob The blob that contains vocabs.
55+
* \param merges_blob The blob that contains the merges.
56+
* \param added_tokens The added tokens.
57+
* \return The created tokenizer.
58+
*/
59+
static std::unique_ptr<Tokenizer> FromBlobByteLevelBPE(const std::string& vocab_blob,
60+
const std::string& merges_blob,
61+
const std::string& added_tokens = "");
62+
/*!
63+
* \brief Create SentencePiece.
64+
*
65+
* \param model_blob The blob that contains vocabs.
66+
* \return The created tokenizer.
67+
*/
68+
static std::unique_ptr<Tokenizer> FromBlobSentencePiece(const std::string& model_blob);
69+
};
70+
71+
} // namespace tokenizers
72+
#endif // TOKENIZERS_CPP_H_

‎rust/Cargo.toml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[package]
2+
name = "tokenizers-c"
3+
version = "0.1.0"
4+
edition = "2018"
5+
6+
[lib]
7+
crate-type = ["staticlib"]
8+
9+
[dependencies]
10+
11+
tokenizers = { version = "0.13.3", default-features = false, features = ["onig"] }
12+
serde = { version = "1.0", features = [ "derive" ] }
13+
serde_json = "1.0"

0 commit comments

Comments
 (0)