Use tokenizers add_subdirectory in MediaTek CMake build (#17683)

The MediaTek CMakeLists.txt duplicated the entire tokenizers build
setup: abseil, re2, json, unicode, pcre2 subdirectories, a hardcoded
source file list, and regex_lookahead configuration. This list was
missing post_processor.cpp, causing linker errors for undefined
PostProcessorConfig symbols.

Replace ~110 lines of duplicated build logic with a single
add_subdirectory call to the canonical tokenizers CMakeLists.txt, which
already handles all dependencies and source files correctly.

Author: larryliu0820

examples/mediatek/CMakeLists.txt

@@ -116,116 +116,11 @@ if(${ANDROID})
   list(PREPEND _mtk_llama_executor_runner__srcs
        ${CMAKE_CURRENT_LIST_DIR}/executor_runner/mtk_llama_executor_runner.cpp
   )
-  # Build ABSL and RE2
-  set(EXTENSIONS_LLM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../extension/llm)
-  set(THIRD_PARTY_ABSL_DIR
-      ${EXTENSIONS_LLM_DIR}/tokenizers/third-party/abseil-cpp
-  )
-  set(THIRD_PARTY_RE2_DIR ${EXTENSIONS_LLM_DIR}/tokenizers/third-party/re2)
-  set(THIRD_PARTY_JSON_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/json)
-  set(THIRD_PARTY_UNICODE_DIR
-      ${EXTENSIONS_LLM_DIR}/tokenizers/third-party/llama.cpp-unicode
-  )
-  set(THIRD_PARTY_PCRE2_DIR ${EXTENSIONS_LLM_DIR}/tokenizers/third-party/pcre2)
-  set(ABSL_ENABLE_INSTALL ON)
-  set(ABSL_PROPAGATE_CXX_STD ON)
-  set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE})
-  set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-  add_subdirectory(
-    ${THIRD_PARTY_ABSL_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/abseil
-  )
-  add_subdirectory(
-    ${THIRD_PARTY_RE2_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/re2
-  )
-  add_subdirectory(
-    ${THIRD_PARTY_JSON_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/json
-  )
-  add_subdirectory(
-    ${THIRD_PARTY_UNICODE_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/llama.cpp-unicode
-  )
-  add_subdirectory(
-    ${THIRD_PARTY_PCRE2_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/pcre2
-  )
-  set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag})
-
-  # Build tokenizers
+  # Build tokenizers library
   set(SUPPORT_REGEX_LOOKAHEAD ON)
-  set(LLAMA2_TOKENIZER_DIR ${EXTENSIONS_LLM_DIR}/tokenizers)
-  add_library(tokenizer STATIC)
-  target_include_directories(
-    tokenizer
-    PUBLIC ${_common_include_directories}
-           ${THIRD_PARTY_ABSL_DIR}
-           ${THIRD_PARTY_RE2_DIR}
-           ${LLAMA2_TOKENIZER_DIR}/include
-           ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/pcre2
-           ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
-           ${THIRD_PARTY_JSON_DIR}
-           ${THIRD_PARTY_UNICODE_DIR}/include
-           ${THIRD_PARTY_PCRE2_DIR}
-  )
-  target_link_libraries(tokenizer PRIVATE re2::re2)
-
-  target_sources(
-    tokenizer
-    PRIVATE
-      ${LLAMA2_TOKENIZER_DIR}/src/tiktoken.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/llama2c_tokenizer.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/regex.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/bpe_tokenizer_base.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/re2_regex.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/hf_tokenizer.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/pre_tokenizer.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/token_decoder.cpp
-      ${LLAMA2_TOKENIZER_DIR}/src/normalizer.cpp
-      ${LLAMA2_TOKENIZER_DIR}/third-party/llama.cpp-unicode/src/unicode.cpp
-      ${LLAMA2_TOKENIZER_DIR}/third-party/llama.cpp-unicode/src/unicode-data.cpp
-      ${CMAKE_CURRENT_SOURCE_DIR}/../models/llama/tokenizer/llama_tiktoken.cpp
-  )
-
-  # Add support for regex_lookahead
-  set(PCRE2_STATIC_PIC ON)
-  set(PCRE2_BUILD_PCRE2_8 ON)
-  set(PCRE2_BUILD_PCRE2_16 OFF)
-  set(PCRE2_BUILD_PCRE2_32 OFF)
-  set(PCRE2_BUILD_TESTS OFF)
-  set(PCRE2_BUILD_PCRE2GREP OFF)
-  set(PCRE2_BUILD_PCRE2TEST OFF)
-  set(PCRE2_BUILD_PCRE2GPERF OFF)
-  set(PCRE2_BUILD_DOCS OFF)
-  set(PCRE2_BUILD_LIBPCRE2_PDB OFF)
-
-  # Set the INTERFACE_INCLUDE_DIRECTORIES property for pcre2-8-static
-  set_target_properties(
-    pcre2-8-static
-    PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/pcre2>
-  )
-  add_library(
-    regex_lookahead STATIC
-    ${LLAMA2_TOKENIZER_DIR}/src/pcre2_regex.cpp
-    ${LLAMA2_TOKENIZER_DIR}/src/regex_lookahead.cpp
-    ${LLAMA2_TOKENIZER_DIR}/src/std_regex.cpp
-  )
-  add_library(tokenizer::regex_lookahead ALIAS regex_lookahead)
-  target_link_libraries(regex_lookahead PUBLIC pcre2-8-static)
-  target_include_directories(
-    regex_lookahead
-    PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-  )
-  target_link_libraries(tokenizer PUBLIC regex_lookahead)
-  install(
-    TARGETS regex_lookahead pcre2-8-static
-    EXPORT tokenizers-targets
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  add_subdirectory(
+    ${EXECUTORCH_ROOT}/extension/llm/tokenizers
+    ${CMAKE_CURRENT_BINARY_DIR}/tokenizers
   )
 
   # Include directory for neuron headers
@@ -244,10 +139,12 @@ if(${ANDROID})
     mtk_llama_executor_runner ${_executor_runner_libs} neuron_backend gflags
     mtk_llama_executor_lib
   )
-  target_link_libraries(
-    mtk_llama_executor_runner tokenizer
-    $<LINK_LIBRARY:WHOLE_ARCHIVE,regex_lookahead>
+  target_sources(
+    mtk_llama_executor_runner
+    PRIVATE
+      ${CMAKE_CURRENT_SOURCE_DIR}/../models/llama/tokenizer/llama_tiktoken.cpp
   )
+  target_link_libraries(mtk_llama_executor_runner tokenizers)
   target_compile_options(
     mtk_llama_executor_runner PUBLIC ${_common_compile_options}
   )