bugfix: forward REC tokenizer methods through proxy. (#1249)

Author: DragonFive

xllm/core/framework/tokenizer/tokenizer_proxy.cpp

@@ -36,11 +36,22 @@ bool TokenizerProxy::encode(const std::string_view& text,
   return get_tls_tokenizer()->encode(text, ids, add_special_tokens);
 }
 
+bool TokenizerProxy::encode(int64_t item_id,
+                            std::vector<int32_t>* token_ids) const {
+  return get_tls_tokenizer()->encode(item_id, token_ids);
+}
+
 std::string TokenizerProxy::decode(const Slice<int32_t>& ids,
                                    bool skip_special_tokens) const {
   return get_tls_tokenizer()->decode(ids, skip_special_tokens);
 }
 
+bool TokenizerProxy::decode(const Slice<int32_t>& token_ids,
+                            bool skip_special_tokens,
+                            std::vector<int64_t>* item_ids) const {
+  return get_tls_tokenizer()->decode(token_ids, skip_special_tokens, item_ids);
+}
+
 std::optional<int32_t> TokenizerProxy::token_to_id(
     const std::string_view& token) const {
   return get_tls_tokenizer()->token_to_id(token);
@@ -58,4 +69,4 @@ Tokenizer* TokenizerProxy::get_tls_tokenizer() const {
   thread_local std::unique_ptr<Tokenizer> tls_tokenizer(tokenizer_->clone());
   return tls_tokenizer.get();
 }
-}  // namespace xllm
+}  // namespace xllm

xllm/core/framework/tokenizer/tokenizer_proxy.h

@@ -29,9 +29,15 @@ class TokenizerProxy : public Tokenizer {
               std::vector<int32_t>* ids,
               bool add_special_tokens = true) const override;
 
+  bool encode(int64_t item_id, std::vector<int32_t>* token_ids) const override;
+
   std::string decode(const Slice<int32_t>& ids,
                      bool skip_special_tokens) const override;
 
+  bool decode(const Slice<int32_t>& token_ids,
+              bool skip_special_tokens,
+              std::vector<int64_t>* item_ids) const override;
+
   std::optional<int32_t> token_to_id(
       const std::string_view& token) const override;
 
@@ -46,4 +52,4 @@ class TokenizerProxy : public Tokenizer {
 
   std::unique_ptr<Tokenizer> tokenizer_;
 };
-}  // namespace xllm
+}  // namespace xllm