Skip to content

Commit b2f4b7e

Browse files
DennisadiraDennisadira
committed
feat(api): add /v1/detokenize endpoint
Closes #1649. Mirror of the existing /v1/tokenize path, requested by @benniekiss in the issue thread for "complete API workflow" use cases that need to turn token IDs back into text without local processing. - Add Detokenize gRPC RPC with DetokenizeRequest{tokens} / DetokenizeResponse{content} messages. - Implement in the llama.cpp backend using common_token_to_piece, the same primitive TokenizeString already uses internally. - Other backends inherit the default Unimplemented from base.Base, in line with how Detect, Rerank, etc. are gated per-backend. - Wire up the Go gRPC interface, server, client, and in-process embed wrapper alongside their TokenizeString counterparts. - Add the schema types, ModelDetokenize wrapper, HTTP handler, route registration, RouteFeatureRegistry entry (gated by FeatureTokenize so no new feature flag is needed), and the discovery map entry under ai_functions. - Regenerated swagger reflects the new endpoint and types. - Update authentication.md to list /v1/detokenize alongside /v1/tokenize. Assisted-by: Claude:claude-opus-4-7
1 parent ebd9fcb commit b2f4b7e

19 files changed

Lines changed: 337 additions & 6 deletions

File tree

‎backend/backend.proto‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ service Backend {
2222
rpc TTSStream(TTSRequest) returns (stream Reply) {}
2323
rpc SoundGeneration(SoundGenerationRequest) returns (Result) {}
2424
rpc TokenizeString(PredictOptions) returns (TokenizationResponse) {}
25+
rpc Detokenize(DetokenizeRequest) returns (DetokenizeResponse) {}
2526
rpc Status(HealthMessage) returns (StatusResponse) {}
2627
rpc Detect(DetectOptions) returns (DetectResponse) {}
2728
rpc FaceVerify(FaceVerifyRequest) returns (FaceVerifyResponse) {}
@@ -437,6 +438,14 @@ message TokenizationResponse {
437438
repeated int32 tokens = 2;
438439
}
439440

441+
message DetokenizeRequest {
442+
repeated int32 tokens = 1;
443+
}
444+
445+
message DetokenizeResponse {
446+
string content = 1;
447+
}
448+
440449
message MemoryUsageData {
441450
uint64 total = 1;
442451
map<string, uint64> breakdown = 2;

‎backend/cpp/llama-cpp/grpc-server.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2790,6 +2790,21 @@ class BackendServiceImpl final : public backend::Backend::Service {
27902790
return grpc::Status::OK;
27912791
}
27922792

2793+
grpc::Status Detokenize(ServerContext* context, const backend::DetokenizeRequest* request, backend::DetokenizeResponse* response) override {
2794+
auto auth = checkAuth(context);
2795+
if (!auth.ok()) return auth;
2796+
if (params_base.model.path.empty()) {
2797+
return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
2798+
}
2799+
2800+
std::string content;
2801+
for (const auto token : request->tokens()) {
2802+
content.append(common_token_to_piece(ctx_server.get_llama_context(), token));
2803+
}
2804+
response->set_content(content);
2805+
return grpc::Status::OK;
2806+
}
2807+
27932808
grpc::Status GetMetrics(ServerContext* /*context*/, const backend::MetricsRequest* /*request*/, backend::MetricsResponse* response) override {
27942809

27952810
// request slots data using task queue

‎core/backend/detokenize.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package backend
2+
3+
import (
4+
"time"
5+
6+
"github.com/mudler/LocalAI/core/config"
7+
"github.com/mudler/LocalAI/core/schema"
8+
"github.com/mudler/LocalAI/core/trace"
9+
"github.com/mudler/LocalAI/pkg/grpc"
10+
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
11+
"github.com/mudler/LocalAI/pkg/model"
12+
)
13+
14+
func ModelDetokenize(tokens []int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (schema.DetokenizeResponse, error) {
15+
16+
var inferenceModel grpc.Backend
17+
var err error
18+
19+
opts := ModelOptions(modelConfig, appConfig)
20+
inferenceModel, err = loader.Load(opts...)
21+
if err != nil {
22+
recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
23+
return schema.DetokenizeResponse{}, err
24+
}
25+
26+
var startTime time.Time
27+
if appConfig.EnableTracing {
28+
trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
29+
startTime = time.Now()
30+
}
31+
32+
resp, err := inferenceModel.Detokenize(appConfig.Context, &pb.DetokenizeRequest{Tokens: tokens})
33+
34+
if appConfig.EnableTracing {
35+
errStr := ""
36+
if err != nil {
37+
errStr = err.Error()
38+
}
39+
40+
content := ""
41+
if resp != nil {
42+
content = resp.Content
43+
}
44+
45+
trace.RecordBackendTrace(trace.BackendTrace{
46+
Timestamp: startTime,
47+
Duration: time.Since(startTime),
48+
Type: trace.BackendTraceTokenize,
49+
ModelName: modelConfig.Name,
50+
Backend: modelConfig.Backend,
51+
Summary: trace.TruncateString(content, 200),
52+
Error: errStr,
53+
Data: map[string]any{
54+
"token_count": len(tokens),
55+
"output_text": trace.TruncateString(content, 1000),
56+
},
57+
})
58+
}
59+
60+
if err != nil {
61+
return schema.DetokenizeResponse{}, err
62+
}
63+
64+
return schema.DetokenizeResponse{
65+
Content: resp.Content,
66+
}, nil
67+
}

‎core/http/auth/features.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ var RouteFeatureRegistry = []RouteFeature{
9292

9393
// Tokenize
9494
{"POST", "/v1/tokenize", FeatureTokenize},
95+
{"POST", "/v1/detokenize", FeatureTokenize},
9596

9697
// Rerank
9798
{"POST", "/v1/rerank", FeatureRerank},

‎core/http/endpoints/localai/detokenize.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package localai
2+
3+
import (
4+
"github.com/labstack/echo/v4"
5+
"github.com/mudler/LocalAI/core/backend"
6+
"github.com/mudler/LocalAI/core/config"
7+
"github.com/mudler/LocalAI/core/http/middleware"
8+
"github.com/mudler/LocalAI/core/schema"
9+
"github.com/mudler/LocalAI/pkg/model"
10+
)
11+
12+
// DetokenizeEndpoint exposes a REST API to convert token IDs back to text.
13+
// @Summary Detokenize the input.
14+
// @Tags tokenize
15+
// @Param request body schema.DetokenizeRequest true "Request"
16+
// @Success 200 {object} schema.DetokenizeResponse "Response"
17+
// @Router /v1/detokenize [post]
18+
func DetokenizeEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc {
19+
return func(c echo.Context) error {
20+
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.DetokenizeRequest)
21+
if !ok || input.Model == "" {
22+
return echo.ErrBadRequest
23+
}
24+
25+
cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
26+
if !ok || cfg == nil {
27+
return echo.ErrBadRequest
28+
}
29+
30+
resp, err := backend.ModelDetokenize(input.Tokens, ml, *cfg, appConfig)
31+
if err != nil {
32+
return err
33+
}
34+
return c.JSON(200, resp)
35+
}
36+
}

‎core/http/routes/localai.go‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -274,11 +274,12 @@ func RegisterLocalAIRoutes(router *echo.Echo,
274274
"reload": "/models/reload",
275275
},
276276
"ai_functions": map[string]string{
277-
"tts": "/tts",
278-
"vad": "/vad",
279-
"video": "/video",
280-
"detection": "/v1/detection",
281-
"tokenize": "/v1/tokenize",
277+
"tts": "/tts",
278+
"vad": "/vad",
279+
"video": "/video",
280+
"detection": "/v1/detection",
281+
"tokenize": "/v1/tokenize",
282+
"detokenize": "/v1/detokenize",
282283
},
283284
"monitoring": monitoringRoutes,
284285
"mcp": map[string]string{
@@ -347,6 +348,12 @@ func RegisterLocalAIRoutes(router *echo.Echo,
347348
requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)),
348349
requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.TokenizeRequest) }))
349350

351+
detokenizeHandler := localai.DetokenizeEndpoint(cl, ml, appConfig)
352+
router.POST("/v1/detokenize",
353+
detokenizeHandler,
354+
requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)),
355+
requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.DetokenizeRequest) }))
356+
350357
// MCP endpoint - supports both streaming and non-streaming modes
351358
// Note: streaming mode is NOT compatible with the OpenAI apis. We have a set which streams more states.
352359
if evaluator != nil && !appConfig.DisableMCP {

‎core/schema/tokenize.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,12 @@ type TokenizeRequest struct {
88
type TokenizeResponse struct {
99
Tokens []int32 `json:"tokens"` // token IDs
1010
}
11+
12+
type DetokenizeRequest struct {
13+
BasicModelRequest
14+
Tokens []int32 `json:"tokens"` // token IDs to convert back to text
15+
}
16+
17+
type DetokenizeResponse struct {
18+
Content string `json:"content"` // detokenized text
19+
}

‎core/services/nodes/health_mock_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ func (c *fakeBackendClient) AudioTranscriptionStream(_ context.Context, _ *pb.Tr
193193
func (c *fakeBackendClient) TokenizeString(_ context.Context, _ *pb.PredictOptions, _ ...ggrpc.CallOption) (*pb.TokenizationResponse, error) {
194194
return nil, nil
195195
}
196+
func (c *fakeBackendClient) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) {
197+
return nil, nil
198+
}
196199
func (c *fakeBackendClient) Status(_ context.Context) (*pb.StatusResponse, error) {
197200
return nil, nil
198201
}

‎core/services/nodes/inflight_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,10 @@ func (f *fakeGRPCBackend) TokenizeString(_ context.Context, _ *pb.PredictOptions
123123
return &pb.TokenizationResponse{}, nil
124124
}
125125

126+
func (f *fakeGRPCBackend) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) {
127+
return &pb.DetokenizeResponse{}, nil
128+
}
129+
126130
func (f *fakeGRPCBackend) Status(_ context.Context) (*pb.StatusResponse, error) {
127131
return &pb.StatusResponse{}, nil
128132
}

‎docs/content/features/authentication.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ When authentication is enabled, the following endpoints require admin role:
176176
**User-Accessible Endpoints (all authenticated users):**
177177
- `POST /v1/chat/completions`, `POST /v1/embeddings`, `POST /v1/completions`
178178
- `POST /v1/images/generations`, `POST /v1/audio/*`, `POST /tts`, `POST /vad`, `POST /video`
179-
- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detection`
179+
- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detokenize`, `POST /v1/detection`
180180
- `POST /v1/mcp/chat/completions`, `POST /v1/messages`, `POST /v1/responses`
181181
- `POST /stores/*`, `GET /api/cors-proxy`
182182
- `GET /version`, `GET /api/features`, `GET /swagger/*`, `GET /metrics`

0 commit comments

Comments
 (0)