Skip to content

Commit d1e01cc

Browse files
DennisadiraDennisadira
committed
feat(api): add /v1/detokenize endpoint
Closes #1649. Mirror of the existing /v1/tokenize path, requested by @benniekiss in the issue thread for "complete API workflow" use cases that need to turn token IDs back into text without local processing. - Add Detokenize gRPC RPC with DetokenizeRequest{tokens} / DetokenizeResponse{content} messages. - Implement in the llama.cpp backend using common_token_to_piece, the same primitive TokenizeString already uses internally. - Other backends inherit the default Unimplemented from base.Base, in line with how Detect, Rerank, etc. are gated per-backend. - Wire up the Go gRPC interface, server, client, and in-process embed wrapper alongside their TokenizeString counterparts. - Add the schema types, ModelDetokenize wrapper, HTTP handler, route registration, RouteFeatureRegistry entry (gated by FeatureTokenize so no new feature flag is needed), and the discovery map entry under ai_functions. - Regenerated swagger reflects the new endpoint and types. - Update authentication.md to list /v1/detokenize alongside /v1/tokenize. Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Adira Denis Muhando <dennisadira@gmail.com>
1 parent 415b561 commit d1e01cc

19 files changed

Lines changed: 337 additions & 6 deletions

File tree

‎backend/backend.proto‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ service Backend {
2222
rpc TTSStream(TTSRequest) returns (stream Reply) {}
2323
rpc SoundGeneration(SoundGenerationRequest) returns (Result) {}
2424
rpc TokenizeString(PredictOptions) returns (TokenizationResponse) {}
25+
rpc Detokenize(DetokenizeRequest) returns (DetokenizeResponse) {}
2526
rpc Status(HealthMessage) returns (StatusResponse) {}
2627
rpc Detect(DetectOptions) returns (DetectResponse) {}
2728
rpc FaceVerify(FaceVerifyRequest) returns (FaceVerifyResponse) {}
@@ -613,6 +614,14 @@ message TokenizationResponse {
613614
repeated int32 tokens = 2;
614615
}
615616

617+
message DetokenizeRequest {
618+
repeated int32 tokens = 1;
619+
}
620+
621+
message DetokenizeResponse {
622+
string content = 1;
623+
}
624+
616625
message MemoryUsageData {
617626
uint64 total = 1;
618627
map<string, uint64> breakdown = 2;

‎backend/cpp/llama-cpp/grpc-server.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3448,6 +3448,21 @@ class BackendServiceImpl final : public backend::Backend::Service {
34483448
return grpc::Status::OK;
34493449
}
34503450

3451+
grpc::Status Detokenize(ServerContext* context, const backend::DetokenizeRequest* request, backend::DetokenizeResponse* response) override {
3452+
auto auth = checkAuth(context);
3453+
if (!auth.ok()) return auth;
3454+
if (params_base.model.path.empty()) {
3455+
return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
3456+
}
3457+
3458+
std::string content;
3459+
for (const auto token : request->tokens()) {
3460+
content.append(common_token_to_piece(ctx_server.get_llama_context(), token));
3461+
}
3462+
response->set_content(content);
3463+
return grpc::Status::OK;
3464+
}
3465+
34513466
grpc::Status GetMetrics(ServerContext* /*context*/, const backend::MetricsRequest* /*request*/, backend::MetricsResponse* response) override {
34523467

34533468
conflict_guard guard("GetMetrics", slot_loop_inflight, score_inflight, "score_inflight");

‎core/backend/detokenize.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package backend
2+
3+
import (
4+
"time"
5+
6+
"github.com/mudler/LocalAI/core/config"
7+
"github.com/mudler/LocalAI/core/schema"
8+
"github.com/mudler/LocalAI/core/trace"
9+
"github.com/mudler/LocalAI/pkg/grpc"
10+
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
11+
"github.com/mudler/LocalAI/pkg/model"
12+
)
13+
14+
func ModelDetokenize(tokens []int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (schema.DetokenizeResponse, error) {
15+
16+
var inferenceModel grpc.Backend
17+
var err error
18+
19+
opts := ModelOptions(modelConfig, appConfig)
20+
inferenceModel, err = loader.Load(opts...)
21+
if err != nil {
22+
recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
23+
return schema.DetokenizeResponse{}, err
24+
}
25+
26+
var startTime time.Time
27+
if appConfig.EnableTracing {
28+
trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
29+
startTime = time.Now()
30+
}
31+
32+
resp, err := inferenceModel.Detokenize(appConfig.Context, &pb.DetokenizeRequest{Tokens: tokens})
33+
34+
if appConfig.EnableTracing {
35+
errStr := ""
36+
if err != nil {
37+
errStr = err.Error()
38+
}
39+
40+
content := ""
41+
if resp != nil {
42+
content = resp.Content
43+
}
44+
45+
trace.RecordBackendTrace(trace.BackendTrace{
46+
Timestamp: startTime,
47+
Duration: time.Since(startTime),
48+
Type: trace.BackendTraceTokenize,
49+
ModelName: modelConfig.Name,
50+
Backend: modelConfig.Backend,
51+
Summary: trace.TruncateString(content, 200),
52+
Error: errStr,
53+
Data: map[string]any{
54+
"token_count": len(tokens),
55+
"output_text": trace.TruncateString(content, 1000),
56+
},
57+
})
58+
}
59+
60+
if err != nil {
61+
return schema.DetokenizeResponse{}, err
62+
}
63+
64+
return schema.DetokenizeResponse{
65+
Content: resp.Content,
66+
}, nil
67+
}

‎core/http/auth/features.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ var RouteFeatureRegistry = []RouteFeature{
101101

102102
// Tokenize
103103
{"POST", "/v1/tokenize", FeatureTokenize},
104+
{"POST", "/v1/detokenize", FeatureTokenize},
104105

105106
// Rerank
106107
{"POST", "/v1/rerank", FeatureRerank},

‎core/http/endpoints/localai/detokenize.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package localai
2+
3+
import (
4+
"github.com/labstack/echo/v4"
5+
"github.com/mudler/LocalAI/core/backend"
6+
"github.com/mudler/LocalAI/core/config"
7+
"github.com/mudler/LocalAI/core/http/middleware"
8+
"github.com/mudler/LocalAI/core/schema"
9+
"github.com/mudler/LocalAI/pkg/model"
10+
)
11+
12+
// DetokenizeEndpoint exposes a REST API to convert token IDs back to text.
13+
// @Summary Detokenize the input.
14+
// @Tags tokenize
15+
// @Param request body schema.DetokenizeRequest true "Request"
16+
// @Success 200 {object} schema.DetokenizeResponse "Response"
17+
// @Router /v1/detokenize [post]
18+
func DetokenizeEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc {
19+
return func(c echo.Context) error {
20+
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.DetokenizeRequest)
21+
if !ok || input.Model == "" {
22+
return echo.ErrBadRequest
23+
}
24+
25+
cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
26+
if !ok || cfg == nil {
27+
return echo.ErrBadRequest
28+
}
29+
30+
resp, err := backend.ModelDetokenize(input.Tokens, ml, *cfg, appConfig)
31+
if err != nil {
32+
return err
33+
}
34+
return c.JSON(200, resp)
35+
}
36+
}

‎core/http/routes/localai.go‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -299,11 +299,12 @@ func RegisterLocalAIRoutes(router *echo.Echo,
299299
"reload": "/models/reload",
300300
},
301301
"ai_functions": map[string]string{
302-
"tts": "/tts",
303-
"vad": "/vad",
304-
"video": "/video",
305-
"detection": "/v1/detection",
306-
"tokenize": "/v1/tokenize",
302+
"tts": "/tts",
303+
"vad": "/vad",
304+
"video": "/video",
305+
"detection": "/v1/detection",
306+
"tokenize": "/v1/tokenize",
307+
"detokenize": "/v1/detokenize",
307308
},
308309
"monitoring": monitoringRoutes,
309310
"mcp": map[string]string{
@@ -372,6 +373,12 @@ func RegisterLocalAIRoutes(router *echo.Echo,
372373
requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)),
373374
requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.TokenizeRequest) }))
374375

376+
detokenizeHandler := localai.DetokenizeEndpoint(cl, ml, appConfig)
377+
router.POST("/v1/detokenize",
378+
detokenizeHandler,
379+
requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)),
380+
requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.DetokenizeRequest) }))
381+
375382
// MCP endpoint - supports both streaming and non-streaming modes
376383
// Note: streaming mode is NOT compatible with the OpenAI apis. We have a set which streams more states.
377384
if evaluator != nil && !appConfig.DisableMCP {

‎core/schema/tokenize.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,12 @@ type TokenizeRequest struct {
88
type TokenizeResponse struct {
99
Tokens []int32 `json:"tokens"` // token IDs
1010
}
11+
12+
type DetokenizeRequest struct {
13+
BasicModelRequest
14+
Tokens []int32 `json:"tokens"` // token IDs to convert back to text
15+
}
16+
17+
type DetokenizeResponse struct {
18+
Content string `json:"content"` // detokenized text
19+
}

‎core/services/nodes/health_mock_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ func (c *fakeBackendClient) AudioTranscriptionStream(_ context.Context, _ *pb.Tr
193193
func (c *fakeBackendClient) TokenizeString(_ context.Context, _ *pb.PredictOptions, _ ...ggrpc.CallOption) (*pb.TokenizationResponse, error) {
194194
return nil, nil
195195
}
196+
func (c *fakeBackendClient) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) {
197+
return nil, nil
198+
}
196199
func (c *fakeBackendClient) Status(_ context.Context) (*pb.StatusResponse, error) {
197200
return nil, nil
198201
}

‎core/services/nodes/inflight_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,10 @@ func (f *fakeGRPCBackend) TokenizeString(_ context.Context, _ *pb.PredictOptions
124124
return &pb.TokenizationResponse{}, nil
125125
}
126126

127+
func (f *fakeGRPCBackend) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) {
128+
return &pb.DetokenizeResponse{}, nil
129+
}
130+
127131
func (f *fakeGRPCBackend) Status(_ context.Context) (*pb.StatusResponse, error) {
128132
return &pb.StatusResponse{}, nil
129133
}

‎docs/content/features/authentication.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ When authentication is enabled, the following endpoints require admin role:
176176
**User-Accessible Endpoints (all authenticated users):**
177177
- `POST /v1/chat/completions`, `POST /v1/embeddings`, `POST /v1/completions`
178178
- `POST /v1/images/generations`, `POST /v1/audio/*`, `POST /tts`, `POST /vad`, `POST /video`
179-
- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detection`
179+
- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detokenize`, `POST /v1/detection`
180180
- `POST /v1/mcp/chat/completions`, `POST /v1/messages`, `POST /v1/responses`
181181
- `POST /stores/*`, `GET /api/cors-proxy`
182182
- `GET /version`, `GET /api/features`, `GET /swagger/*`, `GET /metrics`

0 commit comments

Comments
 (0)