feat(api): add ollama compatibility (#9284)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

core/cli/run.go

@@ -62,6 +62,7 @@ type RunCMD struct {
 	UploadLimit                        int      `env:"LOCALAI_UPLOAD_LIMIT,UPLOAD_LIMIT" default:"15" help:"Default upload-limit in MB" group:"api"`
 	APIKeys                            []string `env:"LOCALAI_API_KEY,API_KEY" help:"List of API Keys to enable API authentication. When this is set, all the requests must be authenticated with one of these API keys" group:"api"`
 	DisableWebUI                       bool     `env:"LOCALAI_DISABLE_WEBUI,DISABLE_WEBUI" default:"false" help:"Disables the web user interface. When set to true, the server will only expose API endpoints without serving the web interface" group:"api"`
+	OllamaAPIRootEndpoint              bool     `env:"LOCALAI_OLLAMA_API_ROOT_ENDPOINT" default:"false" help:"Register Ollama-compatible health check on / (replaces web UI on root path). The /api/* Ollama endpoints are always available regardless of this flag" group:"api"`
 	DisableRuntimeSettings             bool     `env:"LOCALAI_DISABLE_RUNTIME_SETTINGS,DISABLE_RUNTIME_SETTINGS" default:"false" help:"Disables the runtime settings. When set to true, the server will not load the runtime settings from the runtime_settings.json file" group:"api"`
 	DisablePredownloadScan             bool     `env:"LOCALAI_DISABLE_PREDOWNLOAD_SCAN" help:"If true, disables the best-effort security scanner before downloading any files." group:"hardening" default:"false"`
 	OpaqueErrors                       bool     `env:"LOCALAI_OPAQUE_ERRORS" default:"false" help:"If true, all error responses are replaced with blank 500 errors. This is intended only for hardening against information leaks and is normally not recommended." group:"hardening"`
@@ -295,6 +296,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		opts = append(opts, config.DisableWebUI)
 	}
 
+	if r.OllamaAPIRootEndpoint {
+		opts = append(opts, config.EnableOllamaAPIRootEndpoint)
+	}
+
 	if r.DisableGalleryEndpoint {
 		opts = append(opts, config.DisableGalleryEndpoint)
 	}

core/config/application_config.go

@@ -40,6 +40,7 @@ type ApplicationConfig struct {
 	Federated                     bool
 
 	DisableWebUI                       bool
+	OllamaAPIRootEndpoint              bool
 	EnforcePredownloadScans            bool
 	OpaqueErrors                       bool
 	UseSubtleKeyComparison             bool
@@ -263,6 +264,10 @@ var DisableWebUI = func(o *ApplicationConfig) {
 	o.DisableWebUI = true
 }
 
+var EnableOllamaAPIRootEndpoint = func(o *ApplicationConfig) {
+	o.OllamaAPIRootEndpoint = true
+}
+
 var DisableRuntimeSettings = func(o *ApplicationConfig) {
 	o.DisableRuntimeSettings = true
 }

core/http/app.go

@@ -391,6 +391,10 @@ func API(application *application.Application) (*echo.Echo, error) {
 	routes.RegisterOpenAIRoutes(e, requestExtractor, application)
 	routes.RegisterAnthropicRoutes(e, requestExtractor, application)
 	routes.RegisterOpenResponsesRoutes(e, requestExtractor, application)
+	routes.RegisterOllamaRoutes(e, requestExtractor, application)
+	if application.ApplicationConfig().OllamaAPIRootEndpoint {
+		routes.RegisterOllamaRootEndpoint(e)
+	}
 	if !application.ApplicationConfig().DisableWebUI {
 		routes.RegisterUIAPIRoutes(e, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService(), opcache, application, adminMiddleware)
 		routes.RegisterUIRoutes(e, application.ModelConfigLoader(), application.ApplicationConfig(), application.GalleryService(), adminMiddleware)

core/http/endpoints/ollama/chat.go

@@ -0,0 +1,153 @@
+package ollama
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/backend"
+	"github.com/mudler/LocalAI/core/config"
+	openaiEndpoint "github.com/mudler/LocalAI/core/http/endpoints/openai"
+	"github.com/mudler/LocalAI/core/http/middleware"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/core/templates"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/xlog"
+)
+
+// ChatEndpoint handles Ollama-compatible /api/chat requests
+func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OllamaChatRequest)
+		if !ok || input.Model == "" {
+			return ollamaError(c, 400, "model is required")
+		}
+
+		cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
+		if !ok || cfg == nil {
+			return ollamaError(c, 400, "model configuration not found")
+		}
+
+		// Apply Ollama options to config
+		applyOllamaOptions(input.Options, cfg)
+
+		// Convert Ollama messages to OpenAI format
+		openAIMessages := ollamaMessagesToOpenAI(input.Messages)
+
+		// Build an OpenAI-compatible request
+		openAIReq := &schema.OpenAIRequest{
+			PredictionOptions: schema.PredictionOptions{
+				BasicModelRequest: schema.BasicModelRequest{Model: input.Model},
+			},
+			Messages: openAIMessages,
+			Stream:   input.IsStream(),
+			Context:  input.Context,
+			Cancel:   input.Cancel,
+		}
+
+		if input.Options != nil {
+			openAIReq.Temperature = input.Options.Temperature
+			openAIReq.TopP = input.Options.TopP
+			openAIReq.TopK = input.Options.TopK
+			openAIReq.RepeatPenalty = input.Options.RepeatPenalty
+			if input.Options.NumPredict != nil {
+				openAIReq.Maxtokens = input.Options.NumPredict
+			}
+			if len(input.Options.Stop) > 0 {
+				openAIReq.Stop = input.Options.Stop
+			}
+		}
+
+		predInput := evaluator.TemplateMessages(*openAIReq, openAIReq.Messages, cfg, nil, false)
+		xlog.Debug("Ollama Chat - Prompt (after templating)", "prompt_len", len(predInput))
+
+		if input.IsStream() {
+			return handleOllamaChatStream(c, input, cfg, ml, cl, appConfig, predInput, openAIReq)
+		}
+
+		return handleOllamaChatNonStream(c, input, cfg, ml, cl, appConfig, predInput, openAIReq)
+	}
+}
+
+func handleOllamaChatNonStream(c echo.Context, input *schema.OllamaChatRequest, cfg *config.ModelConfig, ml *model.ModelLoader, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig, predInput string, openAIReq *schema.OpenAIRequest) error {
+	startTime := time.Now()
+	var result string
+
+	cb := func(s string, choices *[]schema.Choice) {
+		result = s
+	}
+
+	_, tokenUsage, _, err := openaiEndpoint.ComputeChoices(openAIReq, predInput, cfg, cl, appConfig, ml, cb, nil)
+	if err != nil {
+		xlog.Error("Ollama chat inference failed", "error", err)
+		return ollamaError(c, 500, fmt.Sprintf("model inference failed: %v", err))
+	}
+
+	totalDuration := time.Since(startTime)
+
+	resp := schema.OllamaChatResponse{
+		Model:     input.Model,
+		CreatedAt: time.Now().UTC(),
+		Message: schema.OllamaMessage{
+			Role:    "assistant",
+			Content: result,
+		},
+		Done:            true,
+		DoneReason:      "stop",
+		TotalDuration:   totalDuration.Nanoseconds(),
+		PromptEvalCount: tokenUsage.Prompt,
+		EvalCount:       tokenUsage.Completion,
+	}
+
+	return c.JSON(200, resp)
+}
+
+func handleOllamaChatStream(c echo.Context, input *schema.OllamaChatRequest, cfg *config.ModelConfig, ml *model.ModelLoader, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig, predInput string, openAIReq *schema.OpenAIRequest) error {
+	c.Response().Header().Set("Content-Type", "application/x-ndjson")
+	c.Response().Header().Set("Cache-Control", "no-cache")
+	c.Response().Header().Set("Connection", "keep-alive")
+
+	startTime := time.Now()
+
+	tokenCallback := func(token string, usage backend.TokenUsage) bool {
+		chunk := schema.OllamaChatResponse{
+			Model:     input.Model,
+			CreatedAt: time.Now().UTC(),
+			Message: schema.OllamaMessage{
+				Role:    "assistant",
+				Content: token,
+			},
+			Done: false,
+		}
+		return writeNDJSON(c, chunk)
+	}
+
+	_, tokenUsage, _, err := openaiEndpoint.ComputeChoices(openAIReq, predInput, cfg, cl, appConfig, ml, func(s string, choices *[]schema.Choice) {}, tokenCallback)
+	if err != nil {
+		xlog.Error("Ollama chat stream inference failed", "error", err)
+		errChunk := schema.OllamaChatResponse{
+			Model:     input.Model,
+			CreatedAt: time.Now().UTC(),
+			Done:      true,
+			DoneReason: "error",
+		}
+		writeNDJSON(c, errChunk)
+		return nil
+	}
+
+	// Send final done message
+	totalDuration := time.Since(startTime)
+	finalChunk := schema.OllamaChatResponse{
+		Model:           input.Model,
+		CreatedAt:       time.Now().UTC(),
+		Message:         schema.OllamaMessage{Role: "assistant", Content: ""},
+		Done:            true,
+		DoneReason:      "stop",
+		TotalDuration:   totalDuration.Nanoseconds(),
+		PromptEvalCount: tokenUsage.Prompt,
+		EvalCount:       tokenUsage.Completion,
+	}
+	writeNDJSON(c, finalChunk)
+
+	return nil
+}

core/http/endpoints/ollama/embed.go

@@ -0,0 +1,67 @@
+package ollama
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/backend"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/http/middleware"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/xlog"
+)
+
+// EmbedEndpoint handles Ollama-compatible /api/embed and /api/embeddings requests
+func EmbedEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OllamaEmbedRequest)
+		if !ok || input.Model == "" {
+			return ollamaError(c, 400, "model is required")
+		}
+
+		cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
+		if !ok || cfg == nil {
+			return ollamaError(c, 400, "model configuration not found")
+		}
+
+		startTime := time.Now()
+		inputStrings := input.GetInputStrings()
+		if len(inputStrings) == 0 {
+			return ollamaError(c, 400, "input is required")
+		}
+
+		var allEmbeddings [][]float32
+		promptEvalCount := 0
+
+		for _, s := range inputStrings {
+			embedFn, err := backend.ModelEmbedding(s, []int{}, ml, *cfg, appConfig)
+			if err != nil {
+				xlog.Error("Ollama embed failed", "error", err)
+				return ollamaError(c, 500, fmt.Sprintf("embedding failed: %v", err))
+			}
+
+			embeddings, err := embedFn()
+			if err != nil {
+				xlog.Error("Ollama embed computation failed", "error", err)
+				return ollamaError(c, 500, fmt.Sprintf("embedding computation failed: %v", err))
+			}
+
+			allEmbeddings = append(allEmbeddings, embeddings)
+			// Rough token count estimate
+			promptEvalCount += len(s) / 4
+		}
+
+		totalDuration := time.Since(startTime)
+
+		resp := schema.OllamaEmbedResponse{
+			Model:           input.Model,
+			Embeddings:      allEmbeddings,
+			TotalDuration:   totalDuration.Nanoseconds(),
+			PromptEvalCount: promptEvalCount,
+		}
+
+		return c.JSON(200, resp)
+	}
+}