feat(ui): Per model backend logs and various fixes (mudler#9028)

* feat(gallery): Switch to expandable box instead of pop-over and display model files


* feat(ui, backends): Add individual backend logging


* fix(ui): Set the context settings from the model config


---------




Signed-off-by: localai-bot <localai-bot@localai.io>

Author: richiejp

core/application/startup.go

@@ -136,6 +136,8 @@ func New(opts ...config.AppOption) (*Application, error) {
 		loadRuntimeSettingsFromFile(options)
 	}
 
+	application.ModelLoader().SetBackendLoggingEnabled(options.EnableBackendLogging)
+
 	// turn off any process that was started by GRPC if the context is canceled
 	go func() {
 		<-options.Context.Done()
@@ -382,6 +384,12 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
 		}
 	}
 
+	if settings.EnableBackendLogging != nil {
+		if !options.EnableBackendLogging {
+			options.EnableBackendLogging = *settings.EnableBackendLogging
+		}
+	}
+
 	xlog.Debug("Runtime settings loaded from runtime_settings.json")
 }
 

core/backend/detection.go

@@ -3,8 +3,10 @@ package backend
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/trace"
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/model"
 )
@@ -18,16 +20,43 @@ func Detection(
 	opts := ModelOptions(modelConfig, appConfig)
 	detectionModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 
 	if detectionModel == nil {
 		return nil, fmt.Errorf("could not load detection model")
 	}
 
+	var startTime time.Time
+	if appConfig.EnableTracing {
+		trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+		startTime = time.Now()
+	}
+
 	res, err := detectionModel.Detect(context.Background(), &proto.DetectOptions{
 		Src: sourceFile,
 	})
 
+	if appConfig.EnableTracing {
+		errStr := ""
+		if err != nil {
+			errStr = err.Error()
+		}
+
+		trace.RecordBackendTrace(trace.BackendTrace{
+			Timestamp: startTime,
+			Duration:  time.Since(startTime),
+			Type:      trace.BackendTraceDetection,
+			ModelName: modelConfig.Name,
+			Backend:   modelConfig.Backend,
+			Summary:   trace.TruncateString(sourceFile, 200),
+			Error:     errStr,
+			Data: map[string]any{
+				"source_file": sourceFile,
+			},
+		})
+	}
+
 	return res, err
 }

core/backend/embeddings.go

@@ -17,6 +17,7 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, modelConf
 
 	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/image.go

@@ -17,6 +17,7 @@ func ImageGeneration(height, width, step, seed int, positive_prompt, negative_pr
 		opts...,
 	)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/llm.go

@@ -65,6 +65,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 	opts := ModelOptions(*c, o)
 	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(o, c.Name, c.Backend, err, map[string]any{"model_file": modelFile})
 		return nil, err
 	}
 

core/backend/options.go

@@ -1,17 +1,36 @@
 package backend
 
 import (
-	"strings"
 	"math/rand"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/trace"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/model"
 	"github.com/mudler/xlog"
 )
 
+// recordModelLoadFailure records a backend trace when model loading fails.
+func recordModelLoadFailure(appConfig *config.ApplicationConfig, modelName, backend string, err error, data map[string]any) {
+	if !appConfig.EnableTracing {
+		return
+	}
+	trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+	trace.RecordBackendTrace(trace.BackendTrace{
+		Timestamp: time.Now(),
+		Type:      trace.BackendTraceModelLoad,
+		ModelName: modelName,
+		Backend:   backend,
+		Summary:   "Model load failed",
+		Error:     err.Error(),
+		Data:      data,
+	})
+}
+
 func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
 	name := c.Name
 	if name == "" {

core/backend/rerank.go

@@ -15,6 +15,7 @@ func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *
 	opts := ModelOptions(modelConfig, appConfig)
 	rerankModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/soundgeneration.go

@@ -37,6 +37,7 @@ func SoundGeneration(
 	opts := ModelOptions(modelConfig, appConfig)
 	soundGenModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return "", nil, err
 	}
 

core/backend/token_metrics.go

@@ -18,6 +18,7 @@ func TokenMetrics(
 	opts := ModelOptions(modelConfig, appConfig, model.WithModel(modelFile))
 	model, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/tokenize.go

@@ -18,6 +18,7 @@ func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.Model
 	opts := ModelOptions(modelConfig, appConfig)
 	inferenceModel, err = loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return schema.TokenizeResponse{}, err
 	}