fix(modelartifacts): treat CIFS EACCES as lock contention, not failure

core/config/model_artifact_fallback_test.go

@@ -0,0 +1,62 @@
+package config
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/xlog"
+)
+
+// For a backend in managedArtifactBackends the legacy path is not a graceful
+// degradation: the backend consumes a snapshot directory, so falling back means
+// it downloads the whole repo itself, in-band, inside LoadModel on every load.
+// That reliably blows the remote-load deadline, and the operator sees only a
+// timeout unless the cause was logged loudly enough to survive a default log
+// level. #10981 traced a 57 GB in-band pull back to a warn logged 40 minutes
+// earlier.
+var _ = Describe("preload artifact fallback severity", func() {
+	const inferredConfig = `
+name: managed
+backend: transformers
+parameters: {model: owner/repo}
+`
+
+	var captured *bytes.Buffer
+
+	BeforeEach(func() {
+		// Capture at error level so a warn emission is filtered out and the
+		// assertion fails on severity, not on wording.
+		captured = &bytes.Buffer{}
+		handler := slog.NewTextHandler(captured, &slog.HandlerOptions{Level: slog.LevelError})
+		xlog.SetLogger(xlog.NewLoggerWithHandler(handler, xlog.LogLevelError))
+	})
+
+	AfterEach(func() {
+		// xlog exposes no getter for the package logger, so restore the same
+		// default the suite entrypoint installs rather than the prior value.
+		xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
+	})
+
+	It("logs at error, and names the in-band download, when a managed backend degrades", func() {
+		modelsPath := GinkgoT().TempDir()
+		Expect(os.WriteFile(filepath.Join(modelsPath, "managed.yaml"), []byte(inferredConfig), 0644)).To(Succeed())
+
+		fake := &companionMaterializer{fail: "owner/repo"}
+		loader := NewModelConfigLoader(modelsPath, WithArtifactMaterializer(fake))
+		Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
+
+		// Behavior is unchanged: the degradation is still non-fatal.
+		Expect(loader.PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
+
+		logged := captured.String()
+		Expect(logged).To(ContainSubstring("managed"))
+		Expect(logged).To(ContainSubstring("transformers"))
+		Expect(logged).To(ContainSubstring("in-band"))
+	})
+})

core/config/model_artifact_inference.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/mudler/LocalAI/pkg/modelartifacts"
+	"github.com/mudler/xlog"
 )
 
 // managedArtifactBackends is the set of backends that load a model from a
@@ -33,6 +34,26 @@ func IsManagedArtifactBackend(backend string) bool {
 	return ok
 }
 
+// LogArtifactFallback records a degradation to the legacy loading path. For a
+// backend that consumes a snapshot *directory*, "legacy loading" is not a
+// graceful degradation: the backend downloads the whole repo itself, in-band,
+// inside LoadModel, on every load. That reliably exceeds the remote-load
+// deadline and surfaces as a bare timeout, so it is logged at error to survive
+// a default log level and give the timeout a named cause (#10981).
+//
+// Today every inferred artifact already comes from such a backend, but the gate
+// is checked explicitly so widening PrimaryArtifactSpec later cannot silently
+// promote an ordinary fallback to an error.
+func LogArtifactFallback(model, backend string, err error) {
+	if IsManagedArtifactBackend(backend) {
+		xlog.Error("artifact materialization failed; the backend will download the model in-band on every load",
+			"model", model, "backend", backend, "error", err)
+		return
+	}
+	xlog.Warn("falling back to legacy model loading after artifact materialization failed",
+		"model", model, "backend", backend, "error", err)
+}
+
 // PrimaryArtifactSpec returns the managed primary artifact to materialize for
 // this config. The boolean return is false when the config should stay on the
 // legacy path.

core/config/model_config_loader.go

@@ -490,7 +490,7 @@ func (bcl *ModelConfigLoader) preloadOne(
 		result, err := bcl.artifactMaterializer.Ensure(ctx, modelPath, artifactSpec)
 		if err != nil {
 			if inferred {
-				xlog.Warn("falling back to legacy model loading after artifact materialization failed", "model", updated.Name, "error", err)
+				LogArtifactFallback(updated.Name, updated.Backend, err)
 				managedPrimary = false
 			} else {
 				return ModelConfig{}, nil, err

core/gallery/model_artifacts.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/modelartifacts"
-	"github.com/mudler/xlog"
 )
 
 type ArtifactMaterializer interface {
@@ -54,7 +53,7 @@ func bindPrimaryArtifact(ctx context.Context, modelsPath string, typed *config.M
 	result, err := materializer.Ensure(ctx, modelsPath, artifactSpec)
 	if err != nil {
 		if inferred {
-			xlog.Warn("falling back to legacy model loading after artifact materialization failed", "model", typed.Name, "error", err)
+			config.LogArtifactFallback(typed.Name, typed.Backend, err)
 			return false, nil
 		}
 		return false, fmt.Errorf("materialize primary model artifact: %w", err)

core/gallery/model_artifacts_fallback_test.go

@@ -0,0 +1,57 @@
+package gallery_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"log/slog"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
+	"github.com/mudler/xlog"
+)
+
+// Install-time degradation has the same consequence as preload-time
+// degradation: a managed backend that never gets its snapshot downloads the
+// repo itself on every load. See #10981.
+var _ = Describe("install artifact fallback severity", func() {
+	var captured *bytes.Buffer
+
+	BeforeEach(func() {
+		// Capture at error level so a warn emission is filtered out and the
+		// assertion fails on severity, not on wording.
+		captured = &bytes.Buffer{}
+		handler := slog.NewTextHandler(captured, &slog.HandlerOptions{Level: slog.LevelError})
+		xlog.SetLogger(xlog.NewLoggerWithHandler(handler, xlog.LogLevelError))
+	})
+
+	AfterEach(func() {
+		// xlog exposes no getter for the package logger, so restore the same
+		// default the suite entrypoint installs rather than the prior value.
+		xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
+	})
+
+	It("logs at error, and names the in-band download, when a managed backend degrades", func() {
+		state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
+		Expect(err).NotTo(HaveOccurred())
+		fake := &fakeArtifactMaterializer{err: errors.New("materialization refused")}
+		definition := &gallery.ModelConfig{Name: "legacy", ConfigFile: `
+backend: transformers
+parameters:
+  model: owner/legacy
+`}
+
+		// Behavior is unchanged: the install still succeeds on the legacy path.
+		_, err = gallery.InstallModel(context.Background(), state, "legacy", definition, nil, nil, false,
+			gallery.WithArtifactMaterializer(fake))
+		Expect(err).NotTo(HaveOccurred())
+
+		logged := captured.String()
+		Expect(logged).To(ContainSubstring("legacy"))
+		Expect(logged).To(ContainSubstring("transformers"))
+		Expect(logged).To(ContainSubstring("in-band"))
+	})
+})

pkg/modelartifacts/materializer.go

@@ -6,12 +6,14 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/gofrs/flock"
@@ -25,9 +27,39 @@ type SnapshotResolver interface {
 	ResolveSnapshot(context.Context, hfapi.SnapshotRequest) (hfapi.Snapshot, error)
 }
 
+// Locker is the cross-process exclusion primitive that keeps two replicas from
+// materializing the same snapshot at once. It is an interface rather than a
+// concrete *flock.Flock because the contention path has to be exercised without
+// a network filesystem: no test environment can make flock(2) return the
+// CIFS-specific errno this code must tolerate, and it is also the seam a
+// database-backed lock would plug into for multi-node deployments.
+type Locker interface {
+	TryLock() (bool, error)
+	Unlock() error
+}
+
+// ErrLockContended reports that a peer held the artifact lock for the whole
+// wait window and never published a usable snapshot. It is deliberately
+// distinct from an acquisition failure: the work is in progress elsewhere, so
+// callers can say so rather than blaming the model.
+var ErrLockContended = errors.New("artifact lock is held by another process")
+
+const (
+	// DefaultLockWait bounds how long Ensure waits for a peer replica. It is
+	// generous because the peer may legitimately be downloading tens of
+	// gigabytes, and waiting is strictly cheaper than the fallback, which makes
+	// the backend download the same repo in-band.
+	DefaultLockWait = 30 * time.Minute
+
+	initialLockRetryInterval = 100 * time.Millisecond
+	maxLockRetryInterval     = 5 * time.Second
+)
+
 type Manager struct {
 	resolver         SnapshotResolver
 	huggingFaceToken string
+	newLocker        func(string) Locker
+	lockWait         time.Duration
 }
 
 type ManagerOption func(*Manager)
@@ -43,8 +75,32 @@ func WithHuggingFaceToken(token string) ManagerOption {
 	return func(manager *Manager) { manager.huggingFaceToken = token }
 }
 
+// WithLocker overrides how the artifact lock is created. The default is an
+// flock(2) lock on the shared models directory.
+func WithLocker(factory func(path string) Locker) ManagerOption {
+	return func(manager *Manager) {
+		if factory != nil {
+			manager.newLocker = factory
+		}
+	}
+}
+
+// WithLockWait bounds how long Ensure waits for a peer replica holding the
+// artifact lock before giving up with ErrLockContended.
+func WithLockWait(wait time.Duration) ManagerOption {
+	return func(manager *Manager) {
+		if wait > 0 {
+			manager.lockWait = wait
+		}
+	}
+}
+
 func NewManager(resolver SnapshotResolver, options ...ManagerOption) *Manager {
-	manager := &Manager{resolver: resolver}
+	manager := &Manager{
+		resolver:  resolver,
+		newLocker: func(path string) Locker { return flock.New(path) },
+		lockWait:  DefaultLockWait,
+	}
 	for _, option := range options {
 		option(manager)
 	}
@@ -147,25 +203,24 @@ func (m *Manager) Ensure(ctx context.Context, modelsPath string, spec Spec) (Res
 	if err := os.MkdirAll(filepath.Dir(layout.Lock), 0o750); err != nil {
 		return Result{}, err
 	}
-	artifactLock := flock.New(layout.Lock)
-	locked, err := artifactLock.TryLockContext(ctx, 100*time.Millisecond)
-	if err != nil {
-		return Result{}, err
-	}
-	if !locked {
-		if err := ctx.Err(); err != nil {
-			return Result{}, err
+	artifactLock := m.newLocker(layout.Lock)
+	if err := m.acquireLock(ctx, artifactLock, layout.Lock); err != nil {
+		// A peer that held the lock for the whole window was very likely doing
+		// exactly this work. Its committed snapshot is the answer we wanted, so
+		// prefer it over reporting contention to a caller that would degrade to
+		// an in-band download.
+		if errors.Is(err, ErrLockContended) {
+			if cached, ok := committedResult(modelsPath, normalized); ok {
+				return cached, nil
+			}
 		}
-		return Result{}, fmt.Errorf("artifact lock was not acquired")
+		return Result{}, err
 	}
 	defer func() {
 		if err := artifactLock.Unlock(); err != nil {
 			xlog.Warn("failed to unlock model artifact", "lock", layout.Lock, "error", err)
 		}
 	}()
-	if err := os.Chmod(layout.Lock, 0o600); err != nil {
-		return Result{}, err
-	}
 	if cached, ok := committedResult(modelsPath, normalized); ok {
 		return cached, nil
 	}
@@ -175,6 +230,72 @@ func (m *Manager) Ensure(ctx context.Context, modelsPath string, spec Spec) (Res
 	return m.materializeLocked(ctx, modelsPath, normalized, snapshot, token, layout)
 }
 
+// isLockContention reports whether a failed lock attempt means "somebody else
+// holds it" rather than "this will never work".
+//
+// Only EWOULDBLOCK is portable, and it is the only errno gofrs/flock treats as
+// contention. Network filesystems translate their own protocol status codes
+// instead: CIFS/SMB maps STATUS_LOCK_NOT_GRANTED and STATUS_FILE_LOCK_CONFLICT
+// to EACCES, and a busy share can surface EBUSY. Both look like hard errors and
+// aborted materialization outright on a shared /models (#10981).
+//
+// EACCES is ambiguous at the syscall boundary, where it also spells "permission
+// denied", but it is not ambiguous at this call site. The lock file was already opened
+// O_CREATE|O_RDWR before we get here, so a genuine permission problem would
+// have failed the open with an *fs.PathError naming the path. flock(2) itself
+// documents no EACCES on Linux (EBADF, EINTR, EINVAL, ENOLCK, EWOULDBLOCK), so
+// a bare EACCES from the lock call can only have come from a network
+// filesystem's lock-conflict translation. The wait is bounded regardless, so
+// even a misclassification degrades to a delay, not a hang.
+func isLockContention(err error) bool {
+	return errors.Is(err, syscall.EWOULDBLOCK) ||
+		errors.Is(err, syscall.EAGAIN) ||
+		errors.Is(err, syscall.EACCES) ||
+		errors.Is(err, syscall.EBUSY)
+}
+
+// acquireLock blocks until the artifact lock is held, the context is done, or
+// the wait window expires with ErrLockContended.
+func (m *Manager) acquireLock(ctx context.Context, locker Locker, lockPath string) error {
+	wait := m.lockWait
+	if wait <= 0 {
+		wait = DefaultLockWait
+	}
+	deadline := time.Now().Add(wait)
+	interval := initialLockRetryInterval
+	waited := false
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		locked, err := locker.TryLock()
+		if err != nil && !isLockContention(err) {
+			return err
+		}
+		if locked {
+			if waited {
+				xlog.Info("acquired the model artifact lock after waiting for another replica", "lock", lockPath)
+			}
+			return nil
+		}
+		if !waited {
+			waited = true
+			xlog.Info("another replica is materializing this model artifact; waiting for it", "lock", lockPath)
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("%w: %s", ErrLockContended, lockPath)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(interval):
+		}
+		if interval < maxLockRetryInterval {
+			interval = min(interval*2, maxLockRetryInterval)
+		}
+	}
+}
+
 func removeInvalidFinal(layout Layout) error {
 	root, err := os.OpenRoot(layout.Root)
 	if err != nil {