fix(download): do not remove dst dir until we try all fallbacks (mudler#9100)

This actually caused fallbacks to be compeletely no-op as we were
removing the destination dir before calling containerd.Apply

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

core/gallery/backends.go

@@ -198,11 +198,8 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
 	} else {
 		xlog.Debug("Downloading backend", "uri", config.URI, "backendPath", backendPath)
 		if err := uri.DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err != nil {
-			// Clean up the partially downloaded backend directory on failure
-			xlog.Debug("Backend download failed, cleaning up", "backendPath", backendPath, "error", err)
-			if cleanupErr := os.RemoveAll(backendPath); cleanupErr != nil {
-				xlog.Warn("Failed to clean up backend directory", "backendPath", backendPath, "error", cleanupErr)
-			}
+			// Don't remove backendPath here — fallback OCI extractions need the directory to exist
+			xlog.Debug("Backend download failed, trying fallback", "backendPath", backendPath, "error", err)
 
 			success := false
 			// Try to download from mirrors
@@ -215,34 +212,36 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
 				}
 				if err := downloader.URI(mirror).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
 					success = true
-					xlog.Debug("Downloaded backend", "uri", config.URI, "backendPath", backendPath)
+					xlog.Debug("Downloaded backend from mirror", "uri", config.URI, "backendPath", backendPath)
 					break
 				}
 			}
 
-			// Try fallback: replace latestTag + "-" with masterTag + "-" in the URI
-			fallbackURI := strings.Replace(string(config.URI), latestTag+"-", masterTag+"-", 1)
-			if fallbackURI != string(config.URI) {
-				xlog.Debug("Trying fallback URI", "original", config.URI, "fallback", fallbackURI)
-				if err := downloader.URI(fallbackURI).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
-					xlog.Debug("Downloaded backend using fallback URI", "uri", fallbackURI, "backendPath", backendPath)
-					success = true
-				} else {
-					// Try another fallback: add "-" + devSuffix suffix to the backend name
-					// For example: master-gpu-nvidia-cuda-13-ace-step -> master-gpu-nvidia-cuda-13-ace-step-development
-					if !strings.Contains(fallbackURI, "-"+devSuffix) {
-						// Extract backend name from URI and add -development
-						parts := strings.Split(fallbackURI, "-")
-						if len(parts) >= 2 {
-							// Find where the backend name ends (usually the last part before the tag)
-							// Pattern: quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-ace-step
-							lastDash := strings.LastIndex(fallbackURI, "-")
-							if lastDash > 0 {
-								devFallbackURI := fallbackURI[:lastDash] + "-" + devSuffix
-								xlog.Debug("Trying development fallback URI", "fallback", devFallbackURI)
-								if err := downloader.URI(devFallbackURI).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
-									xlog.Debug("Downloaded backend using development fallback URI", "uri", devFallbackURI, "backendPath", backendPath)
-									success = true
+			if !success {
+				// Try fallback: replace latestTag + "-" with masterTag + "-" in the URI
+				fallbackURI := strings.Replace(string(config.URI), latestTag+"-", masterTag+"-", 1)
+				if fallbackURI != string(config.URI) {
+					xlog.Debug("Trying fallback URI", "original", config.URI, "fallback", fallbackURI)
+					if err := downloader.URI(fallbackURI).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
+						xlog.Info("Downloaded backend using fallback URI", "uri", fallbackURI, "backendPath", backendPath)
+						success = true
+					} else {
+						// Try another fallback: add "-" + devSuffix suffix to the backend name
+						// For example: master-gpu-nvidia-cuda-13-ace-step -> master-gpu-nvidia-cuda-13-ace-step-development
+						if !strings.Contains(fallbackURI, "-"+devSuffix) {
+							// Extract backend name from URI and add -development
+							parts := strings.Split(fallbackURI, "-")
+							if len(parts) >= 2 {
+								// Find where the backend name ends (usually the last part before the tag)
+								// Pattern: quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-ace-step
+								lastDash := strings.LastIndex(fallbackURI, "-")
+								if lastDash > 0 {
+									devFallbackURI := fallbackURI[:lastDash] + "-" + devSuffix
+									xlog.Debug("Trying development fallback URI", "fallback", devFallbackURI)
+									if err := downloader.URI(devFallbackURI).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
+										xlog.Info("Downloaded backend using development fallback URI", "uri", devFallbackURI, "backendPath", backendPath)
+										success = true
+									}
 								}
 							}
 						}
@@ -251,6 +250,10 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
 			}
 
 			if !success {
+				// Clean up backend directory only when all download attempts have failed
+				if cleanupErr := os.RemoveAll(backendPath); cleanupErr != nil {
+					xlog.Warn("Failed to clean up backend directory", "backendPath", backendPath, "error", cleanupErr)
+				}
 				xlog.Error("Failed to download backend", "uri", config.URI, "backendPath", backendPath, "error", err)
 				return fmt.Errorf("failed to download backend %q: %v", config.URI, err)
 			}