From e31b280271020b37581414a95bcd0837de4586c6 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:11:49 +0000 Subject: [PATCH] feat(downloads): add resume-safe pause action Give gallery operations distinct pause and cancel paths. Pause preserves partial download data so reinstalling the same model or backend resumes through HTTP Range, while cancel keeps its destructive semantics. Surface the action in the Activity UI and document the API behavior. Assisted-by: Codex:gpt-5 --- core/http/react-ui/e2e/activity-page.spec.js | 34 +++++++ .../react-ui/public/locales/de/admin.json | 2 + .../react-ui/public/locales/en/admin.json | 2 + .../react-ui/public/locales/es/admin.json | 2 + .../react-ui/public/locales/id/admin.json | 2 + .../react-ui/public/locales/it/admin.json | 2 + .../react-ui/public/locales/ko/admin.json | 2 + .../react-ui/public/locales/zh-CN/admin.json | 2 + .../react-ui/src/components/OperationCard.jsx | 12 ++- .../src/contexts/OperationsContext.jsx | 11 +++ core/http/react-ui/src/pages/Activity.jsx | 4 +- core/http/react-ui/src/utils/api.js | 1 + core/http/react-ui/src/utils/config.js | 1 + core/http/routes/ui_api.go | 48 +++++++--- core/http/routes/ui_api_operations_test.go | 27 ++++++ .../services/galleryop/cancel_persist_test.go | 16 ++++ core/services/galleryop/operation.go | 2 + core/services/galleryop/service.go | 90 +++++++++++++------ docs/content/features/authentication.md | 2 +- docs/content/operations/activity.md | 5 ++ 20 files changed, 224 insertions(+), 43 deletions(-) diff --git a/core/http/react-ui/e2e/activity-page.spec.js b/core/http/react-ui/e2e/activity-page.spec.js index d0c5eac5c9e9..09f9b46fe022 100644 --- a/core/http/react-ui/e2e/activity-page.spec.js +++ b/core/http/react-ui/e2e/activity-page.spec.js @@ -43,6 +43,40 @@ test('lists live operations and cancels one from a labelled button', async ({ pa expect(cancelledPath).toBe('/api/operations/job-gemma/cancel') }) +test('pauses a model download without invoking destructive cancel', async ({ page }) => { + await stub(page, { + operations: [{ + id: 'gemma-3-27b-it', + name: 'gemma-3-27b-it', + jobID: 'job-gemma', + progress: 22, + taskType: 'installation', + isBackend: false, + isQueued: false, + isDeletion: false, + cancellable: true, + phase: 'downloading', + }], + }) + + const requests = [] + await page.route('**/api/operations/job-gemma/pause', (route) => { + requests.push(new URL(route.request().url()).pathname) + return route.fulfill({ contentType: 'application/json', body: '{}' }) + }) + await page.route('**/api/operations/job-gemma/cancel', (route) => { + requests.push(new URL(route.request().url()).pathname) + return route.fulfill({ contentType: 'application/json', body: '{}' }) + }) + + await page.goto('/app/activity') + + const card = page.locator('.operation-card').filter({ hasText: 'gemma-3-27b-it' }) + await card.locator('.operation-card__pause').click() + + await expect.poll(() => requests).toEqual(['/api/operations/job-gemma/pause']) +}) + test('separates an unacknowledged failure from the record', async ({ page }) => { await stub(page, { operations: [{ diff --git a/core/http/react-ui/public/locales/de/admin.json b/core/http/react-ui/public/locales/de/admin.json index 58becfd02101..cf70e417e43e 100644 --- a/core/http/react-ui/public/locales/de/admin.json +++ b/core/http/react-ui/public/locales/de/admin.json @@ -12,6 +12,8 @@ "timeLeft": "{{value}} left", "cancel": "Cancel", "cancelLabel": "Cancel {{name}}", + "pause": "Pause", + "pauseLabel": "Pause {{name}} and keep downloaded data", "retry": "Retry", "retryLabel": "Retry {{name}}", "nodeCount": "{{count}} nodes", diff --git a/core/http/react-ui/public/locales/en/admin.json b/core/http/react-ui/public/locales/en/admin.json index f6dccf80b977..3978fa8be82d 100644 --- a/core/http/react-ui/public/locales/en/admin.json +++ b/core/http/react-ui/public/locales/en/admin.json @@ -12,6 +12,8 @@ "timeLeft": "{{value}} left", "cancel": "Cancel", "cancelLabel": "Cancel {{name}}", + "pause": "Pause", + "pauseLabel": "Pause {{name}} and keep downloaded data", "retry": "Retry", "retryLabel": "Retry {{name}}", "nodeCount": "{{count}} nodes", diff --git a/core/http/react-ui/public/locales/es/admin.json b/core/http/react-ui/public/locales/es/admin.json index b1183b24e185..1d486194d2cf 100644 --- a/core/http/react-ui/public/locales/es/admin.json +++ b/core/http/react-ui/public/locales/es/admin.json @@ -12,6 +12,8 @@ "timeLeft": "{{value}} left", "cancel": "Cancel", "cancelLabel": "Cancel {{name}}", + "pause": "Pause", + "pauseLabel": "Pause {{name}} and keep downloaded data", "retry": "Retry", "retryLabel": "Retry {{name}}", "nodeCount": "{{count}} nodes", diff --git a/core/http/react-ui/public/locales/id/admin.json b/core/http/react-ui/public/locales/id/admin.json index 7a11b0677b73..970f463b7fb5 100644 --- a/core/http/react-ui/public/locales/id/admin.json +++ b/core/http/react-ui/public/locales/id/admin.json @@ -12,6 +12,8 @@ "timeLeft": "{{value}} left", "cancel": "Cancel", "cancelLabel": "Cancel {{name}}", + "pause": "Pause", + "pauseLabel": "Pause {{name}} and keep downloaded data", "retry": "Retry", "retryLabel": "Retry {{name}}", "nodeCount": "{{count}} nodes", diff --git a/core/http/react-ui/public/locales/it/admin.json b/core/http/react-ui/public/locales/it/admin.json index 79b185638eb7..7b29b885f72b 100644 --- a/core/http/react-ui/public/locales/it/admin.json +++ b/core/http/react-ui/public/locales/it/admin.json @@ -12,6 +12,8 @@ "timeLeft": "{{value}} left", "cancel": "Cancel", "cancelLabel": "Cancel {{name}}", + "pause": "Pause", + "pauseLabel": "Pause {{name}} and keep downloaded data", "retry": "Retry", "retryLabel": "Retry {{name}}", "nodeCount": "{{count}} nodes", diff --git a/core/http/react-ui/public/locales/ko/admin.json b/core/http/react-ui/public/locales/ko/admin.json index b727ba4c4786..46aa818eb968 100644 --- a/core/http/react-ui/public/locales/ko/admin.json +++ b/core/http/react-ui/public/locales/ko/admin.json @@ -12,6 +12,8 @@ "timeLeft": "{{value}} left", "cancel": "Cancel", "cancelLabel": "Cancel {{name}}", + "pause": "Pause", + "pauseLabel": "Pause {{name}} and keep downloaded data", "retry": "Retry", "retryLabel": "Retry {{name}}", "nodeCount": "{{count}} nodes", diff --git a/core/http/react-ui/public/locales/zh-CN/admin.json b/core/http/react-ui/public/locales/zh-CN/admin.json index 43b6b10f77d0..c06634e57d00 100644 --- a/core/http/react-ui/public/locales/zh-CN/admin.json +++ b/core/http/react-ui/public/locales/zh-CN/admin.json @@ -12,6 +12,8 @@ "timeLeft": "{{value}} left", "cancel": "Cancel", "cancelLabel": "Cancel {{name}}", + "pause": "Pause", + "pauseLabel": "Pause {{name}} and keep downloaded data", "retry": "Retry", "retryLabel": "Retry {{name}}", "nodeCount": "{{count}} nodes", diff --git a/core/http/react-ui/src/components/OperationCard.jsx b/core/http/react-ui/src/components/OperationCard.jsx index fa67ce54e854..ba3c9a367460 100644 --- a/core/http/react-ui/src/components/OperationCard.jsx +++ b/core/http/react-ui/src/components/OperationCard.jsx @@ -29,7 +29,7 @@ function formatEta(seconds) { return `${Math.floor(minutes / 60)}h ${minutes % 60}m` } -export default function OperationCard({ operation, onCancel, onDismiss, onRetry }) { +export default function OperationCard({ operation, onCancel, onPause, onDismiss, onRetry }) { const { t } = useTranslation('admin') const nodes = Array.isArray(operation.nodes) ? operation.nodes : [] // Holds only what the user chose. The default has to stay a live @@ -144,6 +144,16 @@ export default function OperationCard({ operation, onCancel, onDismiss, onRetry
{showProgress && } + {canCancel && ( + + )} {canCancel && ( // A page of cards would otherwise hand a screen reader a list of // identical "Cancel" buttons with nothing to tell them apart. diff --git a/core/http/react-ui/src/contexts/OperationsContext.jsx b/core/http/react-ui/src/contexts/OperationsContext.jsx index 6fc35c2423d0..3c04e3482862 100644 --- a/core/http/react-ui/src/contexts/OperationsContext.jsx +++ b/core/http/react-ui/src/contexts/OperationsContext.jsx @@ -137,6 +137,16 @@ export function OperationsProvider({ children, pollInterval = 1000 }) { } }, [fetchOperations]) + const pauseOperation = useCallback(async (jobID) => { + try { + await operationsApi.pause(jobID) + cancelledRef.current.set(jobID, Date.now()) + await fetchOperations() + } catch (err) { + setError(err.message) + } + }, [fetchOperations]) + // Whether this tab cancelled the job. Read by the strip to tell "the last // operation finished" from "the user called it off": both look identical in // /api/operations, which lists neither. @@ -226,6 +236,7 @@ export function OperationsProvider({ children, pollInterval = 1000 }) { fetchHistory, clearHistory, cancelOperation, + pauseOperation, wasCancelled, dismissFailedOp, refetch: fetchOperations, diff --git a/core/http/react-ui/src/pages/Activity.jsx b/core/http/react-ui/src/pages/Activity.jsx index e4db1b90e796..1df124bbaee0 100644 --- a/core/http/react-ui/src/pages/Activity.jsx +++ b/core/http/react-ui/src/pages/Activity.jsx @@ -83,7 +83,7 @@ export default function Activity() { const { t } = useTranslation('admin') const outlet = useOutletContext() const addToast = outlet?.addToast - const { operations, history, fetchHistory, clearHistory, cancelOperation, dismissFailedOp } = useOperations() + const { operations, history, fetchHistory, clearHistory, cancelOperation, pauseOperation, dismissFailedOp } = useOperations() const [filter, setFilter] = useState('all') useEffect(() => { fetchHistory() }, [fetchHistory]) @@ -195,7 +195,7 @@ export default function Activity() { {t('activity.inProgress')} {live.length} {live.map((op) => ( - + ))} )} diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js index 0c9eb94c331f..6f4236dd9c15 100644 --- a/core/http/react-ui/src/utils/api.js +++ b/core/http/react-ui/src/utils/api.js @@ -173,6 +173,7 @@ export const resourcesApi = { export const operationsApi = { list: () => fetchJSON(API_CONFIG.endpoints.operations), cancel: (jobID) => postJSON(API_CONFIG.endpoints.cancelOperation(jobID), {}), + pause: (jobID) => postJSON(API_CONFIG.endpoints.pauseOperation(jobID), {}), dismiss: (jobID) => postJSON(API_CONFIG.endpoints.dismissOperation(jobID), {}), history: () => fetchJSON(API_CONFIG.endpoints.operationsHistory), clearHistory: () => fetchJSON(API_CONFIG.endpoints.operationsHistory, { method: 'DELETE' }), diff --git a/core/http/react-ui/src/utils/config.js b/core/http/react-ui/src/utils/config.js index 6ba619a1acda..3cc699b1bfa0 100644 --- a/core/http/react-ui/src/utils/config.js +++ b/core/http/react-ui/src/utils/config.js @@ -4,6 +4,7 @@ export const API_CONFIG = { operations: '/api/operations', operationsHistory: '/api/operations/history', cancelOperation: (jobID) => `/api/operations/${jobID}/cancel`, + pauseOperation: (jobID) => `/api/operations/${jobID}/pause`, dismissOperation: (jobID) => `/api/operations/${jobID}/dismiss`, // Models gallery diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index 300cc0a3b2f6..e378dc9549fe 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -390,6 +390,24 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model }) }, adminMiddleware) + // Pause operation endpoint (admin only). Unlike cancel, pause preserves a + // partial download so submitting the same install later resumes it. + app.POST("/api/operations/:jobID/pause", func(c echo.Context) error { + jobID := c.Param("jobID") + xlog.Debug("API request to pause operation", "jobID", jobID) + + if err := galleryService.PauseOperation(jobID); err != nil { + xlog.Error("Failed to pause operation", "error", err, "jobID", jobID) + return c.JSON(http.StatusBadRequest, map[string]any{"error": err.Error()}) + } + + opcache.DeleteUUID(jobID) + return c.JSON(200, map[string]any{ + "success": true, + "message": "Operation paused", + }) + }, adminMiddleware) + // Dismiss a failed operation (acknowledge the error and remove it from the list) app.POST("/api/operations/:jobID/dismiss", func(c echo.Context) error { jobID := c.Param("jobID") @@ -1004,7 +1022,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model uid := id.String() opcache.Set(galleryID, uid) - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background()) op := galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{ ID: uid, GalleryElementName: galleryID, @@ -1013,9 +1031,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model BackendGalleries: appConfig.BackendGalleries, Context: ctx, CancelFunc: cancelFunc, + PauseFunc: pauseFunc, } // Store cancellation function immediately so queued operations can be cancelled - galleryService.StoreCancellation(uid, cancelFunc) + galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc) galleryService.EnqueueModelOp(op) return c.JSON(200, map[string]any{ @@ -1051,7 +1070,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model opcache.Set(galleryID, uid) - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background()) op := galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{ ID: uid, Delete: true, @@ -1060,9 +1079,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model BackendGalleries: appConfig.BackendGalleries, Context: ctx, CancelFunc: cancelFunc, + PauseFunc: pauseFunc, } // Store cancellation function immediately so queued operations can be cancelled - galleryService.StoreCancellation(uid, cancelFunc) + galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc) galleryService.EnqueueModelOp(op) cl.RemoveModelConfig(galleryName) @@ -1457,19 +1477,20 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model uid := id.String() opcache.SetBackend(backendID, uid) - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background()) op := galleryop.ManagementOp[gallery.GalleryBackend, any]{ ID: uid, GalleryElementName: backendID, Galleries: appConfig.BackendGalleries, Context: ctx, CancelFunc: cancelFunc, + PauseFunc: pauseFunc, // The React UI's "Reinstall backend" action reuses this route, so // the op must force even when the backend is already installed. Force: true, } // Store cancellation function immediately so queued operations can be cancelled - galleryService.StoreCancellation(uid, cancelFunc) + galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc) galleryService.EnqueueBackendOp(op) return c.JSON(200, map[string]any{ @@ -1519,19 +1540,20 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model } opcache.SetBackend(cacheKey, uid) - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background()) op := galleryop.ManagementOp[gallery.GalleryBackend, any]{ ID: uid, GalleryElementName: req.Name, // May be empty, will be derived during installation Galleries: appConfig.BackendGalleries, Context: ctx, CancelFunc: cancelFunc, + PauseFunc: pauseFunc, ExternalURI: req.URI, ExternalName: req.Name, ExternalAlias: req.Alias, } // Store cancellation function immediately so queued operations can be cancelled - galleryService.StoreCancellation(uid, cancelFunc) + galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc) galleryService.EnqueueBackendOp(op) return c.JSON(200, map[string]any{ @@ -1567,7 +1589,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model opcache.SetBackend(backendID, uid) - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background()) op := galleryop.ManagementOp[gallery.GalleryBackend, any]{ ID: uid, Delete: true, @@ -1575,9 +1597,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model Galleries: appConfig.BackendGalleries, Context: ctx, CancelFunc: cancelFunc, + PauseFunc: pauseFunc, } // Store cancellation function immediately so queued operations can be cancelled - galleryService.StoreCancellation(uid, cancelFunc) + galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc) galleryService.EnqueueBackendOp(op) return c.JSON(200, map[string]any{ @@ -1686,7 +1709,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model // and the Backends UI can reflect progress on the affected row. opcache.SetBackend(backendName, uid) - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background()) op := galleryop.ManagementOp[gallery.GalleryBackend, any]{ ID: uid, GalleryElementName: backendName, @@ -1694,9 +1717,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model Upgrade: true, Context: ctx, CancelFunc: cancelFunc, + PauseFunc: pauseFunc, } // Store cancellation function immediately so queued operations can be cancelled - galleryService.StoreCancellation(uid, cancelFunc) + galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc) galleryService.EnqueueBackendOp(op) return c.JSON(200, map[string]any{ diff --git a/core/http/routes/ui_api_operations_test.go b/core/http/routes/ui_api_operations_test.go index 7656aa4e284e..c8cca19be3fa 100644 --- a/core/http/routes/ui_api_operations_test.go +++ b/core/http/routes/ui_api_operations_test.go @@ -339,6 +339,33 @@ var _ = Describe("/api/operations with node-scoped backend ops", func() { Expect(envelope.Operations[0]).ToNot(HaveKey("isCancelled")) }) + It("pauses through the resume-safe operation callback", func() { + state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir())) + Expect(err).NotTo(HaveOccurred()) + appCfg := &config.ApplicationConfig{SystemState: state} + galleryService := galleryop.NewGalleryService(appCfg, nil) + opcache := galleryop.NewOpCache(galleryService) + opcache.Set("localai@gemma", "job-pause") + + var cancelled, paused bool + galleryService.StoreCancellationActions( + "job-pause", + func() { cancelled = true }, + func() { paused = true }, + ) + + e := echo.New() + routes.RegisterUIAPIRoutes(e, nil, nil, appCfg, galleryService, opcache, &application.Application{}, noopMw) + req := httptest.NewRequest(http.MethodPost, "/api/operations/job-pause/pause", nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(paused).To(BeTrue()) + Expect(cancelled).To(BeFalse()) + Expect(opcache.Get("localai@gemma")).To(BeEmpty()) + }) + It("reports a running removal as a deletion", func() { state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir())) Expect(err).NotTo(HaveOccurred()) diff --git a/core/services/galleryop/cancel_persist_test.go b/core/services/galleryop/cancel_persist_test.go index c75a7bd85b8c..7cbf4e9701b7 100644 --- a/core/services/galleryop/cancel_persist_test.go +++ b/core/services/galleryop/cancel_persist_test.go @@ -54,6 +54,22 @@ var _ = Describe("GalleryService.CancelOperation persistence", func() { Expect(fresh.GetStatus("op-cancel")).To(BeNil(), "a cancelled op must not hydrate back as active after a restart") }) + + It("pauses with the resume-safe callback instead of the destructive cancel callback", func() { + svc := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + var cancelled, paused bool + svc.StoreCancellationActions("op-pause", func() { cancelled = true }, func() { paused = true }) + + Expect(svc.PauseOperation("op-pause")).To(Succeed()) + Expect(paused).To(BeTrue()) + Expect(cancelled).To(BeFalse()) + + status := svc.GetStatus("op-pause") + Expect(status).ToNot(BeNil()) + Expect(status.Processed).To(BeTrue()) + Expect(status.Cancelled).To(BeTrue()) + Expect(status.Message).To(Equal("paused")) + }) }) // Reproduces "an op orphaned by a replica that died mid-flight stays 'pending' diff --git a/core/services/galleryop/operation.go b/core/services/galleryop/operation.go index 73bcc0a7f6d9..4322b626dae1 100644 --- a/core/services/galleryop/operation.go +++ b/core/services/galleryop/operation.go @@ -32,6 +32,7 @@ type ManagementOp[T any, E any] struct { // Context for cancellation support Context context.Context CancelFunc context.CancelFunc + PauseFunc context.CancelFunc // External backend installation parameters (for OCI/URL/path) // These are used when installing backends from external sources rather than galleries @@ -189,6 +190,7 @@ type GalleryProgressEvent struct { // runs the cancel func on whichever replica registered it. type GalleryCancelEvent struct { JobID string `json:"id"` + Pause bool `json:"pause,omitempty"` } // NodeStatus values shared between NodeProgress (per-node tick) and the diff --git a/core/services/galleryop/service.go b/core/services/galleryop/service.go index 309f9fa9d202..7a51fe440c1a 100644 --- a/core/services/galleryop/service.go +++ b/core/services/galleryop/service.go @@ -28,7 +28,7 @@ type GalleryService struct { modelManager ModelManager backendManager BackendManager statuses map[string]*OpStatus - cancellations map[string]context.CancelFunc + cancellations map[string]cancellationActions // Distributed mode (nil when not in distributed mode). // natsClient is the wider MessagingClient (Publisher + subscribe methods) @@ -67,7 +67,7 @@ func NewGalleryService(appConfig *config.ApplicationConfig, ml *model.ModelLoade modelManager: NewLocalModelManager(appConfig, ml), backendManager: NewLocalBackendManager(appConfig, ml), statuses: make(map[string]*OpStatus), - cancellations: make(map[string]context.CancelFunc), + cancellations: make(map[string]cancellationActions), } } @@ -387,6 +387,17 @@ func (g *GalleryService) failStaleStatus(id string) { // SubjectGalleryCancelWildcard subscriber and runs it locally. The caller // gets a non-error reply so the UI shows the cancel as accepted. func (g *GalleryService) CancelOperation(id string) error { + return g.stopOperation(id, false) +} + +// PauseOperation stops an in-progress download while preserving its partial +// file. Re-submitting the same install resumes it through the downloader's +// existing HTTP Range support. +func (g *GalleryService) PauseOperation(id string) error { + return g.stopOperation(id, true) +} + +func (g *GalleryService) stopOperation(id string, pause bool) error { g.Lock() if status, ok := g.statuses[id]; ok && status.Cancelled { @@ -394,7 +405,7 @@ func (g *GalleryService) CancelOperation(id string) error { return fmt.Errorf("operation %q is already cancelled", id) } - cancelFunc, localExists := g.cancellations[id] + actions, localExists := g.cancellations[id] if localExists { delete(g.cancellations, id) } @@ -410,12 +421,12 @@ func (g *GalleryService) CancelOperation(id string) error { if status, ok := g.statuses[id]; ok { status.Cancelled = true status.Processed = true - status.Message = "cancelled" + status.Message = map[bool]string{true: "paused", false: "cancelled"}[pause] } else { g.statuses[id] = &OpStatus{ Cancelled: true, Processed: true, - Message: "cancelled", + Message: map[bool]string{true: "paused", false: "cancelled"}[pause], Cancellable: false, } } @@ -435,11 +446,15 @@ func (g *GalleryService) CancelOperation(id string) error { // I/O and user-provided callback after Unlock — the cancel-wildcard // subscriber loops back into applyCancel on this same replica, which // would otherwise deadlock on g.Mutex. - if cancelFunc != nil { - cancelFunc() + stopFunc := actions.cancel + if pause { + stopFunc = actions.pause + } + if stopFunc != nil { + stopFunc() } if nc != nil { - if err := nc.Publish(messaging.SubjectGalleryCancel(id), GalleryCancelEvent{JobID: id}); err != nil { + if err := nc.Publish(messaging.SubjectGalleryCancel(id), GalleryCancelEvent{JobID: id, Pause: pause}); err != nil { xlog.Warn("Failed to broadcast gallery cancel", "op_id", id, "error", err) } } @@ -452,9 +467,9 @@ func (g *GalleryService) CancelOperation(id string) error { // run the local cancel func if we have one (no echo via NATS), and reflect // the cancellation in the local statuses map. Idempotent: a replica that // already cancelled this op locally treats the inbound event as a no-op. -func (g *GalleryService) applyCancel(id string) { +func (g *GalleryService) applyCancel(id string, pause bool) { g.Lock() - cancelFunc, hasCancel := g.cancellations[id] + actions, hasCancel := g.cancellations[id] if hasCancel { delete(g.cancellations, id) } @@ -465,12 +480,12 @@ func (g *GalleryService) applyCancel(id string) { } status.Cancelled = true status.Processed = true - status.Message = "cancelled" + status.Message = map[bool]string{true: "paused", false: "cancelled"}[pause] } else { g.statuses[id] = &OpStatus{ Cancelled: true, Processed: true, - Message: "cancelled", + Message: map[bool]string{true: "paused", false: "cancelled"}[pause], Cancellable: false, } } @@ -478,8 +493,12 @@ func (g *GalleryService) applyCancel(id string) { // Invoke the cancel func after Unlock so a callback that touches // GalleryService doesn't re-enter the mutex. - if hasCancel { - cancelFunc() + stopFunc := actions.cancel + if pause { + stopFunc = actions.pause + } + if hasCancel && stopFunc != nil { + stopFunc() } } @@ -488,23 +507,40 @@ func (g *GalleryService) applyCancel(id string) { // distinguish a deliberate user cancel (discard the half-downloaded .partial) // from an incidental cancellation such as process shutdown (keep the .partial // so the next run resumes via Range instead of restarting from zero). -func newUserCancellableContext(parent context.Context) (context.Context, context.CancelFunc) { +// NewUserCancellableContext creates distinct callbacks for destructive cancel +// and resume-safe pause while sharing one operation context. +func NewUserCancellableContext(parent context.Context) (context.Context, context.CancelFunc, context.CancelFunc) { ctx, cancelCause := context.WithCancelCause(parent) - return ctx, func() { cancelCause(downloader.ErrUserCancelled) } + return ctx, + func() { cancelCause(downloader.ErrUserCancelled) }, + func() { cancelCause(context.Canceled) } +} + +type cancellationActions struct { + cancel context.CancelFunc + pause context.CancelFunc } -// storeCancellation stores a cancellation function for an operation -func (g *GalleryService) storeCancellation(id string, cancelFunc context.CancelFunc) { +func (g *GalleryService) storeCancellation(id string, cancelFunc, pauseFunc context.CancelFunc) { g.Lock() defer g.Unlock() - g.cancellations[id] = cancelFunc + if pauseFunc == nil { + pauseFunc = cancelFunc + } + g.cancellations[id] = cancellationActions{cancel: cancelFunc, pause: pauseFunc} } // StoreCancellation is a public method to store a cancellation function for an operation // This allows cancellation functions to be stored immediately when operations are created, // enabling cancellation of queued operations that haven't started processing yet. func (g *GalleryService) StoreCancellation(id string, cancelFunc context.CancelFunc) { - g.storeCancellation(id, cancelFunc) + g.storeCancellation(id, cancelFunc, cancelFunc) +} + +// StoreCancellationActions registers distinct destructive-cancel and +// resume-safe pause callbacks for an operation. +func (g *GalleryService) StoreCancellationActions(id string, cancelFunc, pauseFunc context.CancelFunc) { + g.storeCancellation(id, cancelFunc, pauseFunc) } // removeCancellation removes a cancellation function when operation completes @@ -554,10 +590,10 @@ func (g *GalleryService) Start(c context.Context, cl *config.ModelConfigLoader, case op := <-g.BackendGalleryChannel: // Create context if not provided if op.Context == nil { - op.Context, op.CancelFunc = newUserCancellableContext(c) - g.storeCancellation(op.ID, op.CancelFunc) + op.Context, op.CancelFunc, op.PauseFunc = NewUserCancellableContext(c) + g.storeCancellation(op.ID, op.CancelFunc, op.PauseFunc) } else if op.CancelFunc != nil { - g.storeCancellation(op.ID, op.CancelFunc) + g.storeCancellation(op.ID, op.CancelFunc, op.PauseFunc) } // Create DB record for distributed tracking if g.galleryStore != nil { @@ -597,10 +633,10 @@ func (g *GalleryService) Start(c context.Context, cl *config.ModelConfigLoader, case op := <-g.ModelGalleryChannel: // Create context if not provided if op.Context == nil { - op.Context, op.CancelFunc = newUserCancellableContext(c) - g.storeCancellation(op.ID, op.CancelFunc) + op.Context, op.CancelFunc, op.PauseFunc = NewUserCancellableContext(c) + g.storeCancellation(op.ID, op.CancelFunc, op.PauseFunc) } else if op.CancelFunc != nil { - g.storeCancellation(op.ID, op.CancelFunc) + g.storeCancellation(op.ID, op.CancelFunc, op.PauseFunc) } // Create DB record for distributed tracking if g.galleryStore != nil { @@ -663,7 +699,7 @@ func (g *GalleryService) SubscribeBroadcasts() error { if evt.JobID == "" { return } - g.applyCancel(evt.JobID) + g.applyCancel(evt.JobID, evt.Pause) }) if err != nil { if uerr := progressSub.Unsubscribe(); uerr != nil { diff --git a/docs/content/features/authentication.md b/docs/content/features/authentication.md index 6c206adbb35a..0df9f762b8f0 100644 --- a/docs/content/features/authentication.md +++ b/docs/content/features/authentication.md @@ -157,7 +157,7 @@ When authentication is enabled, the following endpoints require admin role: **Model & Backend Management:** - `GET /api/models`, `POST /api/models/install/*`, `POST /api/models/delete/*` - `GET /api/backends`, `POST /api/backends/install/*`, `POST /api/backends/delete/*` -- `GET /api/operations`, `POST /api/operations/*/cancel`, `POST /api/operations/*/dismiss` +- `GET /api/operations`, `POST /api/operations/*/cancel`, `POST /api/operations/*/pause`, `POST /api/operations/*/dismiss` - `GET /api/operations/history`, `DELETE /api/operations/history` - `GET /models/available`, `GET /models/galleries`, `GET /models/jobs/*` - `GET /backends`, `GET /backends/available`, `GET /backends/galleries` diff --git a/docs/content/operations/activity.md b/docs/content/operations/activity.md index f06865615278..f6172c3291a9 100644 --- a/docs/content/operations/activity.md +++ b/docs/content/operations/activity.md @@ -170,10 +170,15 @@ Both surfaces are backed by these endpoints. All of them are admin-only when | -------- | -------------------------------- | ------------------------------------------------------------------------ | | `GET` | `/api/operations` | Running, queued and failed operations, least advanced first. | | `POST` | `/api/operations/{jobID}/cancel` | Cancel a queued operation, or a running install. Not a running removal. | +| `POST` | `/api/operations/{jobID}/pause` | Pause a queued operation or running download and preserve partial data. | | `POST` | `/api/operations/{jobID}/dismiss`| Acknowledge a failed operation and move it into the record. | | `GET` | `/api/operations/history` | List finished operations, newest first. | | `DELETE` | `/api/operations/history` | Clear the record. Live operations are untouched. | +Pausing preserves the download's `.partial` file. Start the same model or +backend installation again to resume from the saved bytes when the origin +supports HTTP Range requests. Cancelling intentionally discards partial data. + Both `GET` endpoints wrap their list in an `operations` key rather than returning a bare array: