feat: enhance runtime scroll creation with UUID generation and timeout settings

- Added UUID generation for runtime scroll IDs when omitted during creation, ensuring unique identifiers.
- Implemented a timeout for HTTP clients in the OpenAPI client to improve reliability.
- Updated OpenAPI specifications and related documentation to reflect changes in ID handling.
- Refactored error handling in the runtime supervisor to manage scroll creation more effectively.
- Enhanced tests to cover new functionality and ensure robustness in scroll management.

Author: MarcStdt

Makefile

@@ -35,8 +35,9 @@ k3d-build-pull-image: ## Build the unified Druid runtime image and import it int
 build-x86-docker:
 	docker run -e GOOS=linux -e GOARCH=amd64 -it --rm -v ./:/app -w /app --entrypoint=/bin/bash docker.elastic.co/beats-dev/golang-crossbuild:1.22.5-main  -c 'CGO_ENABLED=1 go build -ldflags "-X github.com/highcard-dev/daemon/internal.Version=$(VERSION)" -o ./bin/x86/druid'
 
-install: ## Install Daemon
-	cp ./bin/druid /usr/local/bin/druid
+install: build ## Build and install Druid binaries
+	install -m 0755 ./bin/druid /usr/local/bin/druid
+	install -m 0755 ./bin/druid-coldstarter /usr/local/bin/druid-coldstarter
 
 generate-md-docs:
 	go run ./docs_md/main.go

api/openapi.yaml

@@ -76,11 +76,11 @@ components:
       properties:
         id:
           type: string
-          description: Deprecated alias for name. Optional local runtime scroll id/name.
+          description: Deprecated alias for name. Optional stable runtime id/name. If omitted, the daemon generates an id.
           example: jobs
         name:
           type: string
-          description: Optional local runtime scroll id/name. If omitted, the daemon derives it from scroll.yaml name.
+          description: Optional stable runtime id/name. If omitted, the daemon generates an id; the display name still comes from scroll.yaml.
           example: jobs
         artifact:
           type: string

apps/druid/adapters/daemonclient/openapi_client.go

@@ -4,19 +4,19 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/highcard-dev/daemon/internal/api"
 	"github.com/highcard-dev/daemon/internal/utils"
 )
 
-var ErrMaterializationUnsupported = errors.New("daemon materialization unsupported")
+const daemonRequestTimeout = 5 * time.Second
 
 type OpenAPIClient struct {
 	client     *api.ClientWithResponses
@@ -31,21 +31,22 @@ func NewOpenAPIClient(daemonSocket string) (*OpenAPIClient, error) {
 func NewOpenAPIClientForTarget(daemonSocket string, daemonURL string) (*OpenAPIClient, error) {
 	if daemonURL != "" {
 		server := strings.TrimRight(daemonURL, "/")
-		client, err := api.NewClientWithResponses(server)
+		httpClient := &http.Client{Timeout: daemonRequestTimeout}
+		client, err := api.NewClientWithResponses(server, api.WithHTTPClient(httpClient))
 		if err != nil {
 			return nil, err
 		}
-		return &OpenAPIClient{client: client, server: server, httpClient: http.DefaultClient}, nil
+		return &OpenAPIClient{client: client, server: server, httpClient: httpClient}, nil
 	}
 	if daemonSocket == "" {
 		daemonSocket = utils.DefaultRuntimeSocketPath()
 	}
 	transport := &http.Transport{
 		DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
-			return (&net.Dialer{}).DialContext(ctx, "unix", daemonSocket)
+			return (&net.Dialer{Timeout: daemonRequestTimeout}).DialContext(ctx, "unix", daemonSocket)
 		},
 	}
-	httpClient := &http.Client{Transport: transport}
+	httpClient := &http.Client{Transport: transport, Timeout: daemonRequestTimeout}
 	client, err := api.NewClientWithResponses("http://druid", api.WithHTTPClient(httpClient))
 	if err != nil {
 		return nil, err
@@ -69,9 +70,6 @@ func (c *OpenAPIClient) CreateScroll(ctx context.Context, name string, artifact
 	if err != nil {
 		return nil, err
 	}
-	if res.StatusCode() == http.StatusNotImplemented {
-		return nil, ErrMaterializationUnsupported
-	}
 	if err := ensureStatus(res.StatusCode(), res.Body); err != nil {
 		return nil, err
 	}

apps/druid/adapters/daemonclient/openapi_client_test.go

@@ -9,6 +9,24 @@ import (
 	"github.com/highcard-dev/daemon/internal/api"
 )
 
+func TestOpenAPIClientHasDaemonTimeout(t *testing.T) {
+	socketClient, err := NewOpenAPIClientForTarget("/tmp/druid-test.sock", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if socketClient.httpClient.Timeout != daemonRequestTimeout {
+		t.Fatalf("socket timeout = %s, want %s", socketClient.httpClient.Timeout, daemonRequestTimeout)
+	}
+
+	urlClient, err := NewOpenAPIClientForTarget("", "http://127.0.0.1:1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if urlClient.httpClient.Timeout != daemonRequestTimeout {
+		t.Fatalf("url timeout = %s, want %s", urlClient.httpClient.Timeout, daemonRequestTimeout)
+	}
+}
+
 func TestCreateScrollDoesNotSendStart(t *testing.T) {
 	var got map[string]interface{}
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

apps/druid/adapters/http/handlers/scroll_handler.go

@@ -79,9 +79,6 @@ func (h *ScrollHandler) CreateScroll(c *fiber.Ctx) error {
 		if errors.Is(err, domain.ErrRuntimeScrollAlreadyExists) {
 			return fiber.NewError(fiber.StatusConflict, err.Error())
 		}
-		if errors.Is(err, appservices.ErrRuntimeMaterializationUnsupported) {
-			return fiber.NewError(fiber.StatusNotImplemented, err.Error())
-		}
 		return err
 	}
 	return c.Status(fiber.StatusCreated).JSON(runtimeScroll)
@@ -108,9 +105,6 @@ func (h *ScrollHandler) EnsureScroll(c *fiber.Ctx) error {
 	}
 	runtimeScroll, err := h.supervisor.EnsureWithOwner(request.Artifact, name, ownerID, namespace, registryCredentials(request.RegistryCredentials))
 	if err != nil {
-		if errors.Is(err, appservices.ErrRuntimeMaterializationUnsupported) {
-			return fiber.NewError(fiber.StatusNotImplemented, err.Error())
-		}
 		return err
 	}
 	return c.JSON(runtimeScroll)

apps/druid/core/services/runtime_materialization.go

@@ -9,23 +9,19 @@ import (
 
 	"github.com/highcard-dev/daemon/internal/core/domain"
 	"github.com/highcard-dev/daemon/internal/core/ports"
-	coreservices "github.com/highcard-dev/daemon/internal/core/services"
 	"github.com/highcard-dev/daemon/internal/core/services/registry"
 	"github.com/highcard-dev/daemon/internal/utils/logger"
 	"go.uber.org/zap"
 )
 
-var ErrRuntimeMaterializationUnsupported = errors.New("runtime backend does not support daemon materialization")
-
-func (s *RuntimeSupervisor) materializeNewScroll(ctx context.Context, runtimeService ports.RuntimeBackendInterface, artifact string, name string, namespace string, registryCredentials []domain.RegistryCredential) (*ports.RuntimeMaterialization, error) {
-	id := coreservices.RuntimeScrollIDFromName(name)
-	if id == "" {
-		return nil, ErrRuntimeMaterializationUnsupported
-	}
-	return s.runPullWorker(ctx, runtimeService, ports.RuntimeWorkerModeCreate, id, artifact, runtimeService.RootRef(id, namespace), registryCredentials)
+func (s *RuntimeSupervisor) materializeNewScroll(ctx context.Context, runtimeService ports.RuntimeBackendInterface, artifact string, runtimeID string, namespace string, registryCredentials []domain.RegistryCredential) (*ports.RuntimeMaterialization, error) {
+	return s.runPullWorker(ctx, runtimeService, ports.RuntimeWorkerModeCreate, runtimeID, artifact, runtimeService.RootRef(runtimeID, namespace), registryCredentials)
 }
 
 func (s *RuntimeSupervisor) runPullWorker(ctx context.Context, runtimeService ports.RuntimeBackendInterface, mode ports.RuntimeWorkerMode, runtimeID string, artifact string, root string, registryCredentials []domain.RegistryCredential) (*ports.RuntimeMaterialization, error) {
+	if s.workerCallbacks == nil || s.workerCallbackURL == "" {
+		return nil, fmt.Errorf("daemon materialization requires --worker-callback-url and --worker-callback-listen")
+	}
 	token, resultCh, err := s.workerCallbacks.Register(runtimeID)
 	if err != nil {
 		return nil, err

apps/druid/core/services/runtime_supervisor.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/google/uuid"
 	"github.com/highcard-dev/daemon/internal/core/domain"
 	"github.com/highcard-dev/daemon/internal/core/ports"
 	coreservices "github.com/highcard-dev/daemon/internal/core/services"
@@ -83,54 +84,44 @@ func (s *RuntimeSupervisor) Create(artifact string, name string, registryCredent
 
 func (s *RuntimeSupervisor) CreateWithOwner(artifact string, name string, ownerID string, namespace string, registryCredentials []domain.RegistryCredential) (*domain.RuntimeScroll, error) {
 	id := coreservices.RuntimeScrollIDFromName(name)
-	var placeholder *domain.RuntimeScroll
-	if id != "" {
-		if _, err := s.store.GetScroll(id); err == nil {
-			return nil, fmt.Errorf("%w: %s", domain.ErrRuntimeScrollAlreadyExists, id)
-		} else if !errors.Is(err, domain.ErrRuntimeScrollNotFound) {
-			return nil, err
-		}
-		placeholder = &domain.RuntimeScroll{
-			ID:       id,
-			OwnerID:  ownerID,
-			Artifact: artifact,
-			Root:     s.runtimeBackend.RootRef(id, namespace),
-			Status:   domain.RuntimeScrollStatusCreated,
-			Commands: map[string]domain.LockStatus{},
-		}
-		if err := s.store.CreateScroll(placeholder); err != nil {
-			return nil, err
-		}
+	if id == "" {
+		id = uuid.NewString()
+	}
+	if _, err := s.store.GetScroll(id); err == nil {
+		return nil, fmt.Errorf("%w: %s", domain.ErrRuntimeScrollAlreadyExists, id)
+	} else if !errors.Is(err, domain.ErrRuntimeScrollNotFound) {
+		return nil, err
+	}
+	placeholder := &domain.RuntimeScroll{
+		ID:       id,
+		OwnerID:  ownerID,
+		Artifact: artifact,
+		Root:     s.runtimeBackend.RootRef(id, namespace),
+		Status:   domain.RuntimeScrollStatusCreated,
+		Commands: map[string]domain.LockStatus{},
+	}
+	if err := s.store.CreateScroll(placeholder); err != nil {
+		return nil, err
 	}
 	markPlaceholderError := func(cause error) {
-		if placeholder == nil {
-			return
-		}
 		placeholder.Status = domain.RuntimeScrollStatusError
 		placeholder.LastError = cause.Error()
 		_ = s.store.UpdateScroll(placeholder)
 	}
 
-	materialized, err := s.materializeNewScroll(context.Background(), s.runtimeBackend, artifact, name, namespace, registryCredentials)
+	materialized, err := s.materializeNewScroll(context.Background(), s.runtimeBackend, artifact, id, namespace, registryCredentials)
 	if err != nil {
 		markPlaceholderError(err)
 		return nil, err
 	}
 	if materialized.Artifact != "" {
 		artifact = materialized.Artifact
 	}
-	if placeholder != nil {
-		placeholder, err = s.applyMaterializedScroll(placeholder, artifact, materialized)
-		if err != nil {
-			return nil, err
-		}
-		return placeholder, nil
-	}
-	runtimeScroll, err := s.manager.CreateWithDigest(artifact, materialized.ArtifactDigest, name, ownerID, materialized.Root, materialized.ScrollYAML)
+	placeholder, err = s.applyMaterializedScroll(placeholder, artifact, materialized)
 	if err != nil {
 		return nil, err
 	}
-	return runtimeScroll, nil
+	return placeholder, nil
 }
 
 func (s *RuntimeSupervisor) Ensure(artifact string, name string, registryCredentials []domain.RegistryCredential) (*domain.RuntimeScroll, error) {
@@ -161,7 +152,7 @@ func (s *RuntimeSupervisor) EnsureWithOwner(artifact string, name string, ownerI
 				if artifact == "" {
 					artifact = runtimeScroll.Artifact
 				}
-				materialized, err := s.materializeNewScroll(context.Background(), s.runtimeBackend, artifact, name, namespace, registryCredentials)
+				materialized, err := s.materializeNewScroll(context.Background(), s.runtimeBackend, artifact, id, namespace, registryCredentials)
 				if err != nil {
 					runtimeScroll.Status = domain.RuntimeScrollStatusError
 					runtimeScroll.LastError = err.Error()

apps/druid/core/services/runtime_supervisor_test.go

@@ -218,6 +218,38 @@ func TestRuntimeSupervisorCreateCanCreate(t *testing.T) {
 	}
 }
 
+func TestRuntimeSupervisorCreateGeneratesIDWhenNameOmitted(t *testing.T) {
+	store := newTestStateStore(t)
+	callbacks := NewWorkerCallbackManager()
+	backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: cachedScrollYAML("start"), digest: "sha256:generated"}
+	supervisor := NewRuntimeSupervisor(
+		store,
+		coreservices.NewRuntimeScrollManager(store),
+		backend,
+	)
+	supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083")
+
+	runtimeScroll, err := supervisor.Create("registry.local/lab:1.0", "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runtimeScroll.ID == "" || runtimeScroll.ID == "cached" {
+		t.Fatalf("id = %q, want generated runtime id independent from scroll.yaml name", runtimeScroll.ID)
+	}
+	if runtimeScroll.ScrollName != "cached" {
+		t.Fatalf("scroll name = %q, want cached", runtimeScroll.ScrollName)
+	}
+	if backend.action.RuntimeID != runtimeScroll.ID || backend.action.RootRef != backend.RootRef(runtimeScroll.ID, "") {
+		t.Fatalf("worker action = %#v scroll = %#v", backend.action, runtimeScroll)
+	}
+	if backend.action.Mode != ports.RuntimeWorkerModeCreate || backend.action.CallbackToken == "" {
+		t.Fatalf("worker action = %#v", backend.action)
+	}
+	if runtimeScroll.ArtifactDigest != "sha256:generated" || runtimeScroll.Status != domain.RuntimeScrollStatusCreated {
+		t.Fatalf("runtime scroll = %#v", runtimeScroll)
+	}
+}
+
 func TestRuntimeSupervisorCreateUsesPullWorkerBeforeStateMutation(t *testing.T) {
 	store := newTestStateStore(t)
 	callbacks := NewWorkerCallbackManager()
@@ -250,6 +282,52 @@ func TestRuntimeSupervisorCreateUsesPullWorkerBeforeStateMutation(t *testing.T)
 	}
 }
 
+func TestRuntimeSupervisorCreateWorkerFailureLeavesGeneratedPlaceholder(t *testing.T) {
+	store := newTestStateStore(t)
+	callbacks := NewWorkerCallbackManager()
+	backend := &fakeWorkerBackend{callbacks: callbacks, workerErr: errors.New("pull image failed")}
+	supervisor := NewRuntimeSupervisor(
+		store,
+		coreservices.NewRuntimeScrollManager(store),
+		backend,
+	)
+	supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083")
+
+	if _, err := supervisor.Create("registry.local/missing:1.0", "", nil); err == nil {
+		t.Fatal("Create error = nil, want worker error")
+	}
+	scrolls, err := store.ListScrolls()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(scrolls) != 1 {
+		t.Fatalf("scrolls = %#v, want one failed placeholder", scrolls)
+	}
+	if scrolls[0].Status != domain.RuntimeScrollStatusError || !strings.Contains(scrolls[0].LastError, "pull image failed") {
+		t.Fatalf("placeholder = %#v, want remembered worker failure", scrolls[0])
+	}
+}
+
+func TestRuntimeSupervisorCreateRequiresWorkerCallbackConfig(t *testing.T) {
+	store := newTestStateStore(t)
+	supervisor := NewRuntimeSupervisor(
+		store,
+		coreservices.NewRuntimeScrollManager(store),
+		&fakeWorkerBackend{scrollYAML: cachedScrollYAML("start")},
+	)
+
+	if _, err := supervisor.Create("registry.local/lab:1.0", "missing-callbacks", nil); err == nil || !strings.Contains(err.Error(), "daemon materialization requires --worker-callback-url and --worker-callback-listen") {
+		t.Fatalf("Create error = %v, want explicit callback config error", err)
+	}
+	runtimeScroll, err := store.GetScroll("missing-callbacks")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runtimeScroll.Status != domain.RuntimeScrollStatusError || !strings.Contains(runtimeScroll.LastError, "--worker-callback-url") {
+		t.Fatalf("runtime scroll = %#v, want callback config error", runtimeScroll)
+	}
+}
+
 func TestRuntimeSupervisorCreateUsesRequestedNamespaceForRoot(t *testing.T) {
 	store := newTestStateStore(t)
 	callbacks := NewWorkerCallbackManager()

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/gofiber/contrib/websocket v1.3.4
 	github.com/gofiber/fiber/v2 v2.52.9
+	github.com/google/uuid v1.6.0
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1
@@ -22,7 +23,6 @@ require (
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.1 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/joho/godotenv v1.5.1
 	github.com/josharian/intern v1.0.0 // indirect