Fork VM (#114)

* Implement cloud-hypervisor fork

* move fork support checks behind hypervisor starter interface

* api: add fork instance endpoint

* refactor: move CH fork snapshot rewrite into CH package

* fork: support running source via standby-resume flow

* Fix fork restore guest IP reconfiguration

* Add QEMU fork support for running standby flow

* Add fork target state and fix running-fork cleanup

* Stabilize Linux CI test timeouts

* Deep-copy metadata refs in fork path

* Fix fork CID persistence semantics and cleanup validation

* Stabilize firecracker fork integration test assertions

* Harden fork rewrite safety and volume fork validation

* Add Firecracker standby fork support

* Address remaining fork review findings

* Add firecracker fork support

* Use running Firecracker fork test and gate on guest-agent readiness

* Fail running firecracker fork when guest agent is not ready

* Serialize firecracker snapshot source aliasing during restore

* Fix fork name collisions and running-source restore ordering

Author: sjmiller609

.github/workflows/test.yml

@@ -70,7 +70,7 @@ jobs:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           TLS_TEST_DOMAIN: "test.hypeman-development.com"
           TLS_ALLOWED_DOMAINS: '*.hypeman-development.com'
-        run: make test
+        run: make test TEST_TIMEOUT=20m
 
   test-darwin:
     runs-on: [self-hosted, macos, arm64]

cmd/api/api/instances.go

@@ -447,6 +447,74 @@ func (s *ApiService) RestoreInstance(ctx context.Context, request oapi.RestoreIn
 	return oapi.RestoreInstance200JSONResponse(instanceToOAPI(*result)), nil
 }
 
+// ForkInstance forks an instance from stopped or standby into a new instance.
+// The id parameter can be an instance ID, name, or ID prefix.
+// Note: Resolution is handled by ResolveResource middleware.
+func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstanceRequestObject) (oapi.ForkInstanceResponseObject, error) {
+	inst := mw.GetResolvedInstance[instances.Instance](ctx)
+	if inst == nil {
+		return oapi.ForkInstance500JSONResponse{
+			Code:    "internal_error",
+			Message: "resource not resolved",
+		}, nil
+	}
+	log := logger.FromContext(ctx)
+
+	if request.Body == nil {
+		return oapi.ForkInstance400JSONResponse{
+			Code:    "invalid_request",
+			Message: "request body is required",
+		}, nil
+	}
+
+	targetState := instances.State("")
+	if request.Body.TargetState != nil {
+		targetState = instances.State(*request.Body.TargetState)
+	}
+
+	result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{
+		Name:        request.Body.Name,
+		FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning,
+		TargetState: targetState,
+	})
+	if err != nil {
+		switch {
+		case errors.Is(err, instances.ErrNotFound):
+			return oapi.ForkInstance404JSONResponse{
+				Code:    "not_found",
+				Message: "instance not found",
+			}, nil
+		case errors.Is(err, instances.ErrInvalidState):
+			return oapi.ForkInstance409JSONResponse{
+				Code:    "invalid_state",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, instances.ErrInvalidRequest):
+			return oapi.ForkInstance400JSONResponse{
+				Code:    "invalid_request",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, instances.ErrAlreadyExists), errors.Is(err, network.ErrNameExists):
+			return oapi.ForkInstance409JSONResponse{
+				Code:    "name_conflict",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, instances.ErrNotSupported):
+			return oapi.ForkInstance501JSONResponse{
+				Code:    "not_supported",
+				Message: err.Error(),
+			}, nil
+		default:
+			log.ErrorContext(ctx, "failed to fork instance", "error", err)
+			return oapi.ForkInstance500JSONResponse{
+				Code:    "internal_error",
+				Message: "failed to fork instance",
+			}, nil
+		}
+	}
+	return oapi.ForkInstance201JSONResponse(instanceToOAPI(*result)), nil
+}
+
 // StopInstance gracefully stops a running instance
 // The id parameter can be an instance ID, name, or ID prefix
 // Note: Resolution is handled by ResolveResource middleware

cmd/api/api/instances_test.go

@@ -2,13 +2,15 @@ package api
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/c2h5oh/datasize"
 	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/instances"
+	mw "github.com/kernel/hypeman/lib/middleware"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/paths"
 	"github.com/kernel/hypeman/lib/system"
@@ -137,6 +139,24 @@ type captureCreateManager struct {
 	lastReq *instances.CreateInstanceRequest
 }
 
+type captureForkManager struct {
+	instances.Manager
+	lastID  string
+	lastReq *instances.ForkInstanceRequest
+	result  *instances.Instance
+	err     error
+}
+
+func (m *captureForkManager) ForkInstance(ctx context.Context, id string, req instances.ForkInstanceRequest) (*instances.Instance, error) {
+	reqCopy := req
+	m.lastID = id
+	m.lastReq = &reqCopy
+	if m.err != nil {
+		return nil, m.err
+	}
+	return m.result, nil
+}
+
 func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances.CreateInstanceRequest) (*instances.Instance, error) {
 	reqCopy := req
 	m.lastReq = &reqCopy
@@ -190,6 +210,185 @@ func TestCreateInstance_OmittedHotplugSizeDefaultsToZero(t *testing.T) {
 	assert.Equal(t, int64(0), int64(hotplugBytes), "response should report zero hotplug_size when omitted")
 }
 
+func TestForkInstance_Success(t *testing.T) {
+	svc := newTestService(t)
+
+	now := time.Now()
+	source := instances.Instance{
+		StoredMetadata: instances.StoredMetadata{
+			Id:             "src-instance",
+			Name:           "src-instance",
+			Image:          "docker.io/library/alpine:latest",
+			CreatedAt:      now,
+			HypervisorType: hypervisor.TypeCloudHypervisor,
+		},
+		State: instances.StateStopped,
+	}
+
+	forked := &instances.Instance{
+		StoredMetadata: instances.StoredMetadata{
+			Id:             "forked-instance",
+			Name:           "forked-instance",
+			Image:          "docker.io/library/alpine:latest",
+			CreatedAt:      now,
+			HypervisorType: hypervisor.TypeCloudHypervisor,
+		},
+		State: instances.StateStopped,
+	}
+
+	mockMgr := &captureForkManager{
+		Manager: svc.InstanceManager,
+		result:  forked,
+	}
+	svc.InstanceManager = mockMgr
+
+	resp, err := svc.ForkInstance(
+		mw.WithResolvedInstance(ctx(), source.Id, source),
+		oapi.ForkInstanceRequestObject{
+			Id: source.Id,
+			Body: &oapi.ForkInstanceRequest{
+				Name: "forked-instance",
+			},
+		},
+	)
+	require.NoError(t, err)
+
+	created, ok := resp.(oapi.ForkInstance201JSONResponse)
+	require.True(t, ok, "expected 201 response")
+	assert.Equal(t, "forked-instance", created.Name)
+	assert.Equal(t, source.Id, mockMgr.lastID)
+	require.NotNil(t, mockMgr.lastReq)
+	assert.Equal(t, "forked-instance", mockMgr.lastReq.Name)
+	assert.False(t, mockMgr.lastReq.FromRunning)
+	assert.Equal(t, instances.State(""), mockMgr.lastReq.TargetState)
+}
+
+func TestForkInstance_NotSupported(t *testing.T) {
+	svc := newTestService(t)
+
+	source := instances.Instance{
+		StoredMetadata: instances.StoredMetadata{
+			Id:             "src-instance",
+			Name:           "src-instance",
+			Image:          "docker.io/library/alpine:latest",
+			CreatedAt:      time.Now(),
+			HypervisorType: hypervisor.TypeQEMU,
+		},
+		State: instances.StateStopped,
+	}
+
+	mockMgr := &captureForkManager{
+		Manager: svc.InstanceManager,
+		err:     instances.ErrNotSupported,
+	}
+	svc.InstanceManager = mockMgr
+
+	resp, err := svc.ForkInstance(
+		mw.WithResolvedInstance(ctx(), source.Id, source),
+		oapi.ForkInstanceRequestObject{
+			Id: source.Id,
+			Body: &oapi.ForkInstanceRequest{
+				Name: "forked-instance",
+			},
+		},
+	)
+	require.NoError(t, err)
+
+	notSupported, ok := resp.(oapi.ForkInstance501JSONResponse)
+	require.True(t, ok, "expected 501 response")
+	assert.Equal(t, "not_supported", notSupported.Code)
+}
+
+func TestForkInstance_InvalidRequest(t *testing.T) {
+	svc := newTestService(t)
+
+	source := instances.Instance{
+		StoredMetadata: instances.StoredMetadata{
+			Id:             "src-instance",
+			Name:           "src-instance",
+			Image:          "docker.io/library/alpine:latest",
+			CreatedAt:      time.Now(),
+			HypervisorType: hypervisor.TypeCloudHypervisor,
+		},
+		State: instances.StateStopped,
+	}
+
+	mockMgr := &captureForkManager{
+		Manager: svc.InstanceManager,
+		err:     fmt.Errorf("%w: name is required", instances.ErrInvalidRequest),
+	}
+	svc.InstanceManager = mockMgr
+
+	resp, err := svc.ForkInstance(
+		mw.WithResolvedInstance(ctx(), source.Id, source),
+		oapi.ForkInstanceRequestObject{
+			Id: source.Id,
+			Body: &oapi.ForkInstanceRequest{
+				Name: "",
+			},
+		},
+	)
+	require.NoError(t, err)
+
+	badReq, ok := resp.(oapi.ForkInstance400JSONResponse)
+	require.True(t, ok, "expected 400 response")
+	assert.Equal(t, "invalid_request", badReq.Code)
+}
+
+func TestForkInstance_FromRunningFlagForwarded(t *testing.T) {
+	svc := newTestService(t)
+
+	now := time.Now()
+	source := instances.Instance{
+		StoredMetadata: instances.StoredMetadata{
+			Id:             "src-instance",
+			Name:           "src-instance",
+			Image:          "docker.io/library/alpine:latest",
+			CreatedAt:      now,
+			HypervisorType: hypervisor.TypeCloudHypervisor,
+		},
+		State: instances.StateRunning,
+	}
+
+	forked := &instances.Instance{
+		StoredMetadata: instances.StoredMetadata{
+			Id:             "forked-instance",
+			Name:           "forked-instance",
+			Image:          "docker.io/library/alpine:latest",
+			CreatedAt:      now,
+			HypervisorType: hypervisor.TypeCloudHypervisor,
+		},
+		State: instances.StateStandby,
+	}
+
+	mockMgr := &captureForkManager{
+		Manager: svc.InstanceManager,
+		result:  forked,
+	}
+	svc.InstanceManager = mockMgr
+
+	fromRunning := true
+	targetState := oapi.ForkTargetStateRunning
+	resp, err := svc.ForkInstance(
+		mw.WithResolvedInstance(ctx(), source.Id, source),
+		oapi.ForkInstanceRequestObject{
+			Id: source.Id,
+			Body: &oapi.ForkInstanceRequest{
+				Name:        "forked-instance",
+				FromRunning: &fromRunning,
+				TargetState: &targetState,
+			},
+		},
+	)
+	require.NoError(t, err)
+
+	_, ok := resp.(oapi.ForkInstance201JSONResponse)
+	require.True(t, ok, "expected 201 response")
+	require.NotNil(t, mockMgr.lastReq)
+	assert.True(t, mockMgr.lastReq.FromRunning)
+	assert.Equal(t, instances.StateRunning, mockMgr.lastReq.TargetState)
+}
+
 func TestInstanceLifecycle_StopStart(t *testing.T) {
 	// Require KVM access for VM creation
 	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {

lib/builds/manager_test.go

@@ -84,6 +84,10 @@ func (m *mockInstanceManager) DeleteInstance(ctx context.Context, id string) err
 	return nil
 }
 
+func (m *mockInstanceManager) ForkInstance(ctx context.Context, id string, req instances.ForkInstanceRequest) (*instances.Instance, error) {
+	return nil, instances.ErrNotFound
+}
+
 func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string) (*instances.Instance, error) {
 	return nil, nil
 }

lib/forkvm/README.md

@@ -0,0 +1,64 @@
+# VM Forking: Hypervisor Behavior
+
+This document describes hypervisor-specific fork behavior and how fork is made
+to work across implementations.
+
+## Common fork model
+
+- **Stopped source**: clone VM data and start a new VM from copied state.
+- **Standby source**: clone data + snapshot artifacts, then adapt snapshot
+  identity for the fork (paths, network, vsock behavior varies by hypervisor).
+- **Running source**: transition source to standby, fork from that standby
+  snapshot, then restore the source.
+
+For networked forks, the fork gets a fresh host/guest identity (IP, MAC, TAP)
+instead of reusing the source identity.
+
+## Cloud Hypervisor
+
+- Snapshot-based forks are supported by rewriting snapshot configuration before
+  restore.
+- Path rewrites are constrained to exact source-directory matches or source-dir
+  path prefixes to avoid mutating unrelated values.
+- Serial log path, vsock socket path, and network fields are updated for the
+  fork.
+- Vsock CID is intentionally kept stable for snapshot restore compatibility.
+- Running-source fork works by standby -> fork -> restore source, with source
+  and fork separated by rewritten runtime endpoints.
+
+## QEMU
+
+- Snapshot-based forks are supported by rewriting QEMU snapshot VM config.
+- Rewrites are explicit and path-safe (source-dir exact/prefix replacement),
+  applied to disk/kernel/initrd/serial/vsock socket paths.
+- Kernel arguments are left unchanged (not blanket-rewritten), to avoid
+  accidental mutation of non-path text.
+- Network identity is updated in snapshot config for the fork.
+- Vsock CID updates are supported for snapshot state, so running-source fork can
+  rotate source CID when needed to avoid CID collision after restore.
+
+## Firecracker
+
+- Firecracker snapshot restore supports **network overrides** but does not
+  expose a full snapshot-config rewrite surface for arbitrary embedded paths.
+- To make standby/running fork work, fork preparation stores desired network
+  override data and source->target data-directory mapping.
+- During restore, the source data path is temporarily aliased to the fork data
+  path so embedded snapshot paths resolve for the fork, then aliasing is
+  cleaned up.
+- Network override fields are supplied at snapshot load to bind the fork to its
+  own TAP device.
+- Vsock CID remains stable for snapshot-based flows.
+
+## VZ (Virtualization.framework)
+
+- Fork is not supported.
+- Snapshot restore for Linux guests is not available in this mode, so standby
+  snapshot-based fork mechanics cannot be implemented.
+
+## Operational constraints
+
+- Writable attached volumes are rejected for fork to prevent concurrent
+  cross-VM writes to the same backing data.
+- If a post-fork target-state transition fails, the partially created fork is
+  cleaned up rather than left orphaned.