fork: hardlink shared mem-file to dodge restore-alias ELOOP

The previous fork-mem-file symlink looped through withSnapshotSourceDirAlias
during firecracker restore: fork/.../memory -> source/.../memory, but the
alias dance temporarily symlinks source dir -> fork dir, so resolution
ping-pongs back to fork/.../memory and trips ELOOP. Switching to a hardlink
makes the fork's mem-file resolve by inode so the temporary directory alias
no longer affects it.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: sjmiller609

lib/instances/firecracker_test.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -557,7 +558,8 @@ func TestFirecrackerSnapshotFeature(t *testing.T) {
 // Standby, the first fork implicitly promotes it to Template, and the fork:
 //
 //	(a) reaches Running,
-//	(b) has its mem-file as a symlink into the template's snapshot dir,
+//	(b) has its mem-file hardlinked to the source's snapshot mem-file
+//	    (the fan-out optimisation),
 //	(c) bumps the template's ForkCount to 1,
 //	(d) registers with the per-template uffd page server,
 //	(e) on delete, decrements the ForkCount and detaches from uffd.
@@ -617,15 +619,20 @@ func TestFirecrackerForkFromTemplate(t *testing.T) {
 		}
 	})
 
-	// (b) The fork's mem-file must be a symlink into the source's snapshot.
+	// (b) The fork's mem-file must share the source's inode (hardlink), not
+	// be a copy. We can't compare paths because the link is by inode; we
+	// compare st_ino + st_dev between the two instances' mem-files.
 	forkMemPath := filepath.Join(p.InstanceSnapshotLatest(forkID), templateSharedMemFileName)
-	info, err := os.Lstat(forkMemPath)
+	srcMemPath := filepath.Join(p.InstanceSnapshotLatest(sourceID), templateSharedMemFileName)
+	forkInfo, err := os.Stat(forkMemPath)
 	require.NoError(t, err, "fork mem-file should exist at snapshot-latest/memory")
-	assert.Equal(t, os.ModeSymlink, info.Mode()&os.ModeSymlink, "fork mem-file should be a symlink, not a copy")
-	target, err := os.Readlink(forkMemPath)
+	assert.True(t, forkInfo.Mode().IsRegular(), "fork mem-file should be a regular file (hardlink), not a symlink")
+	srcInfo, err := os.Stat(srcMemPath)
 	require.NoError(t, err)
-	expectedTarget := filepath.Join(p.InstanceSnapshotLatest(sourceID), templateSharedMemFileName)
-	assert.Equal(t, expectedTarget, target, "fork mem-file symlink should point at the template's mem-file")
+	forkSys := forkInfo.Sys().(*syscall.Stat_t)
+	srcSys := srcInfo.Sys().(*syscall.Stat_t)
+	assert.Equal(t, srcSys.Ino, forkSys.Ino, "fork mem-file should share the source's inode (hardlink, not copy)")
+	assert.Equal(t, srcSys.Dev, forkSys.Dev, "fork mem-file should be on the same filesystem as source")
 
 	// (c) The source instance is now a Template with ForkCount=1.
 	sourceMeta, err := mgr.loadMetadata(sourceID)

lib/instances/templates.go

@@ -11,15 +11,22 @@ const (
 	templateSharedMemFileRelPath = "snapshots/snapshot-latest/memory"
 )
 
-// installForkSharedMemFile arranges the fork's snapshot directory so the
-// guest mem-file is a symlink into the source template instance's snapshot
-// directory instead of a per-fork copy. firecracker mmaps the mem-file
-// MAP_PRIVATE during restore, so all forks COW from the same backing file.
+// guest mem-file is a hardlink to the source template instance's snapshot
+// mem-file instead of a per-fork copy. firecracker mmaps the mem-file
+// MAP_PRIVATE during restore, so all forks COW from the same backing inode.
 //
 // Layout: forkDataDir is the fork's data dir. The snapshot dir is at
 // <forkDataDir>/snapshots/snapshot-latest, and the mem-file lives at
-// <snapshot dir>/memory. The symlink target is the source instance's
-// standby snapshot mem-file.
+// <snapshot dir>/memory. The hardlink shares the inode with the source
+// instance's standby snapshot mem-file.
+//
+// We use a hardlink rather than a symlink because RestoreVM temporarily
+// aliases the source data dir to the fork data dir while firecracker
+// loads the snapshot (see withSnapshotSourceDirAlias). A symlink whose
+// target traverses the source dir would resolve back into the fork dir
+// during that window and trip ELOOP; a hardlink resolves by inode so
+// the alias has no effect on it. Hardlinks require both paths on the
+// same filesystem, which holds for our standard data-dir layout.
 func (m *manager) installForkSharedMemFile(forkDataDir, sourceInstanceID string) error {
 	srcMem := filepath.Join(m.paths.InstanceSnapshotLatest(sourceInstanceID), templateSharedMemFileName)
 	if _, err := os.Stat(srcMem); err != nil {
@@ -31,8 +38,8 @@ func (m *manager) installForkSharedMemFile(forkDataDir, sourceInstanceID string)
 	}
 	dstMem := filepath.Join(dstSnapshotDir, templateSharedMemFileName)
 	_ = os.Remove(dstMem)
-	if err := os.Symlink(srcMem, dstMem); err != nil {
-		return fmt.Errorf("symlink shared mem-file: %w", err)
+	if err := os.Link(srcMem, dstMem); err != nil {
+		return fmt.Errorf("hardlink shared mem-file: %w", err)
 	}
 	return nil
 }

lib/instances/templates_shared_memfile_test.go

@@ -4,17 +4,29 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"syscall"
 	"testing"
 
 	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// TestInstallForkSharedMemFile_SymlinksSourceMemFile verifies that the helper
-// creates a symlink at the fork's snapshot mem-file path pointing back to the
-// source instance's mem-file.
-func TestInstallForkSharedMemFile_SymlinksSourceMemFile(t *testing.T) {
+func sameInode(t *testing.T, a, b string) bool {
+	t.Helper()
+	ai, err := os.Stat(a)
+	require.NoError(t, err)
+	bi, err := os.Stat(b)
+	require.NoError(t, err)
+	as := ai.Sys().(*syscall.Stat_t)
+	bs := bi.Sys().(*syscall.Stat_t)
+	return as.Ino == bs.Ino && as.Dev == bs.Dev
+}
+
+// TestInstallForkSharedMemFile_HardlinksSourceMemFile verifies that the helper
+// creates a hardlink at the fork's snapshot mem-file path that shares the
+// source instance's mem-file inode.
+func TestInstallForkSharedMemFile_HardlinksSourceMemFile(t *testing.T) {
 	t.Parallel()
 
 	mgr, _ := newStorageOnlyManager(t)
@@ -32,16 +44,13 @@ func TestInstallForkSharedMemFile_SymlinksSourceMemFile(t *testing.T) {
 	forkMem := filepath.Join(forkDir, "snapshots", "snapshot-latest", templateSharedMemFileName)
 	info, err := os.Lstat(forkMem)
 	require.NoError(t, err)
-	assert.NotZero(t, info.Mode()&os.ModeSymlink, "fork mem-file must be a symlink, not a regular file")
-
-	target, err := os.Readlink(forkMem)
-	require.NoError(t, err)
-	assert.Equal(t, srcMem, target)
+	assert.True(t, info.Mode().IsRegular(), "fork mem-file must be a regular file (hardlink), not a symlink")
+	assert.True(t, sameInode(t, srcMem, forkMem), "fork mem-file must share the source's inode")
 }
 
 // TestInstallForkSharedMemFile_ErrorsWhenSourceMissing makes sure the helper
-// refuses to silently create a dangling symlink when the source mem-file does
-// not exist.
+// refuses to silently create a dangling link when the source mem-file does not
+// exist.
 func TestInstallForkSharedMemFile_ErrorsWhenSourceMissing(t *testing.T) {
 	t.Parallel()
 
@@ -54,7 +63,7 @@ func TestInstallForkSharedMemFile_ErrorsWhenSourceMissing(t *testing.T) {
 
 // TestForkFirecrackerSharesMemFile_FromTemplate verifies the end-to-end fork
 // path: when the source is a Firecracker Template instance, the fork's
-// mem-file is a symlink to the source's mem-file instead of a copy. This
+// mem-file is a hardlink to the source's mem-file instead of a copy. This
 // preserves the firecracker MAP_PRIVATE COW semantics that let multiple forks
 // share the heavy backing file.
 func TestForkFirecrackerSharesMemFile_FromTemplate(t *testing.T) {
@@ -84,11 +93,8 @@ func TestForkFirecrackerSharesMemFile_FromTemplate(t *testing.T) {
 	forkMem := filepath.Join(mgr.paths.InstanceSnapshotLatest(forked.Id), templateSharedMemFileName)
 	info, err := os.Lstat(forkMem)
 	require.NoError(t, err)
-	assert.NotZero(t, info.Mode()&os.ModeSymlink, "fork mem-file must be a symlink for firecracker fan-out")
-
-	target, err := os.Readlink(forkMem)
-	require.NoError(t, err)
-	assert.Equal(t, srcMem, target)
+	assert.True(t, info.Mode().IsRegular(), "fork mem-file must be a regular file (hardlink) for firecracker fan-out")
+	assert.True(t, sameInode(t, srcMem, forkMem), "fork mem-file must share the source's inode")
 }
 
 // TestForkFirecrackerStandbySourceDoesNotShareMemFile guards the
@@ -121,7 +127,8 @@ func TestForkFirecrackerStandbySourceDoesNotShareMemFile(t *testing.T) {
 	forkMem := filepath.Join(mgr.paths.InstanceSnapshotLatest(forked.Id), templateSharedMemFileName)
 	info, err := os.Lstat(forkMem)
 	require.NoError(t, err)
-	assert.Zero(t, info.Mode()&os.ModeSymlink, "standby-source fork mem-file must be a regular file copy, not a symlink")
+	require.True(t, info.Mode().IsRegular(), "standby-source fork mem-file must be a regular file copy")
+	assert.False(t, sameInode(t, srcMem, forkMem), "standby-source fork mem-file must be a copy, not a hardlink to source")
 }
 
 // promoteFixtureToTemplate marks the source's stored metadata as a Template