Pre-allocate the output file on Darwin to avoid APFS sparse-file issues (#361)

AssembleFile truncated the output file to its target size, which on
APFS creates a sparse file. Concurrent WriteAt calls on adjacent
regions of large sparse files have shown non-deterministic corruption
there, with null-chunk regions occasionally reading back wrong
(#326).

Replace the plain truncate with preallocateFile: on Darwin it uses
fcntl(F_PREALLOCATE) to physically allocate blocks before truncating.
Since F_PREALLOCATE with F_PEOFPOSMODE allocates relative to the
current end of file, only the missing difference is requested,
keeping in-place reassembly of existing files from over-allocating.
Filesystems without F_PREALLOCATE support (SMB, FUSE) fall back to
the previous plain-truncate behavior. All other platforms delegate
to Truncate as before.

Author: folbricht

assemble.go

@@ -128,9 +128,11 @@ func AssembleFile(ctx context.Context, name string, idx Index, s Store, seeds []
 
 	// Truncate the output file to the full expected size. Not only does this
 	// confirm there's enough disk space, but it allows for an optimization
-	// when dealing with the Null Chunk
+	// when dealing with the Null Chunk. On Darwin, the file is physically
+	// pre-allocated as well since sparse files on APFS have shown to cause
+	// issues when written to concurrently.
 	if !isBlkDevice {
-		if err := os.Truncate(name, idx.Length()); err != nil {
+		if err := preallocateFile(name, idx.Length()); err != nil {
 			return stats, err
 		}
 	}

preallocate_darwin.go

@@ -0,0 +1,51 @@
+//go:build darwin
+
+package desync
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// preallocateFile physically allocates disk blocks and sets the file size.
+// On APFS, a plain Truncate creates sparse holes. When concurrent workers
+// call WriteAt on adjacent regions, copy-on-write of sparse blocks can
+// cause non-deterministic data corruption. Pre-allocating real blocks
+// avoids this.
+func preallocateFile(name string, size int64) error {
+	f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	// F_PREALLOCATE with F_PEOFPOSMODE allocates relative to the current
+	// end of file, so only request the difference. Nothing to allocate if
+	// the file is already large enough or no growth is needed.
+	if extra := size - info.Size(); extra > 0 {
+		store := unix.Fstore_t{
+			Flags:   unix.F_ALLOCATEALL,
+			Posmode: unix.F_PEOFPOSMODE,
+			Offset:  0,
+			Length:  extra,
+		}
+		if err := unix.FcntlFstore(f.Fd(), unix.F_PREALLOCATE, &store); err != nil {
+			// Not all filesystems support F_PREALLOCATE (e.g. SMB or FUSE
+			// mounts). The sparse-hole issue is specific to APFS, so fall
+			// back to a plain truncate there.
+			if !errors.Is(err, unix.ENOTSUP) {
+				return fmt.Errorf("F_PREALLOCATE %s: %w", name, err)
+			}
+		}
+	}
+
+	return f.Truncate(size)
+}

preallocate_darwin_test.go

@@ -0,0 +1,48 @@
+//go:build darwin
+
+package desync
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// TestPreallocateTightDisk re-preallocates an existing file to its current
+// size on a nearly-full volume. This must not require additional disk
+// space: F_PREALLOCATE with F_PEOFPOSMODE allocates relative to the
+// existing end of file, so requesting the full size instead of only the
+// missing difference over-allocates and fails with ENOSPC.
+func TestPreallocateTightDisk(t *testing.T) {
+	if _, err := exec.LookPath("hdiutil"); err != nil {
+		t.Skip("hdiutil not available")
+	}
+	dir := t.TempDir()
+	img := filepath.Join(dir, "small.dmg")
+	mount := filepath.Join(dir, "mnt")
+	require.NoError(t, os.Mkdir(mount, 0755))
+
+	out, err := exec.Command("hdiutil", "create", "-size", "32m", "-fs", "APFS", "-volname", "desync-test", img).CombinedOutput()
+	require.NoError(t, err, string(out))
+	out, err = exec.Command("hdiutil", "attach", img, "-mountpoint", mount, "-nobrowse").CombinedOutput()
+	require.NoError(t, err, string(out))
+	t.Cleanup(func() { _ = exec.Command("hdiutil", "detach", mount, "-force").Run() })
+
+	var st unix.Statfs_t
+	require.NoError(t, unix.Statfs(mount, &st))
+	free := int64(st.Bavail) * int64(st.Bsize)
+
+	// Fill most of the volume with an existing file, leaving less free
+	// space than the size of the file itself
+	size := free * 2 / 3
+	name := filepath.Join(mount, "existing")
+	require.NoError(t, os.WriteFile(name, bytes.Repeat([]byte{0xab}, int(size)), 0666))
+
+	// The file already has the right size, nothing should be allocated
+	require.NoError(t, preallocateFile(name, size))
+}

preallocate_other.go

@@ -0,0 +1,18 @@
+//go:build !darwin
+
+package desync
+
+import "os"
+
+// preallocateFile truncates the file to the given size, creating it if
+// it doesn't exist. On Linux (ext4) and other platforms, Truncate
+// produces a file that reads back as zeros without sparse-hole issues,
+// so no special preallocation is needed.
+func preallocateFile(name string, size int64) error {
+	f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return f.Truncate(size)
+}

preallocate_test.go

@@ -0,0 +1,70 @@
+package desync
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPreallocateNewFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "new")
+
+	require.NoError(t, preallocateFile(name, 2*1024*1024))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Len(t, b, 2*1024*1024)
+	require.Equal(t, make([]byte, 2*1024*1024), b)
+}
+
+func TestPreallocateGrowExistingFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "grow")
+	data := bytes.Repeat([]byte{0xab}, 4096)
+	require.NoError(t, os.WriteFile(name, data, 0666))
+
+	require.NoError(t, preallocateFile(name, 64*1024))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Len(t, b, 64*1024)
+	// The original content must be preserved and the new region read as zeros
+	require.Equal(t, data, b[:len(data)])
+	require.Equal(t, make([]byte, 64*1024-len(data)), b[len(data):])
+}
+
+func TestPreallocateShrinkExistingFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "shrink")
+	data := bytes.Repeat([]byte{0xab}, 64*1024)
+	require.NoError(t, os.WriteFile(name, data, 0666))
+
+	require.NoError(t, preallocateFile(name, 4096))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Equal(t, data[:4096], b)
+}
+
+func TestPreallocateSameSize(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "same")
+	data := bytes.Repeat([]byte{0xab}, 4096)
+	require.NoError(t, os.WriteFile(name, data, 0666))
+
+	require.NoError(t, preallocateFile(name, int64(len(data))))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Equal(t, data, b)
+}
+
+func TestPreallocateEmptyFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "empty")
+
+	require.NoError(t, preallocateFile(name, 0))
+
+	info, err := os.Stat(name)
+	require.NoError(t, err)
+	require.Zero(t, info.Size())
+}