metadata: set NoFreelistSync on bolt databases to avoid write amplification

bbolt rewrites the entire freelist on every transaction commit. Once a
metadata database accumulates many free pages (for example after GC frees
most of a large database), each small metadata write amplifies into a
multi-MB disk write. With a high commit rate this can saturate disk write
bandwidth and, because every commit holds bbolt's single writer lock while
fsyncing, stall otherwise-cheap build operations.

Set NoFreelistSync on the bolt databases that buildkit opens itself
(containerdmeta.db, metadata_v2.db and cache.db). The freelist is not data;
it is reconstructed by scanning the database on open, so durability is
unaffected. containerd applies the same option to its own metadata database
for the same reason: containerd/containerd#6761

Signed-off-by: ZRHann <zrhann@foxmail.com>

Author: ZRHann

cache/metadata/metadata.go

@@ -41,7 +41,10 @@ func NewStore(dbPath string) (*Store, error) {
 			)
 		}
 	}
-	db, err := boltutil.Open(dbPath, 0600, nil)
+	// Disable freelist syncing to avoid rewriting the whole freelist on every
+	// commit (write amplification on a fragmented database). See the comment in
+	// worker/runc and https://github.com/containerd/containerd/pull/6761
+	db, err := boltutil.Open(dbPath, 0600, &bolt.Options{NoFreelistSync: true})
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to open database file %s", dbPath)
 	}

solver/bboltcachestorage/storage.go

@@ -27,6 +27,9 @@ type Store struct {
 func NewStore(dbPath string) (*Store, error) {
 	db, err := boltutil.SafeOpen(dbPath, 0600, &bolt.Options{
 		NoSync: true,
+		// Avoid rewriting the whole freelist on every commit (write
+		// amplification). See https://github.com/containerd/containerd/pull/6761
+		NoFreelistSync: true,
 	})
 	if err != nil {
 		return nil, err

worker/runc/runc.go

@@ -95,7 +95,14 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc
 		return opt, err
 	}
 
-	db, err := bolt.Open(filepath.Join(root, "containerdmeta.db"), 0644, nil)
+	// Disable freelist syncing. bbolt rewrites the whole freelist on every
+	// commit, so once the database accumulates many free pages a single small
+	// metadata write amplifies into a multi-MB write, which can saturate disk
+	// write bandwidth. containerd disables it on its own metadata database for
+	// the same reason: https://github.com/containerd/containerd/pull/6761
+	db, err := bolt.Open(filepath.Join(root, "containerdmeta.db"), 0644, &bolt.Options{
+		NoFreelistSync: true,
+	})
 	if err != nil {
 		return opt, err
 	}