actually port most tests

Author: zachmu

integration-tests/go-sql-server-driver/full_gc_no_conjoin_test.go

@@ -1,4 +1,4 @@
-// Copyright 2024 Dolthub, Inc.
+// Copyright 2026 Dolthub, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,16 +14,227 @@
 
 package main
 
-import "testing"
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
 
-// TestFullGCNoOldgenConjoin is ported from Dolt's full_gc_no_conjoin_test.go.
-// It runs a full GC and asserts that the oldgen table files are not conjoined,
-// by inspecting on-disk storage (it imports dolt's store/hash package). The
-// doltgres test driver cannot import Dolt storage internals, so this is
-// skipped.
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
+)
+
+// This test asserts that running a full gc into the old gen at the
+// point that it is at its conjoin limit will not generate a conjoin
+// in the middle of the GC.
+//
+// It first generates empty commits and runs GC in a loop. After every
+// GC, it checks how many table files are in the oldgen. When it sees
+// them get conjoined, it assumes max table files is the observed high
+// water mark.
+//
+// It then creates exactly that many table files in oldgen by creating
+// empty commits and running gc however many times it needs to.
+//
+// Then it needs to create one more commit and run a
+// dolt_gc(--full). Currently the sql-server logs whenever it begins
+// a conjoin, and that determination is made synchronously on the
+// write path. So, no real attempt is made to get a theoretically
+// started conjoin to win the race against the newgen collection, for
+// example.
+//
+// As described above, this test is tightly coupled with
+// GenerationalNBS, NomsBlockStore and the file persisters, with
+// conjoin strategy behavior. Doltgres uses the same Dolt storage
+// engine, so the conjoin log lines (pkg=store.noms) and the on-disk
+// oldgen table-file layout are identical.
+//
+// At the end of the test, there should be one file in the oldgen, the
+// results of the --full. There should be exactly one "beginning
+// conjoin of database" in the server logs and it should correspond to
+// when we measured the high water mark. After shutting down the
+// server should happily start again.
 //
-// The full test logic lives in the Dolt source at
-// integration-tests/go-sql-server-driver/full_gc_no_conjoin_test.go.
+// NOTE: a successful online dolt_gc() invalidates every open connection
+// to the doltgres server, so the helpers below open a fresh connection
+// for each commit/gc iteration (db.SetMaxIdleConns(0) ensures each
+// connection is a new one rather than a pooled, now-invalid connection).
 func TestFullGCNoOldgenConjoin(t *testing.T) {
-	t.Skip("depends on Dolt store/hash storage internals not accessible from the doltgres test driver")
+	t.Parallel()
+	ports := newPorts(t)
+	u, err := driver.NewDoltUser()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		u.Cleanup()
+	})
+
+	dbname := "full_gc_no_oldgen_conjoin_test"
+
+	rs, err := u.MakeRepoStore()
+	require.NoError(t, err)
+	repo, err := rs.MakeRepo(dbname)
+	require.NoError(t, err)
+	srvSettings := &driver.Server{
+		Args:        []string{},
+		DynamicPort: "server",
+	}
+	var conjoinStarted, conjoinFinished atomic.Bool
+	upstreamLenOnConjoin := 0
+	server := MakeServer(t, rs, rs.Dir, srvSettings, ports, driver.WithOutputVisitor(func(out string) {
+		// Matching a line like:
+		// time="..." level=info msg="beginning conjoin of database" database=full_gc_no_oldgen_conjoin_test generation=old pkg=store.noms upstream_len=257
+		// Extracting its upstream_len to see exactly when the conjoin was triggered.
+		if upstreamLenOnConjoin <= 0 && strings.Contains(out, "beginning conjoin of database") {
+			if i := strings.Index(out, "upstream_len="); i != -1 {
+				i += len("upstream_len=")
+				if _, err := fmt.Sscanf(out[i:], "%d", &upstreamLenOnConjoin); err != nil {
+					upstreamLenOnConjoin = -1
+				}
+			}
+			conjoinStarted.Store(true)
+		}
+		// Matching the line:
+		// time="..." level=info msg="conjoin completed successfully" database=full_gc_no_oldgen_conjoin_test generation=old new_upstream_len=2 pkg=store.noms
+		// So we can block on further operations until conjoin is done.
+		if strings.Contains(out, "conjoin completed successfully") {
+			conjoinFinished.Store(true)
+		}
+	}))
+	server.DBName = dbname
+
+	oldgendir := filepath.Join(repo.Dir, "/.dolt/noms/oldgen")
+
+	CommitAndGCUntilConjoin(t, server, &conjoinStarted, oldgendir)
+	require.Greater(t, upstreamLenOnConjoin, 0)
+	require.Eventually(t, func() bool {
+		return conjoinFinished.Load()
+	}, 5*time.Second, 32*time.Millisecond)
+
+	CreateUpToNumFiles(t, server, oldgendir, upstreamLenOnConjoin)
+	cnt := CountTableFiles(t, oldgendir)
+	t.Logf("now there are %d", cnt)
+
+	RunGCFull(t, server, oldgendir)
+
+	require.NoError(t, server.GracefulStop())
+	output := server.Output.String()
+	assert.Equal(t, 1, strings.Count(output, "beginning conjoin of database"))
+	// The line for triggering a conjoin on policy but not proceeding because
+	// conjoin is dynamically disabled looks like:
+	// time="..." level=info msg="conjoin dynamically disabled. not conjoining." database=full_gc_no_oldgen_conjoin_test generation=old pkg=store.noms
+	assert.Equal(t, 1, strings.Count(output, "conjoin dynamically disabled"))
+	cnt = CountTableFiles(t, oldgendir)
+	assert.Equal(t, 1, cnt)
+
+	newServer := MakeServer(t, rs, rs.Dir, srvSettings, ports)
+	newServer.DBName = dbname
+	db, err := newServer.DB(driver.Connection{})
+	require.NoError(t, err)
+	defer db.Close()
+	require.NoError(t, db.PingContext(t.Context()))
+}
+
+// isTableFileName reports whether n is the name of a Dolt/Noms table file: a
+// 32-character string in Noms' base32 hash alphabet. This mirrors
+// hash.MaybeParse from Dolt's store/hash package without importing it.
+func isTableFileName(n string) bool {
+	const alphabet = "0123456789abcdefghijklmnopqrstuv"
+	if len(n) != 32 {
+		return false
+	}
+	for _, c := range n {
+		if !strings.ContainsRune(alphabet, c) {
+			return false
+		}
+	}
+	return true
+}
+
+func CountTableFiles(t *testing.T, dir string) int {
+	var count int
+	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		n := d.Name()
+		// Archive table files carry a .darc suffix.
+		n = strings.TrimSuffix(n, ".darc")
+		if isTableFileName(n) {
+			count += 1
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	return count
+}
+
+func CommitAndGCUntilConjoin(t *testing.T, srv *driver.SqlServer, conjoinStarted *atomic.Bool, path string) {
+	ctx := t.Context()
+	db, err := srv.DB(driver.Connection{})
+	require.NoError(t, err)
+	defer db.Close()
+	// Ensure each db.Conn() hands out a new connection rather than a pooled
+	// one that a prior online GC has invalidated.
+	db.SetMaxIdleConns(0)
+
+	for {
+		func() {
+			conn, err := db.Conn(ctx)
+			require.NoError(t, err)
+			defer conn.Close()
+			_, err = conn.ExecContext(ctx, "SELECT DOLT_COMMIT('-A', '--allow-empty', '-m', 'creating a new commit')")
+			require.NoError(t, err)
+			_, err = conn.ExecContext(ctx, "SELECT DOLT_GC()")
+			require.NoError(t, err)
+		}()
+		if conjoinStarted.Load() {
+			return
+		}
+	}
+}
+
+func CreateUpToNumFiles(t *testing.T, srv *driver.SqlServer, path string, numFiles int) {
+	ctx := t.Context()
+	db, err := srv.DB(driver.Connection{})
+	require.NoError(t, err)
+	defer db.Close()
+	db.SetMaxIdleConns(0)
+
+	for {
+		cnt := CountTableFiles(t, path)
+		if cnt == numFiles {
+			return
+		}
+		func() {
+			conn, err := db.Conn(ctx)
+			require.NoError(t, err)
+			defer conn.Close()
+			_, err = conn.ExecContext(ctx, "SELECT DOLT_COMMIT('-A', '--allow-empty', '-m', 'creating a new commit')")
+			require.NoError(t, err)
+			_, err = conn.ExecContext(ctx, "SELECT DOLT_GC()")
+			require.NoError(t, err)
+		}()
+	}
+}
+
+func RunGCFull(t *testing.T, srv *driver.SqlServer, path string) {
+	ctx := t.Context()
+	db, err := srv.DB(driver.Connection{})
+	require.NoError(t, err)
+	defer db.Close()
+	db.SetMaxIdleConns(0)
+
+	conn, err := db.Conn(ctx)
+	require.NoError(t, err)
+	defer conn.Close()
+	_ = CountTableFiles(t, path)
+	_, err = conn.ExecContext(ctx, "SELECT DOLT_GC('--full')")
+	require.NoError(t, err)
+	cnt := CountTableFiles(t, path)
+	require.Equal(t, 1, cnt)
 }