Instantiate measurement and logging during optimization, but take no action. Update README.

Author: StephenHinck

README.md

@@ -55,3 +55,23 @@ export CONNECTION_STRING="neo4j://neo4j:weneedbetterpasswords@localhost:7687"
 ```
 
 Use `make test` for unit tests only and `make test_integration` for integration tests only.
+
+## PostgreSQL Extensions
+
+The PostgreSQL driver's schema bootstrap (`drivers/pg/query/sql/schema_up.sql`) installs the following extensions:
+
+| Extension     | Required | Purpose                                                                                                  |
+|---------------|----------|----------------------------------------------------------------------------------------------------------|
+| `pg_trgm`     | yes      | GIN trigram indexes for `contains` / `starts with` / `ends with` lookups on graph entity properties.     |
+| `intarray`    | yes      | Extended integer array operations used when maintaining node kind arrays.                                |
+| `pgstattuple` | no       | Measures btree leaf density and fragmentation for the driver-managed index optimization assessment.      |
+
+`pgstattuple` is treated as best-effort. Installing it typically requires a superuser role and on some managed
+Postgres deployments the contrib package is not exposed at all. Bootstrap wraps the install in an exception
+handler that downgrades any failure to a `WARNING`, and the driver's `Optimize` implementation re-checks for the
+extension at runtime and logs a warning when it is missing rather than failing the caller. To enable index
+optimization in such environments, have an administrator install the extension out-of-band:
+
+```sql
+create extension if not exists pgstattuple;
+```

drivers/pg/optimize.go

@@ -1,11 +1,162 @@
 package pg
 
-import "context"
+import (
+	"context"
+	"fmt"
+	"log/slog"
+)
 
-// Optimize satisfies the graph.Optimizer interface. The body is currently a
-// no-op stub; the actual index maintenance logic (assessment via pgstattuple
-// followed by REINDEX CONCURRENTLY of bloated indexes) is implemented in a
-// later phase.
+const (
+	// bloatedIndexLeafDensityThreshold is the average leaf-page fill percentage
+	// below which a btree index is considered dense enough that REINDEX would
+	// reclaim significant space. Calibrated against seven production tenant
+	// samples whose live indexes regularly fell to 21-55%, against a freshly
+	// rebuilt baseline (edge_1_pkey on a recently restored tenant) of 73.8%.
+	bloatedIndexLeafDensityThreshold = 60.0
+
+	// highIndexFragmentationThreshold is the leaf-page fragmentation percentage
+	// at or above which a btree index has accumulated enough out-of-order page
+	// splits to warrant a rebuild even when leaf density alone has not crossed
+	// its threshold.
+	highIndexFragmentationThreshold = 40.0
+)
+
+const (
+	sqlPgstattupleInstalled = `select exists(select 1 from pg_extension where extname = 'pgstattuple')`
+
+	sqlSelectGraphPartitionBtreeIndexes = `
+		select i.oid                  as index_oid,
+		       n.nspname              as schema_name,
+		       c.relname              as table_name,
+		       i.relname              as index_name,
+		       pg_relation_size(i.oid) as index_size_bytes
+		from pg_inherits inh
+		         join pg_class p on p.oid = inh.inhparent
+		         join pg_class c on c.oid = inh.inhrelid
+		         join pg_namespace n on n.oid = c.relnamespace
+		         join pg_index x on x.indrelid = c.oid
+		         join pg_class i on i.oid = x.indexrelid
+		         join pg_am a on a.oid = i.relam
+		where p.relname in ('node', 'edge')
+		  and p.relnamespace = n.oid
+		  and a.amname = 'btree'
+		order by c.relname, i.relname`
+
+	sqlSelectIndexBloatMetrics = `select avg_leaf_density, leaf_fragmentation from pgstatindex($1::regclass)`
+)
+
+// indexRow is a candidate index discovered by the listing query, prior to
+// per-index pgstatindex assessment.
+type indexRow struct {
+	oid       uint32
+	schema    string
+	table     string
+	index     string
+	sizeBytes int64
+}
+
+// indexCandidate is an index whose pgstatindex measurement caused it to be
+// flagged for rebuild by needsReindex.
+type indexCandidate struct {
+	indexRow
+	leafDensity   float64
+	fragmentation float64
+	reason        string
+}
+
+// Optimize satisfies the graph.Optimizer interface. The current phase performs
+// a read-only assessment: it identifies btree indexes on partitions of the
+// node and edge tables whose leaf density or fragmentation cross the rebuild
+// thresholds and logs the candidates. No DDL is executed in this phase.
 func (s *Driver) Optimize(ctx context.Context) error {
+	if installed, err := s.pgstattupleInstalled(ctx); err != nil {
+		return fmt.Errorf("checking pgstattuple extension: %w", err)
+	} else if !installed {
+		slog.WarnContext(ctx, "Index optimization skipped: pgstattuple extension is not installed; verify the DAWGS schema bootstrap completed successfully")
+		return nil
+	}
+
+	indexes, err := s.listGraphPartitionBtreeIndexes(ctx)
+	if err != nil {
+		return fmt.Errorf("listing graph partition btree indexes: %w", err)
+	}
+
+	slog.InfoContext(ctx, fmt.Sprintf("Index optimization assessment starting: %d btree index(es) under consideration", len(indexes)))
+
+	var (
+		candidates []indexCandidate
+		totalBytes int64
+	)
+	for _, idx := range indexes {
+		density, fragmentation, err := s.measureIndexBloat(ctx, idx.oid)
+		if err != nil {
+			slog.WarnContext(ctx, fmt.Sprintf("Skipping bloat assessment for index %s.%s: %v", idx.schema, idx.index, err))
+			continue
+		}
+
+		candidate := indexCandidate{indexRow: idx, leafDensity: density, fragmentation: fragmentation}
+		if reason, flagged := needsReindex(candidate); flagged {
+			candidate.reason = reason
+			candidates = append(candidates, candidate)
+			totalBytes += candidate.sizeBytes
+			slog.InfoContext(ctx, fmt.Sprintf(
+				"Index optimization candidate: %s.%s on %s (size=%d bytes, leaf_density=%.1f%%, fragmentation=%.1f%%, reason=%s)",
+				candidate.schema, candidate.index, candidate.table,
+				candidate.sizeBytes, candidate.leafDensity, candidate.fragmentation, candidate.reason,
+			))
+		}
+	}
+
+	slog.InfoContext(ctx, fmt.Sprintf(
+		"Index optimization assessment complete: %d candidate(s) totaling %d bytes",
+		len(candidates), totalBytes,
+	))
 	return nil
 }
+
+// needsReindex applies the rebuild thresholds to a measured index and returns
+// a short human-readable reason when the index is flagged. Pure function;
+// unit-tested in optimize_test.go.
+func needsReindex(c indexCandidate) (string, bool) {
+	if c.leafDensity < bloatedIndexLeafDensityThreshold {
+		return fmt.Sprintf("leaf density %.1f%% below %.1f%% threshold", c.leafDensity, bloatedIndexLeafDensityThreshold), true
+	}
+	if c.fragmentation >= highIndexFragmentationThreshold {
+		return fmt.Sprintf("fragmentation %.1f%% at or above %.1f%% threshold", c.fragmentation, highIndexFragmentationThreshold), true
+	}
+	return "", false
+}
+
+func (s *Driver) pgstattupleInstalled(ctx context.Context) (bool, error) {
+	var installed bool
+	if err := s.pool.QueryRow(ctx, sqlPgstattupleInstalled).Scan(&installed); err != nil {
+		return false, err
+	}
+	return installed, nil
+}
+
+func (s *Driver) listGraphPartitionBtreeIndexes(ctx context.Context) ([]indexRow, error) {
+	rows, err := s.pool.Query(ctx, sqlSelectGraphPartitionBtreeIndexes)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var out []indexRow
+	for rows.Next() {
+		var r indexRow
+		if err := rows.Scan(&r.oid, &r.schema, &r.table, &r.index, &r.sizeBytes); err != nil {
+			return nil, err
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
+func (s *Driver) measureIndexBloat(ctx context.Context, indexOID uint32) (float64, float64, error) {
+	var density, fragmentation float64
+	if err := s.pool.QueryRow(ctx, sqlSelectIndexBloatMetrics, indexOID).Scan(&density, &fragmentation); err != nil {
+		return 0, 0, err
+	}
+	return density, fragmentation, nil
+}

drivers/pg/optimize_test.go

@@ -1,17 +1,113 @@
 package pg
 
 import (
-	"context"
+	"strings"
 	"testing"
 
 	"github.com/specterops/dawgs/graph"
-	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/assert"
 )
 
 // Compile-time assertion that *Driver implements graph.Optimizer.
 var _ graph.Optimizer = (*Driver)(nil)
 
-func TestDriver_Optimize_NoopReturnsNil(t *testing.T) {
-	d := &Driver{}
-	require.NoError(t, d.Optimize(context.Background()))
+// TestNeedsReindex exercises the threshold logic that decides whether a
+// measured index is flagged as a rebuild candidate. The function is pure;
+// integration coverage of the surrounding pg_extension / pg_inherits /
+// pgstatindex queries lands with the Phase 5 REINDEX work.
+func TestNeedsReindex(t *testing.T) {
+	cases := []struct {
+		name          string
+		leafDensity   float64
+		fragmentation float64
+		wantFlagged   bool
+		wantReasonHas string
+	}{
+		{
+			name:          "healthy index is not flagged",
+			leafDensity:   85.0,
+			fragmentation: 5.0,
+			wantFlagged:   false,
+		},
+		{
+			name:          "freshly built baseline (73.8%/low frag) is not flagged",
+			leafDensity:   73.8,
+			fragmentation: 2.5,
+			wantFlagged:   false,
+		},
+		{
+			name:          "leaf density exactly at threshold is not flagged",
+			leafDensity:   bloatedIndexLeafDensityThreshold,
+			fragmentation: 0,
+			wantFlagged:   false,
+		},
+		{
+			name:          "leaf density just below threshold is flagged",
+			leafDensity:   bloatedIndexLeafDensityThreshold - 0.01,
+			fragmentation: 0,
+			wantFlagged:   true,
+			wantReasonHas: "leaf density",
+		},
+		{
+			name:          "deeply bloated production sample (21%) is flagged on density",
+			leafDensity:   21.0,
+			fragmentation: 8.0,
+			wantFlagged:   true,
+			wantReasonHas: "leaf density",
+		},
+		{
+			name:          "fragmentation just below threshold is not flagged when density is healthy",
+			leafDensity:   85.0,
+			fragmentation: highIndexFragmentationThreshold - 0.01,
+			wantFlagged:   false,
+		},
+		{
+			name:          "fragmentation exactly at threshold is flagged",
+			leafDensity:   85.0,
+			fragmentation: highIndexFragmentationThreshold,
+			wantFlagged:   true,
+			wantReasonHas: "fragmentation",
+		},
+		{
+			name:          "fragmentation flag triggers when density alone would not",
+			leafDensity:   65.0,
+			fragmentation: 50.0,
+			wantFlagged:   true,
+			wantReasonHas: "fragmentation",
+		},
+		{
+			name:          "density takes precedence in the reason when both cross",
+			leafDensity:   30.0,
+			fragmentation: 60.0,
+			wantFlagged:   true,
+			wantReasonHas: "leaf density",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			candidate := indexCandidate{
+				leafDensity:   tc.leafDensity,
+				fragmentation: tc.fragmentation,
+			}
+			reason, flagged := needsReindex(candidate)
+			assert.Equal(t, tc.wantFlagged, flagged, "unexpected flagged result for %s", tc.name)
+			if tc.wantFlagged {
+				assert.True(t, strings.Contains(reason, tc.wantReasonHas),
+					"reason %q does not mention expected substring %q", reason, tc.wantReasonHas)
+			} else {
+				assert.Empty(t, reason, "expected empty reason when not flagged")
+			}
+		})
+	}
+}
+
+// TestThresholdsAreOrdered guards against accidental reordering of the
+// calibrated thresholds. If these inequalities ever fail the calibration
+// rationale documented on the constants must be revisited.
+func TestThresholdsAreOrdered(t *testing.T) {
+	assert.Greater(t, bloatedIndexLeafDensityThreshold, 0.0, "leaf density threshold must be positive")
+	assert.Less(t, bloatedIndexLeafDensityThreshold, 100.0, "leaf density threshold must be a percentage")
+	assert.Greater(t, highIndexFragmentationThreshold, 0.0, "fragmentation threshold must be positive")
+	assert.Less(t, highIndexFragmentationThreshold, 100.0, "fragmentation threshold must be a percentage")
 }

drivers/pg/query/sql/schema_up.sql

@@ -22,9 +22,19 @@ create extension if not exists pg_trgm;
 create extension if not exists intarray;
 
 -- We need the pgstattuple extension to measure btree leaf density and fragmentation on node and edge indexes
--- during driver-managed index optimization. If this extension cannot be installed (e.g. a managed Postgres that
--- does not expose it), the optimizer logs a warning and skips the assessment rather than failing.
-create extension if not exists pgstattuple;
+-- during driver-managed index optimization. Installing pgstattuple typically requires a superuser role, and on
+-- locked-down managed Postgres deployments (e.g. some RDS or Cloud SQL configurations) the contrib package may
+-- not be exposed at all. We wrap the install in a DO block that downgrades any failure to a WARNING so schema
+-- bootstrap continues to succeed; the optimizer performs its own runtime extension check and will skip the
+-- assessment with a logged warning when pgstattuple is unavailable.
+do $$
+begin
+    create extension if not exists pgstattuple;
+exception
+    when others then
+        raise warning 'pgstattuple extension could not be installed (%); index optimization will be skipped at runtime', sqlerrm;
+end
+$$;
 
 -- This is an optional but useful extension for validating performance of queries
 -- create extension if not exists pg_stat_statements;