Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 36 additions & 16 deletions apps/cli-go/internal/db/declarative/declarative.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,13 @@ const (
// pgDeltaTempDir namespaces pg-delta artifacts under .temp to make ownership
// and cleanup intent explicit.
pgDeltaTempDir = "pgdelta"
// baselineCatalogName caches the catalog of an empty shadow database.
// baselineCatalogName caches the catalog of a shadow database with the Supabase
// platform baseline (auth/storage/realtime) provisioned but no user migrations
// applied — equivalent to diff.MigrateShadowDatabase with zero migrations.
//
// It is used as the "source" baseline when generating declarative files from
// a real database target.
// It is used as the "source" baseline both when generating declarative files
// from a real database target and when syncing with no local migrations, so it
// must stay in parity with the declarative target's platform baseline.
baselineCatalogName = "catalog-baseline-%s.json"
// declarativeCatalogName stores catalogs keyed by declarative-content hash.
declarativeCatalogName = "catalog-%s-declarative-%s-%d.json"
Expand All @@ -58,11 +61,19 @@ var (
exportCatalog = diff.ExportCatalogPgDelta
applyDeclarative = pgdelta.ApplyDeclarative
declarativeExportRef = diff.DeclarativeExportPgDeltaRef
// diffPgDeltaRef diffs a source catalog against a target catalog. It is a
// package var so tests can exercise the full generate -> sync flow without the
// real pg-delta runtime.
diffPgDeltaRef = diff.DiffPgDeltaRef
// setupShadowDatabase provisions the Supabase platform baseline (auth/storage/
// realtime) on a shadow database before declarative schemas are applied, so
// Supabase-managed dependencies (auth.sessions, auth.jwt(), ...) resolve. It is
// a package var so tests can inject a no-op without a real shadow database.
setupShadowDatabase = diff.SetupShadowDatabase
// createShadow provisions a healthy shadow database container. It is a package
// var so tests can exercise the baseline/migrations/declarative paths without a
// real Docker daemon.
createShadow = createShadowContainer
// generateBaselineCatalogRefResolver allows Generate to reuse a freshly
// provisioned baseline shadow for declarative cache warmup.
generateBaselineCatalogRefResolver = getGenerateBaselineCatalogRef
Expand Down Expand Up @@ -123,13 +134,9 @@ func Generate(ctx context.Context, schema []string, config pgconn.Config, overwr
// can reuse it without provisioning another shadow database.
if !noCache {
if baseline.shadow != nil {
// The baseline catalog was already exported from the empty image
// baseline above. Set up the platform baseline on the reused shadow
// before applying declarative schemas so Supabase-managed dependencies
// (auth.sessions, auth.jwt(), ...) resolve during cache warmup.
if err := setupShadowDatabase(ctx, baseline.shadow.container, fsys, options...); err != nil {
return err
}
// The reused baseline shadow already has the platform baseline
// provisioned (getGenerateBaselineCatalogRef), so apply declarative
// schemas directly on top of it without setting it up again.
hash, err := hashDeclarativeSchemas(fsys)
Comment thread
avallete marked this conversation as resolved.
Outdated
if err != nil {
return err
Expand Down Expand Up @@ -172,7 +179,7 @@ func DiffDeclarativeToMigrations(ctx context.Context, schema []string, noCache b
if err != nil {
return nil, err
}
out, err := diff.DiffPgDeltaRef(ctx, sourceRef, targetRef, schema, pgDeltaFormatOptions(), options...)
out, err := diffPgDeltaRef(ctx, sourceRef, targetRef, schema, pgDeltaFormatOptions(), options...)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -308,6 +315,18 @@ func getGenerateBaselineCatalogRef(ctx context.Context, noCache bool, fsys afero
container: shadowID,
config: config,
}
// Provision the Supabase platform baseline before exporting so the baseline
// catalog represents "platform baseline, no user migrations" — the same
// semantics as diff.MigrateShadowDatabase with zero migrations. This baseline is
Comment thread
avallete marked this conversation as resolved.
// reused as the diff source by both Generate (against the live database) and
// sync-with-no-migrations (getMigrationsCatalogRef). Its starting point must
// match the declarative target, which also sets up the platform baseline;
// otherwise platform objects (auth/storage/realtime) surface as spurious
// additions in generated migrations.
if err := setupShadowDatabase(ctx, shadow.container, fsys, options...); err != nil {
Comment thread
avallete marked this conversation as resolved.
Comment thread
avallete marked this conversation as resolved.
shadow.cleanup()
return generateBaselineCatalogRef{}, err
}
snapshot, err := exportCatalog(ctx, utils.ToPostgresURL(config), "postgres", options...)
if err != nil {
shadow.cleanup()
Expand Down Expand Up @@ -345,8 +364,9 @@ func getMigrationsCatalogRef(ctx context.Context, noCache bool, fsys afero.Fs, p
if err != nil {
return "", err
}
// For sync with no local migrations, reuse an existing baseline
// snapshot instead of provisioning a fresh shadow database.
// For sync with no local migrations, the migrations catalog is just the
// platform baseline. Reuse an existing baseline snapshot (which now also
// represents the platform baseline) instead of provisioning a fresh shadow.
if !noCache && len(migrations) == 0 {
baselinePath := filepath.Join(pgDeltaTempPath(), fmt.Sprintf(baselineCatalogName, baselineVersionToken()))
if ok, err := afero.Exists(fsys, baselinePath); err != nil {
Expand Down Expand Up @@ -439,9 +459,9 @@ func writeDeclarativeCatalogFromConfig(ctx context.Context, config pgconn.Config
return path, nil
}

// createShadow provisions and health-checks the temporary Postgres container
// used by declarative conversion and diff operations.
func createShadow(ctx context.Context) (string, pgconn.Config, error) {
// createShadowContainer provisions and health-checks the temporary Postgres
// container used by declarative conversion and diff operations.
func createShadowContainer(ctx context.Context) (string, pgconn.Config, error) {
fmt.Fprintln(os.Stderr, "Creating shadow database...")
shadow, err := diff.CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
if err != nil {
Expand Down
166 changes: 166 additions & 0 deletions apps/cli-go/internal/db/declarative/declarative_flow_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
package declarative

import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"sort"
"strings"
"testing"

"github.com/jackc/pgconn"
"github.com/jackc/pgx/v4"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/supabase/cli/internal/db/diff"
"github.com/supabase/cli/internal/utils"
"github.com/supabase/cli/pkg/config"
)

// catalogObjects models a pg-delta catalog snapshot as the set of object names
// present in a shadow database, so the full generate -> sync flow can be
// exercised without the real pg-delta runtime while still proving that platform
// objects cancel out of the generated diff.
type catalogObjects struct {
Objects []string `json:"objects"`
}

func marshalCatalog(objects []string) string {
sorted := append([]string(nil), objects...)
sort.Strings(sorted)
out, _ := json.Marshal(catalogObjects{Objects: sorted})
return string(out)
}

func readCatalogObjects(t *testing.T, fsys afero.Fs, path string) []string {
t.Helper()
raw, err := afero.ReadFile(fsys, path)
require.NoError(t, err)
var parsed catalogObjects
require.NoError(t, json.Unmarshal(raw, &parsed))
return parsed.Objects
}

// TestGenerateThenSyncWithNoMigrationsCancelsPlatformObjects exercises the full
// generate -> sync (no local migrations) flow end to end through the public
// command functions. The bug it guards: generate writes the baseline catalog
// (catalog-baseline-<version>.json) that sync reuses as its diff source when
// there are no local migrations. If that baseline is captured from a bare image
// instead of the platform baseline, platform-managed objects (auth/storage/
// realtime) leak into the generated migration even though the user only declared
// a single table. The Docker and pg-delta seams are stubbed (the established
// cli-go pattern) so the test runs in the standard `go test ./...` CI job.
func TestGenerateThenSyncWithNoMigrationsCancelsPlatformObjects(t *testing.T) {
fsys := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fsys, utils.ConfigPath, []byte("[db]\n"), 0644))

originalPgDelta := utils.Config.Experimental.PgDelta
originalImage := utils.Config.Db.Image
originalCreateShadow := createShadow
originalSetupShadow := setupShadowDatabase
originalExportCatalog := exportCatalog
originalApplyDeclarative := applyDeclarative
originalExportRef := declarativeExportRef
originalDiffRef := diffPgDeltaRef
t.Cleanup(func() {
utils.Config.Experimental.PgDelta = originalPgDelta
utils.Config.Db.Image = originalImage
createShadow = originalCreateShadow
setupShadowDatabase = originalSetupShadow
exportCatalog = originalExportCatalog
applyDeclarative = originalApplyDeclarative
declarativeExportRef = originalExportRef
diffPgDeltaRef = originalDiffRef
})

utils.Config.Experimental.PgDelta = &config.PgDeltaConfig{Enabled: true}
utils.Config.Db.Image = "public.ecr.aws/supabase/postgres:15.8.1.049"

shadowConfig := pgconn.Config{Host: "127.0.0.1", Port: 5432, User: "postgres", Password: "postgres", Database: "postgres"}
// Model the evolving shadow state: platform baseline provisioning adds the
// auth/storage/realtime schemas, declarative apply adds the user's table.
platformReady := false
declarativeApplied := false
platformObjects := []string{"auth", "realtime", "storage"}
const userObject = "public.profiles"

createShadow = func(_ context.Context) (string, pgconn.Config, error) {
return "test-shadow-container", shadowConfig, nil
}
setupShadowDatabase = func(_ context.Context, _ string, _ afero.Fs, _ ...func(*pgx.ConnConfig)) error {
platformReady = true
return nil
}
applyDeclarative = func(_ context.Context, _ pgconn.Config, _ afero.Fs) error {
declarativeApplied = true
return nil
}
exportCatalog = func(_ context.Context, _ string, role string, _ ...func(*pgx.ConnConfig)) (string, error) {
assert.Equal(t, "postgres", role)
var objects []string
if platformReady {
objects = append(objects, platformObjects...)
}
if declarativeApplied {
objects = append(objects, userObject)
}
return marshalCatalog(objects), nil
}
// generate exports declarative files from the live database; emit a single
// table that depends on auth so WriteDeclarativeSchemas + hashing have content.
declarativeExportRef = func(_ context.Context, _, _ string, _ []string, _ string, _ ...func(*pgx.ConnConfig)) (diff.DeclarativeOutput, error) {
return diff.DeclarativeOutput{
Files: []diff.DeclarativeFile{
{Path: "schemas/public/tables/profiles.sql", SQL: "create table public.profiles (id uuid primary key references auth.users(id));"},
},
}, nil
}
// Stand in for the pg-delta diff: emit DDL for objects present in the target
// catalog but missing from the source catalog. Platform objects that exist in
// both sides must not appear.
diffPgDeltaRef = func(_ context.Context, sourceRef, targetRef string, _ []string, _ string, _ ...func(*pgx.ConnConfig)) (string, error) {
source := readCatalogObjects(t, fsys, sourceRef)
target := readCatalogObjects(t, fsys, targetRef)
inSource := make(map[string]bool, len(source))
for _, obj := range source {
inSource[obj] = true
}
var added []string
for _, obj := range target {
if !inSource[obj] {
added = append(added, obj)
}
}
sort.Strings(added)
var sb strings.Builder
for _, obj := range added {
sb.WriteString("create " + obj + ";\n")
}
return sb.String(), nil
}

liveConfig := pgconn.Config{Host: "db.test.supabase.co", Port: 5432, User: "postgres", Password: "postgres", Database: "postgres"}

// 1. generate writes declarative files and warms the baseline + declarative caches.
require.NoError(t, Generate(t.Context(), nil, liveConfig, true, false, fsys))

// The baseline catalog reused by sync must represent the platform baseline,
// not a bare image.
baselinePath := filepath.Join(pgDeltaTempPath(), fmt.Sprintf(baselineCatalogName, baselineVersionToken()))
assert.ElementsMatch(t, platformObjects, readCatalogObjects(t, fsys, baselinePath),
"baseline catalog must capture the platform baseline (auth/storage/realtime)")

// 2. sync with no local migrations diffs the warmed declarative catalog against
// the baseline. Platform objects exist on both sides, so only the user's table
// should surface in the generated migration.
result, err := DiffDeclarativeToMigrations(t.Context(), nil, false, fsys)
require.NoError(t, err)
assert.Equal(t, baselinePath, result.SourceRef, "no-migration sync must source from the platform baseline catalog")
assert.Contains(t, result.DiffSQL, "public.profiles", "the user's declared table should be generated")
for _, platform := range platformObjects {
assert.NotContains(t, result.DiffSQL, "create "+platform+";",
"platform object %q must cancel out instead of leaking into the migration", platform)
}
}
51 changes: 48 additions & 3 deletions apps/cli-go/internal/db/declarative/declarative_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,52 @@ func TestGetMigrationsCatalogRefUsesBaselineWhenNoMigrations(t *testing.T) {
assert.Equal(t, baselinePath, ref)
}

func TestGetGenerateBaselineCatalogRefSetsUpPlatformBaseline(t *testing.T) {
// The baseline catalog is reused as the diff source for sync-with-no-migrations
// (getMigrationsCatalogRef). Since the declarative target now provisions the
// Supabase platform baseline, the baseline catalog must represent the same
// platform baseline (not the empty image) so platform objects cancel out of the
// diff instead of surfacing as spurious additions. Assert setup runs before the
// catalog is exported.
fsys := afero.NewMemMapFs()
require.NoError(t, fsys.MkdirAll(filepath.Join(utils.TempDir, "pgdelta"), 0755))

originalCreateShadow := createShadow
originalSetupShadow := setupShadowDatabase
originalExportCatalog := exportCatalog
t.Cleanup(func() {
createShadow = originalCreateShadow
setupShadowDatabase = originalSetupShadow
exportCatalog = originalExportCatalog
})

shadowConfig := pgconn.Config{Host: "127.0.0.1", Port: 5432, User: "postgres", Password: "postgres", Database: "postgres"}
createShadow = func(_ context.Context) (string, pgconn.Config, error) {
return "test-shadow-container", shadowConfig, nil
}
var order []string
setupShadowDatabase = func(_ context.Context, container string, _ afero.Fs, _ ...func(*pgx.ConnConfig)) error {
assert.Equal(t, "test-shadow-container", container)
order = append(order, "setup")
return nil
}
exportCatalog = func(_ context.Context, _ string, role string, _ ...func(*pgx.ConnConfig)) (string, error) {
assert.Equal(t, "postgres", role)
order = append(order, "export")
return `{"version":1}`, nil
}

ref, err := getGenerateBaselineCatalogRef(t.Context(), false, fsys)
require.NoError(t, err)
assert.Equal(t, []string{"setup", "export"}, order, "platform baseline must be provisioned before the baseline catalog is exported")

cachePath := filepath.Join(pgDeltaTempPath(), fmt.Sprintf(baselineCatalogName, baselineVersionToken()))
assert.Equal(t, cachePath, ref.ref)
cached, err := afero.ReadFile(fsys, cachePath)
require.NoError(t, err)
assert.JSONEq(t, `{"version":1}`, string(cached))
}

func TestHashDeclarativeSchemasChangesWithContent(t *testing.T) {
fsys := afero.NewMemMapFs()
p1 := filepath.Join(utils.GetDeclarativeDir(), "schemas", "public", "tables", "a.sql")
Expand Down Expand Up @@ -466,9 +512,8 @@ func TestGenerateReusesBaselineShadowForDeclarativeWarmup(t *testing.T) {
}, nil
}
setupCalled := false
setupShadowDatabase = func(_ context.Context, container string, _ afero.Fs, _ ...func(*pgx.ConnConfig)) error {
setupShadowDatabase = func(_ context.Context, _ string, _ afero.Fs, _ ...func(*pgx.ConnConfig)) error {
setupCalled = true
assert.Equal(t, shadowContainer, container)
return nil
}
declarativeExportRef = func(_ context.Context, sourceRef, _ string, _ []string, _ string, _ ...func(*pgx.ConnConfig)) (diff.DeclarativeOutput, error) {
Expand Down Expand Up @@ -498,7 +543,7 @@ func TestGenerateReusesBaselineShadowForDeclarativeWarmup(t *testing.T) {

err := Generate(t.Context(), nil, pgconn.Config{Host: "127.0.0.1", Port: 5432, User: "postgres", Password: "postgres", Database: "postgres"}, true, false, fsys)
require.NoError(t, err)
assert.True(t, setupCalled, "generate should set up the platform baseline on the reused shadow before applying declarative schema")
assert.False(t, setupCalled, "generate must not re-run platform setup on the reused shadow; the baseline resolver already provisioned it")
assert.True(t, applyCalled, "generate should apply declarative schema using reused baseline shadow")
assert.False(t, fallbackCalled, "fallback declarative resolver should not run when baseline shadow is reusable")

Expand Down
Loading