fix: use pgkit migra with experimental flag (#4062)

Author: sweatybridge

cmd/db.go

@@ -101,6 +101,8 @@ var (
 			if usePgSchema {
 				differ = diff.DiffPgSchema
 				fmt.Fprintln(os.Stderr, utils.Yellow("WARNING:"), "--use-pg-schema flag is experimental and may not include all entities, such as views and grants.")
+			} else if !viper.GetBool("EXPERIMENTAL") {
+				differ = diff.DiffSchemaMigraBash
 			}
 			return diff.Run(cmd.Context(), schema, file, flags.DbConfig, differ, afero.NewOsFs())
 		},

internal/db/diff/diff.go

@@ -28,7 +28,7 @@ import (
 	"github.com/supabase/cli/pkg/parser"
 )
 
-type DiffFunc func(context.Context, string, string, []string) (string, error)
+type DiffFunc func(context.Context, string, string, []string, ...func(*pgx.ConnConfig)) (string, error)
 
 func Run(ctx context.Context, schema []string, file string, config pgconn.Config, differ DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (err error) {
 	out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, options...)
@@ -136,7 +136,7 @@ func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs,
 	return migration.ApplyMigrations(ctx, migrations, conn, afero.NewIOFS(fsys))
 }
 
-func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ func(context.Context, string, string, []string) (string, error), options ...func(*pgx.ConnConfig)) (string, error) {
+func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ DiffFunc, options ...func(*pgx.ConnConfig)) (string, error) {
 	fmt.Fprintln(w, "Creating shadow database...")
 	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
 	if err != nil {
@@ -175,7 +175,7 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w
 	}
 	source := utils.ToPostgresURL(shadowConfig)
 	target := utils.ToPostgresURL(config)
-	return differ(ctx, source, target, schema)
+	return differ(ctx, source, target, schema, options...)
 }
 
 func migrateBaseDatabase(ctx context.Context, config pgconn.Config, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {

internal/db/diff/diff_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/h2non/gock"
 	"github.com/jackc/pgconn"
 	"github.com/jackc/pgerrcode"
+	"github.com/jackc/pgx/v4"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -72,7 +73,17 @@ func TestRun(t *testing.T) {
 			Reply("CREATE DATABASE")
 		defer conn.Close(t)
 		// Run test
-		err := Run(context.Background(), []string{"public"}, "file", dbConfig, DiffSchemaMigra, fsys, conn.Intercept)
+		err := Run(context.Background(), []string{"public"}, "file", dbConfig, DiffSchemaMigra, fsys, func(cc *pgx.ConnConfig) {
+			if cc.Host == dbConfig.Host {
+				// Fake a SSL error when connecting to target database
+				cc.LookupFunc = func(ctx context.Context, host string) (addrs []string, err error) {
+					return nil, errors.New("server refused TLS connection")
+				}
+			} else {
+				// Hijack connection to shadow database
+				conn.Intercept(cc)
+			}
+		})
 		// Check error
 		assert.NoError(t, err)
 		assert.Empty(t, apitest.ListUnmatchedRequests())
@@ -306,7 +317,17 @@ create schema public`)
 			Query(migration.INSERT_MIGRATION_VERSION, "0", "test", []string{sql}).
 			Reply("INSERT 0 1")
 		// Run test
-		diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, conn.Intercept)
+		diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, func(cc *pgx.ConnConfig) {
+			if cc.Host == dbConfig.Host {
+				// Fake a SSL error when connecting to target database
+				cc.LookupFunc = func(ctx context.Context, host string) (addrs []string, err error) {
+					return nil, errors.New("server refused TLS connection")
+				}
+			} else {
+				// Hijack connection to shadow database
+				conn.Intercept(cc)
+			}
+		})
 		// Check error
 		assert.Empty(t, diff)
 		assert.ErrorContains(t, err, "error diffing schema")

internal/db/diff/migra.go

@@ -9,9 +9,12 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/go-errors/errors"
+	"github.com/jackc/pgx/v4"
 	"github.com/spf13/viper"
+	"github.com/supabase/cli/internal/gen/types"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/pkg/config"
+	"github.com/supabase/cli/pkg/migration"
 )
 
 var (
@@ -20,6 +23,11 @@ var (
 	//go:embed templates/migra.ts
 	diffSchemaTypeScript string
 
+	//go:embed templates/staging-ca-2021.crt
+	caStaging string
+	//go:embed templates/prod-ca-2021.crt
+	caProd string
+
 	managedSchemas = []string{
 		// Local development
 		"_analytics",
@@ -54,7 +62,14 @@ var (
 )
 
 // Diffs local database schema against shadow, dumps output to stdout.
-func DiffSchemaMigraBash(ctx context.Context, source, target string, schema []string) (string, error) {
+func DiffSchemaMigraBash(ctx context.Context, source, target string, schema []string, options ...func(*pgx.ConnConfig)) (string, error) {
+	// Load all user defined schemas
+	if len(schema) == 0 {
+		var err error
+		if schema, err = loadSchema(ctx, target, options...); err != nil {
+			return "", err
+		}
+	}
 	env := []string{"SOURCE=" + source, "TARGET=" + target}
 	// Passing in script string means command line args must be set manually, ie. "$@"
 	args := "set -- " + strings.Join(schema, " ") + ";"
@@ -80,8 +95,25 @@ func DiffSchemaMigraBash(ctx context.Context, source, target string, schema []st
 	return out.String(), nil
 }
 
-func DiffSchemaMigra(ctx context.Context, source, target string, schema []string) (string, error) {
+func loadSchema(ctx context.Context, dbURL string, options ...func(*pgx.ConnConfig)) ([]string, error) {
+	conn, err := utils.ConnectByUrl(ctx, dbURL, options...)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close(context.Background())
+	// RLS policies in auth and storage schemas can be included with -s flag
+	return migration.ListUserSchemas(ctx, conn)
+}
+
+func DiffSchemaMigra(ctx context.Context, source, target string, schema []string, options ...func(*pgx.ConnConfig)) (string, error) {
 	env := []string{"SOURCE=" + source, "TARGET=" + target}
+	// node-postgres does not support sslmode=prefer
+	if require, err := types.IsRequireSSL(ctx, target, options...); err != nil {
+		return "", err
+	} else if require {
+		rootCA := caStaging + caProd
+		env = append(env, "SSL_CA="+rootCA)
+	}
 	if len(schema) > 0 {
 		env = append(env, "INCLUDED_SCHEMAS="+strings.Join(schema, ","))
 	} else {

internal/db/diff/pgschema.go

@@ -7,10 +7,11 @@ import (
 	"strings"
 
 	"github.com/go-errors/errors"
+	"github.com/jackc/pgx/v4"
 	pgschema "github.com/stripe/pg-schema-diff/pkg/diff"
 )
 
-func DiffPgSchema(ctx context.Context, source, target string, schema []string) (string, error) {
+func DiffPgSchema(ctx context.Context, source, target string, schema []string, _ ...func(*pgx.ConnConfig)) (string, error) {
 	dbSrc, err := sql.Open("pgx", source)
 	if err != nil {
 		return "", errors.Errorf("failed to open source database: %w", err)

internal/db/diff/templates/migra.ts

@@ -1,8 +1,12 @@
 import { createClient } from "npm:@pgkit/client";
 import { Migration } from "npm:@pgkit/migra";
 
+// Avoids error on self-signed certificate
+const ca = Deno.env.get("SSL_CA");
 const clientBase = createClient(Deno.env.get("SOURCE"));
-const clientHead = createClient(Deno.env.get("TARGET"));
+const clientHead = createClient(Deno.env.get("TARGET"), {
+  pgpOptions: { connect: { ssl: ca && { ca } } },
+});
 const includedSchemas = Deno.env.get("INCLUDED_SCHEMAS")?.split(",") ?? [];
 const excludedSchemas = Deno.env.get("EXCLUDED_SCHEMAS")?.split(",") ?? [];
 

internal/db/diff/templates/prod-ca-2021.crt

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIIDxDCCAqygAwIBAgIUbLxMod62P2ktCiAkxnKJwtE9VPYwDQYJKoZIhvcNAQEL
+BQAwazELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB0RlbHdhcmUxEzARBgNVBAcMCk5l
+dyBDYXN0bGUxFTATBgNVBAoMDFN1cGFiYXNlIEluYzEeMBwGA1UEAwwVU3VwYWJh
+c2UgUm9vdCAyMDIxIENBMB4XDTIxMDQyODEwNTY1M1oXDTMxMDQyNjEwNTY1M1ow
+azELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB0RlbHdhcmUxEzARBgNVBAcMCk5ldyBD
+YXN0bGUxFTATBgNVBAoMDFN1cGFiYXNlIEluYzEeMBwGA1UEAwwVU3VwYWJhc2Ug
+Um9vdCAyMDIxIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQXW
+QyHOB+qR2GJobCq/CBmQ40G0oDmCC3mzVnn8sv4XNeWtE5XcEL0uVih7Jo4Dkx1Q
+DmGHBH1zDfgs2qXiLb6xpw/CKQPypZW1JssOTMIfQppNQ87K75Ya0p25Y3ePS2t2
+GtvHxNjUV6kjOZjEn2yWEcBdpOVCUYBVFBNMB4YBHkNRDa/+S4uywAoaTWnCJLUi
+cvTlHmMw6xSQQn1UfRQHk50DMCEJ7Cy1RxrZJrkXXRP3LqQL2ijJ6F4yMfh+Gyb4
+O4XajoVj/+R4GwywKYrrS8PrSNtwxr5StlQO8zIQUSMiq26wM8mgELFlS/32Uclt
+NaQ1xBRizkzpZct9DwIDAQABo2AwXjALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFKjX
+uXY32CztkhImng4yJNUtaUYsMB8GA1UdIwQYMBaAFKjXuXY32CztkhImng4yJNUt
+aUYsMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAB8spzNn+4VU
+tVxbdMaX+39Z50sc7uATmus16jmmHjhIHz+l/9GlJ5KqAMOx26mPZgfzG7oneL2b
+VW+WgYUkTT3XEPFWnTp2RJwQao8/tYPXWEJDc0WVQHrpmnWOFKU/d3MqBgBm5y+6
+jB81TU/RG2rVerPDWP+1MMcNNy0491CTL5XQZ7JfDJJ9CCmXSdtTl4uUQnSuv/Qx
+Cea13BX2ZgJc7Au30vihLhub52De4P/4gonKsNHYdbWjg7OWKwNv/zitGDVDB9Y2
+CMTyZKG3XEu5Ghl1LEnI3QmEKsqaCLv12BnVjbkSeZsMnevJPs1Ye6TjjJwdik5P
+o/bKiIz+Fq8=
+-----END CERTIFICATE-----

internal/db/diff/templates/staging-ca-2021.crt

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID1DCCArygAwIBAgIUbYRdq/8/uNq8G9stMCdOFSBgA2MwDQYJKoZIhvcNAQEL
+BQAwczELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB0RlbHdhcmUxEzARBgNVBAcMCk5l
+dyBDYXN0bGUxFTATBgNVBAoMDFN1cGFiYXNlIEluYzEmMCQGA1UEAwwdU3VwYWJh
+c2UgU3RhZ2luZyBSb290IDIwMjEgQ0EwHhcNMjEwNDI4MTAzNjEzWhcNMzEwNDI2
+MTAzNjEzWjBzMQswCQYDVQQGEwJVUzEQMA4GA1UECAwHRGVsd2FyZTETMBEGA1UE
+BwwKTmV3IENhc3RsZTEVMBMGA1UECgwMU3VwYWJhc2UgSW5jMSYwJAYDVQQDDB1T
+dXBhYmFzZSBTdGFnaW5nIFJvb3QgMjAyMSBDQTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAN0AKRE8a56O8LaZxiOAcHFUFnwiKUvPoXPq26Ifw+Nv+7zg
+N2V5WnMZbbw24q61Os60ZUn0XmbVtuIeJ+stPHsO7qxxuL+bmPR+qU5tkDrIOyEe
+YD/2u8/q6ssVv42k4XcXbhM6RVz7CkCDY0TiBm1bMtRZso3xB6E9wAjxDf43XfV5
+PAGs3JI+Zo/vyqCDlN0hHOrB/aBl01JXqQWI84Gia5ooucq4SjA1CyawBcQ2IAvG
+rXuy1BouY+xM3zRuNvtfFP6rb5Mta+jCYEMh1AZ8yP8sYUWAyhxX6k9EbOb009wQ
+aZljbUCh/UglGWuBxdzePavx+zPjzWXB1NyVkpkCAwEAAaNgMF4wCwYDVR0PBAQD
+AgEGMB0GA1UdDgQWBBQFx+PHLf27iIo/PMfIfGqXF7Zb+DAfBgNVHSMEGDAWgBQF
+x+PHLf27iIo/PMfIfGqXF7Zb+DAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEB
+CwUAA4IBAQB/xIiz5dDqzGXjqYqXZYx4iSfSxsVayeOPDMfmaiCfSMJEUG4cUiwG
+OvMPGztaUEYeip5SCvSKuAAjVkXyP7ahKR7t7lZ9mErVXyxSZoVLbOd578CuYiZk
+OgT17UjPv66WMzEKEr8wGpomTYWWfEkuqt8ENdiM1Z4LNFahdKj36+jm6/a+9R8K
+25VIL68DTaQpBxFWG6ixC1HRMHJ12lDhKsshIi099BVpkGibESlxPrQOdKKqBB/J
+vIX+/Hb+mS4H5zYMeK2wX0onp+GBcD6X9L1UJuXMVd+BRan8RFidXL5s3++xXjQq
+Nzbc6lnA69urKffvcT07YwMsY/OmHzVa
+-----END CERTIFICATE-----

internal/gen/types/types.go

@@ -78,7 +78,7 @@ func Run(ctx context.Context, projectId string, dbConfig pgconn.Config, lang str
 
 	fmt.Fprintln(os.Stderr, "Connecting to", dbConfig.Host, dbConfig.Port)
 	escaped := utils.ToPostgresURL(dbConfig)
-	if require, err := isRequireSSL(ctx, originalURL, options...); err != nil {
+	if require, err := IsRequireSSL(ctx, originalURL, options...); err != nil {
 		return err
 	} else if require {
 		// node-postgres does not support sslmode=prefer
@@ -106,7 +106,7 @@ func Run(ctx context.Context, projectId string, dbConfig pgconn.Config, lang str
 	)
 }
 
-func isRequireSSL(ctx context.Context, dbUrl string, options ...func(*pgx.ConnConfig)) (bool, error) {
+func IsRequireSSL(ctx context.Context, dbUrl string, options ...func(*pgx.ConnConfig)) (bool, error) {
 	conn, err := utils.ConnectByUrl(ctx, dbUrl+"&sslmode=require", options...)
 	if err != nil {
 		if strings.HasSuffix(err.Error(), "(server refused TLS connection)") {

internal/utils/flags/db_url.go

@@ -138,6 +138,8 @@ func NewDbConfigWithPassword(ctx context.Context, projectRef string) pgconn.Conf
 			}
 			config.User = newRole.User
 			return config
+		} else {
+			fmt.Fprintln(utils.GetDebugLogger(), err)
 		}
 	}
 	if config.Password, err = credentials.StoreProvider.Get(projectRef); err == nil {