diff --git a/.github/workflows/sync-go-toolchain.yml b/.github/workflows/sync-go-toolchain.yml index 083a4b4a..b507c13d 100644 --- a/.github/workflows/sync-go-toolchain.yml +++ b/.github/workflows/sync-go-toolchain.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v6 with: - ref: ${{ github.head_ref }} + ref: ${{ github.event.pull_request.head.sha }} - uses: actions/setup-go@v6 with: @@ -43,7 +43,7 @@ jobs: go mod edit -toolchain="go${NEW_VERSION}" - name: Commit and push changes - if: steps.gomod.outputs.toolchain != format('go{0}', steps.dockerfile.outputs.version) + if: steps.gomod.outputs.toolchain != format('go{0}', steps.dockerfile.outputs.version) && github.event.pull_request.head.repo.full_name == github.repository env: NEW_VERSION: ${{ steps.dockerfile.outputs.version }} run: | diff --git a/Dockerfile b/Dockerfile index ea9399b7..3a905e68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,14 +6,14 @@ RUN git config --global --add safe.directory /src # install development tools RUN apt-get update \ - && apt-get install -qq --no-install-recommends \ + && apt-get -y install -qq --no-install-recommends \ curl \ file \ mariadb-client \ - postgresql-client \ - sqlite3 \ nodejs \ npm \ + postgresql-client \ + sqlite3 \ && rm -rf /var/lib/apt/lists/* # golangci-lint diff --git a/README.md b/README.md index 12721284..fbd0b7f4 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ The following options are available with all commands. You must use command line - `--strict` - fail if migrations would be applied out of order _(env: `DBMATE_STRICT`)_ - `--wait` - wait for the db to become available before executing the subsequent command _(env: `DBMATE_WAIT`)_ - `--wait-timeout 60s` - timeout for --wait flag _(env: `DBMATE_WAIT_TIMEOUT`)_ +- `--checksum-mode lenient` - specify checksum mode on applied migrations comparison, see [details about concept](#checksum-validation) _(env: `DBMATE_CHECKSUM_MODE`)_ ## Usage @@ -602,12 +603,26 @@ The table is very simple: ```sql CREATE TABLE IF NOT EXISTS schema_migrations ( - version VARCHAR(255) PRIMARY KEY + version VARCHAR(128) PRIMARY KEY + checksum VARCHAR(64) ) ``` You can customize the name of this table using the `--migrations-table` flag or `DBMATE_MIGRATIONS_TABLE` environment variable. +### Checksum validation + +Dbmate supports validating migration file integrity using checksums. When enabled, each migration's contents are hashed (SHA-256) and stored in the `schema_migrations` table. On subsequent runs, dbmate compares the current migration file's checksum with the stored value to detect any changes. + +You can configure checksum validation using the `DBMATE_CHECKSUM_MODE` environment variable or the `--checksum-mode` command-line option. Supported modes are: +- `NONE`: Disable checksum validation. +- `LENIENT`: Warn if a migration file has changed after being applied (default). +- `STRICT`: Fail if a migration file has changed after being applied. + +Whatever mode is defined, Dbmate will still record each migration file's hash in database. + +This feature helps ensure that applied migrations remain unchanged, improving database integrity and team collaboration by explicitly raising bad practices using database schema migration tool. + ## Alternatives Why another database schema migration tool? Dbmate was inspired by many other tools, primarily [Active Record Migrations](http://guides.rubyonrails.org/active_record_migrations.html), with the goals of being trivial to configure, and language & framework independent. Here is a comparison between dbmate and other popular migration tools. diff --git a/main.go b/main.go index 4b4c9308..cb1550dc 100644 --- a/main.go +++ b/main.go @@ -96,6 +96,11 @@ func NewApp() *cli.App { Usage: "timeout for --wait flag", Value: defaultDB.WaitTimeout, }, + &cli.StringFlag{ + Name: "checksum-mode", + EnvVars: []string{"DBMATE_CHECKSUM_MODE"}, + Usage: "set the checksum mode used during local and applied migrations comparison", + }, } app.Commands = []*cli.Command{ @@ -127,6 +132,9 @@ func NewApp() *cli.App { Action: action(func(db *dbmate.DB, c *cli.Context) error { db.Strict = c.Bool("strict") db.Verbose = c.Bool("verbose") + if db.Verbose { + fmt.Fprintf(db.Log, "Checksum mode: %s\n", dbmate.ModeToString(db.ChecksumMode)) + } return db.CreateAndMigrate() }), }, @@ -163,6 +171,9 @@ func NewApp() *cli.App { Action: action(func(db *dbmate.DB, c *cli.Context) error { db.Strict = c.Bool("strict") db.Verbose = c.Bool("verbose") + if db.Verbose { + fmt.Fprintf(db.Log, "Checksum mode: %s\n", dbmate.ModeToString(db.ChecksumMode)) + } return db.Migrate() }), }, @@ -180,6 +191,9 @@ func NewApp() *cli.App { }, Action: action(func(db *dbmate.DB, c *cli.Context) error { db.Verbose = c.Bool("verbose") + if db.Verbose { + fmt.Fprintf(db.Log, "Checksum mode: %s\n", dbmate.ModeToString(db.ChecksumMode)) + } return db.Rollback() }), }, @@ -293,6 +307,10 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { } db := dbmate.New(u) db.AutoDumpSchema = !c.Bool("no-dump-schema") + db.ChecksumMode, err = dbmate.ParseChecksumMode(c.String("checksum-mode")) + if err != nil { + return err + } db.MigrationsDir = c.StringSlice("migrations-dir") db.MigrationsTableName = c.String("migrations-table") db.SchemaFile = c.String("schema-file") diff --git a/pkg/dbmate/checksum.go b/pkg/dbmate/checksum.go new file mode 100644 index 00000000..72f4072a --- /dev/null +++ b/pkg/dbmate/checksum.go @@ -0,0 +1,62 @@ +package dbmate + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "strings" +) + +type ChecksumMode int + +const ( + ChecksumNone ChecksumMode = iota + ChecksumLenient + ChecksumStrict +) + +var ErrUnknownChecksumMode = errors.New("unknown checksum mode") + +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + +// ParseChecksumMode parses environment/CLI strings to a ChecksumMode. +// Accepted strings (case-insensitive): "NONE", "LENIENT", "STRICT". +func ParseChecksumMode(s string) (ChecksumMode, error) { + switch strings.ToUpper(strings.TrimSpace(s)) { + case "NONE": + return ChecksumNone, nil + case "", "LENIENT": + return ChecksumLenient, nil + case "STRICT": + return ChecksumStrict, nil + default: + return ChecksumLenient, ErrUnknownChecksumMode + } +} + +func ModeToString(m ChecksumMode) string { + switch m { + case ChecksumNone: + return "NONE" + case ChecksumLenient: + return "LENIENT" + case ChecksumStrict: + return "STRICT" + default: + return "UNKNOWN" + } +} + +// ComputeChecksum computes a SHA256 checksum of the given bytes after +// canonicalizing text. We strip a leading UTF-8 BOM (if present) and normalize +// CRLF -> LF so checksums are stable across platforms. +func ComputeChecksum(b []byte) string { + // strip UTF-8 BOM if present + b = bytes.TrimPrefix(b, utf8BOM) + + // normalize CRLF -> LF + b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} diff --git a/pkg/dbmate/checksum_test.go b/pkg/dbmate/checksum_test.go new file mode 100644 index 00000000..5d4ec804 --- /dev/null +++ b/pkg/dbmate/checksum_test.go @@ -0,0 +1,44 @@ +package dbmate + +import ( + "bytes" + "testing" +) + +func TestComputeChecksum_LFvsCRLF(t *testing.T) { + a := []byte("-- migrate:up\nCREATE TABLE foo (id INTEGER);\n") + b := []byte("-- migrate:up\r\nCREATE TABLE foo (id INTEGER);\r\n") + ha := ComputeChecksum(a) + hb := ComputeChecksum(b) + if ha != hb { + t.Fatalf("checksums differ for LF vs CRLF: %s != %s", ha, hb) + } +} + +func TestComputeChecksum_BOMStripped(t *testing.T) { + bom := []byte{0xEF, 0xBB, 0xBF} + body := []byte("-- migrate:up\nCREATE TABLE foo (id INTEGER);\n") + withBOM := append(bom, body...) + h1 := ComputeChecksum(body) + h2 := ComputeChecksum(withBOM) + if h1 != h2 { + t.Fatalf("checksums differ with/without BOM: %s != %s", h1, h2) + } +} + +func TestComputeChecksum_CRLFandBOM(t *testing.T) { + bom := []byte{0xEF, 0xBB, 0xBF} + lf := []byte("-- migrate:up\nCREATE TABLE foo (id INTEGER);\n") + crlf := bytes.ReplaceAll(lf, []byte("\n"), []byte("\r\n")) + hlf := ComputeChecksum(lf) + hcrlf := ComputeChecksum(crlf) + if hlf != hcrlf { + t.Fatalf("checksums differ for CRLF vs LF: %s != %s", hlf, hcrlf) + } + // BOM CRLF + withBOM := append(bom, crlf...) + h3 := ComputeChecksum(withBOM) + if h3 != hlf { + t.Fatalf("checksums differ for BOMCRLF vs LF: %s != %s", h3, hlf) + } +} diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index fee77475..deac2c53 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -60,6 +60,8 @@ type DB struct { WaitInterval time.Duration // WaitTimeout specifies maximum time for connection attempts WaitTimeout time.Duration + // ChecksumMode sepcifies migration checksum validation mode + ChecksumMode ChecksumMode } // StatusResult represents an available migration status @@ -83,6 +85,7 @@ func New(databaseURL *url.URL) *DB { WaitBefore: false, WaitInterval: time.Second, WaitTimeout: 60 * time.Second, + ChecksumMode: ChecksumLenient, } } @@ -402,7 +405,7 @@ func (db *DB) Migrate() error { } // record migration - return drv.InsertMigration(tx, migration.Version) + return drv.InsertMigration(tx, migration.Version, migration.Checksum) } if migrationSection.UpOptions.Transaction() { @@ -468,13 +471,25 @@ func (db *DB) FindMigrations() ([]Migration, error) { defer dbutil.MustClose(sqlDB) // find applied migrations - appliedMigrations := map[string]bool{} + appliedMigrations := map[string]*string{} migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) if err != nil { return nil, err } if migrationsTableExists { + hasChecksumColumn, err := drv.HasChecksumColumn(sqlDB) + if err != nil { + return nil, err + } + + if !hasChecksumColumn { + err = drv.AddChecksumColumn(sqlDB) + if err != nil { + return nil, err + } + } + appliedMigrations, err = drv.SelectMigrations(sqlDB, -1) if err != nil { return nil, err @@ -505,9 +520,30 @@ func (db *DB) FindMigrations() ([]Migration, error) { FilePath: path.Join(dir, matches[0]), FS: db.FS, Version: matches[1], + Checksum: "", } - if ok := appliedMigrations[migration.Version]; ok { + + contents, err := migration.readFile() + if err != nil { + return nil, err + } + + migration.Checksum = ComputeChecksum([]byte(contents)) + + if checksum, ok := appliedMigrations[migration.Version]; ok { migration.Applied = true + + if db.ChecksumMode != ChecksumNone && checksum != nil && *checksum != "" && migration.Checksum != *checksum { + errMsg := fmt.Sprintf("The migration file `%s` has been modified since it was applied. Please ensure that the applied migrations are not modified afterwards.", migration.FileName) + + if db.ChecksumMode == ChecksumStrict { + return nil, fmt.Errorf("%s%s%s", "\x1b[31m", errMsg, "\x1b[0m") + } + + if db.ChecksumMode == ChecksumLenient { + fmt.Fprintf(db.Log, "%sWarning: %s%s\n", "\x1b[33m", errMsg, "\x1b[0m") + } + } } migrations = append(migrations, migration) diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index cd1d43c9..7456cb97 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -1,8 +1,11 @@ package dbmate_test import ( + "bytes" + "fmt" "net/url" "os" + "path" "path/filepath" "regexp" "strings" @@ -228,7 +231,10 @@ func TestLoadSchema(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users and posts tables have been created var count int @@ -367,7 +373,10 @@ func TestMigrate(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users table have records count := 0 @@ -399,7 +408,10 @@ func TestUp(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users table have records count := 0 @@ -438,7 +450,10 @@ func TestRollback(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users and posts tables have been created var count int @@ -659,6 +674,144 @@ func TestFindMigrationsFSMultipleDirs(t *testing.T) { require.Equal(t, "db/migrations_c/006_test_migration_c.sql", actual[5].FilePath) } +func TestFindMigrationsChecksum(t *testing.T) { + testEachURL(t, func(t *testing.T, u *url.URL) { + db := dbmate.New(u) + db.AutoDumpSchema = false + drv, err := db.Driver() + require.NoError(t, err) + + db.ChecksumMode = dbmate.ChecksumLenient + + // prepare + relDir := "migrations" + fileName := "20250101000000_create_foo.sql" + fullKey := path.Join(relDir, fileName) + + // Original file content that will be used to compute the DB-stored checksum. + upSQLOriginal := `-- migrate:up +CREATE TABLE foo (id INTEGER PRIMARY KEY); + +-- migrate:down +DROP TABLE foo; +` + + mfs := fstest.MapFS{ + fullKey: &fstest.MapFile{Data: []byte(upSQLOriginal)}, + } + + db.FS = mfs + db.MigrationsDir = []string{relDir} + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + // one pending + results, err := db.FindMigrations() + require.NoError(t, err) + require.Len(t, results, 1) + require.False(t, results[0].Applied) + migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) + require.NoError(t, err) + require.False(t, migrationsTableExists) + + // run migrations + err = db.Migrate() + require.NoError(t, err) + + // one applied + results, err = db.FindMigrations() + require.NoError(t, err) + require.Len(t, results, 1) + require.True(t, results[0].Applied) + + // Different content to simulate a modified file (triggers mismatch) + upSQLModified := `-- migrate:up +CREATE TABLE foo (id INTEGER); + +-- migrate:down +DROP TABLE foo; +` + + mfs = fstest.MapFS{ + fullKey: &fstest.MapFile{Data: []byte(upSQLModified)}, + } + + db.FS = mfs + + // capture logs + buf := &bytes.Buffer{} + db.Log = buf + + err = db.Migrate() + require.NoError(t, err) + + out := buf.String() + require.Contains(t, out, "Warning: The migration file `20250101000000_create_foo.sql` has been modified since it was applied.") + + appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) + require.NoError(t, err) + // It should keep the original checksum + require.Equal(t, dbmate.ComputeChecksum([]byte(upSQLOriginal)), *appliedMigrations["20250101000000"]) + + // Test Strict mode + db.ChecksumMode = dbmate.ChecksumStrict + + err = db.Migrate() + require.Error(t, err) + }) +} + +func TestChecksumFeatureRetroCompatibility(t *testing.T) { + testEachURL(t, func(t *testing.T, u *url.URL) { + db := newTestDB(t, u) + db.AutoDumpSchema = false + + drv, err := db.Driver() + require.NoError(t, err) + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + err = drv.CreateMigrationsTable(sqlDB) + require.NoError(t, err) + + err = db.Migrate() + require.NoError(t, err) + + _, err = sqlDB.Exec(fmt.Sprintf("alter table %s drop column checksum", db.MigrationsTableName)) + require.NoError(t, err) + + pending, err := db.Status(false) + require.NoError(t, err) + require.Equal(t, 0, pending) + + migrations, err := drv.SelectMigrations(sqlDB, -1) + require.NoError(t, err) + require.NotNil(t, migrations["20200227231541"]) + require.NotNil(t, migrations["20151129054053"]) + // old migrations have no checksum set + require.Equal(t, "", *migrations["20200227231541"]) + require.Equal(t, "", *migrations["20151129054053"]) + }) +} + func TestMigrateUnrestrictedOrder(t *testing.T) { emptyMigration := []byte("-- migrate:up\n-- migrate:down") diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index b468bcef..988ea592 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -18,8 +18,10 @@ type Driver interface { DumpSchema(*sql.DB) ([]byte, error) MigrationsTableExists(*sql.DB) (bool, error) CreateMigrationsTable(*sql.DB) error - SelectMigrations(*sql.DB, int) (map[string]bool, error) - InsertMigration(dbutil.Transaction, string) error + HasChecksumColumn(*sql.DB) (bool, error) + AddChecksumColumn(*sql.DB) error + SelectMigrations(*sql.DB, int) (map[string]*string, error) + InsertMigration(dbutil.Transaction, string, string) error DeleteMigration(dbutil.Transaction, string) error Ping() error QueryError(string, error) error diff --git a/pkg/dbmate/migration.go b/pkg/dbmate/migration.go index 478b560c..d154e24f 100644 --- a/pkg/dbmate/migration.go +++ b/pkg/dbmate/migration.go @@ -15,6 +15,7 @@ type Migration struct { FilePath string FS fs.FS Version string + Checksum string } func (m *Migration) readFile() (string, error) { diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 8c373cf0..9aad469b 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "errors" "fmt" "io" "net/url" @@ -97,11 +98,117 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { Name: "version", Type: bigquery.StringFieldType, }, + &bigquery.FieldSchema{ + Name: "checksum", + Type: bigquery.StringFieldType, + Required: false, + }, }, }) }) } +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return false, err + } + defer conn.Close() + + err = conn.Raw(func(driverConn any) error { + client := getClient(driverConn) + config := getConfig(driverConn) + table := client.Dataset(config.dataSet).Table(drv.migrationsTableName) + meta, err := table.Metadata(context.Background()) + if err != nil { + return err + } + + for _, field := range meta.Schema { + if field.Name == "checksum" { + return nil + } + } + + return errors.New("column not found in table") + }) + + if err != nil { + if err.Error() != "column not found in table" { + return false, err + } + return false, nil + } + + return true, nil +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + + return conn.Raw(func(driverConn any) error { + client := getClient(driverConn) + config := getConfig(driverConn) + + exists, err := tableExists(client, config.dataSet, drv.migrationsTableName) + if err != nil { + return err + } + if !exists { + return errors.New("migrations table not found") + } + + table := client.Dataset(config.dataSet).Table(drv.migrationsTableName) + + meta, err := table.Metadata(context.Background()) + if err != nil { + return fmt.Errorf("failed to get table metadata: %w", err) + } + + for _, f := range meta.Schema { + if strings.EqualFold(f.Name, "checksum") { + return nil + } + } + + checksumField := &bigquery.FieldSchema{ + Name: "checksum", + Type: bigquery.StringFieldType, + Required: false, + } + + newSchema := append(meta.Schema, checksumField) + _, err = table.Update(ctx, bigquery.TableMetadataToUpdate{Schema: newSchema}, meta.ETag) + if err != nil { + return fmt.Errorf("table update failed: %w", err) + } + + meta2, merr := client.Dataset(config.dataSet).Table(drv.migrationsTableName).Metadata(ctx) + if merr == nil { + for _, f := range meta2.Schema { + if strings.EqualFold(f.Name, "checksum") { + return nil // success + } + } + return errors.New("new column not found") + } + + // Verification failed, but table update succeeded. + // Log success message with warning about verification failure. + if drv.log != nil { + fmt.Fprintf(drv.log, "Column %q added successfully to table %s.%s (verification failed: %v)\n", + checksumField.Name, config.dataSet, drv.migrationsTableName, merr) + } + return nil + }) +} + func (drv *Driver) DatabaseExists() (bool, error) { db, err := drv.Open() if err != nil { @@ -223,22 +330,81 @@ func (drv *Driver) schemaDump(db *sql.DB) ([]byte, error) { func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.migrationsTableName + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select version, checksum from %s order by version asc", migrationsTable) + } else { + query = fmt.Sprintf("select version from %s order by version asc", migrationsTable) + } + // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select version from %s order by version asc", migrationsTable)) + rows, err := db.Query(query) if err != nil { return nil, err } + defer dbutil.MustClose(rows) + + migrations := [][]string{} + for rows.Next() { + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } else { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } // build schema migrations table data var buf bytes.Buffer buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("('%s', NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("('%s','%s')", v, c)) + } + } else { + tuples = append(tuples, fmt.Sprintf("('%s')", v)) + } + } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version) VALUES\n ('", migrationsTable) + - strings.Join(migrations, "'),\n ('") + - "');\n") + fmt.Sprintf("INSERT INTO %s (%s) VALUES\n ", migrationsTable, columns) + + strings.Join(tuples, ",\n ") + + ";\n") } return buf.Bytes(), nil @@ -304,7 +470,7 @@ func (drv *Driver) DeleteMigration(util dbutil.Transaction, version string) erro return nil } -func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string, checksum string) error { db, err := drv.Open() if err != nil { return err @@ -316,9 +482,9 @@ func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string) error { return err } - queryTemplate := `INSERT INTO %s.%s (version) VALUES ('%s');` - queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName, version) - _, err = db.Exec(queryString, version) + queryTemplate := `INSERT INTO %s.%s (version, checksum) VALUES (?, ?);` + queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName) + _, err = db.Exec(queryString, version, checksum) if err != nil { return err } @@ -344,13 +510,13 @@ func (*Driver) QueryError(query string, err error) error { return &dbmate.QueryError{Err: err, Query: query} } -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { config, err := drv.getConfig(db) if err != nil { return nil, err } - query := fmt.Sprintf("SELECT version FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) + query := fmt.Sprintf("SELECT version, checksum FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -360,14 +526,19 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + migrations[version] = checksum } if err = rows.Err(); err != nil { diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index 03025a85..a318052a 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -233,22 +233,22 @@ func TestBigQuerySelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestBigQueryInsertMigration(t *testing.T) { @@ -267,7 +267,7 @@ func TestBigQueryInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). @@ -353,9 +353,9 @@ func TestGoogleBigQueryDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema @@ -368,8 +368,8 @@ func TestGoogleBigQueryDumpSchema(t *testing.T) { require.Contains(t, string(schema), "\n--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO schema_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO schema_migrations (version, checksum) VALUES\n"+ + " ('abc1','checksum1'),\n"+ + " ('abc2','checksum2');\n") }) } diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 7a527a99..3a2add4e 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -199,28 +199,82 @@ func (drv *Driver) schemaDump(db *sql.DB, buf *bytes.Buffer) error { func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { migrationsTable := drv.quotedMigrationsTableName() + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select version, checksum from %s final ", migrationsTable) + + "where applied order by version asc" + } else { + query = fmt.Sprintf("select version from %s final ", migrationsTable) + + "where applied order by version asc" + } + // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select version from %s final ", migrationsTable)+ - "where applied order by version asc", - ) + rows, err := db.Query(query) if err != nil { return err } + defer dbutil.MustClose(rows) - quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`) - for i := range migrations { - migrations[i] = "'" + quoter.Replace(migrations[i]) + "'" + type migration struct { + version string + checksum *string + } + migrations := []migration{} + for rows.Next() { + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return err + } + migrations = append(migrations, migration{version, checksum}) + } else { + var version string + if err := rows.Scan(&version); err != nil { + return err + } + migrations = append(migrations, migration{version, nil}) + } } + if err := rows.Err(); err != nil { + return err + } + + quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`) // build schema migrations table data buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + // quote version (always non-NULL) + quotedVersion := "'" + quoter.Replace(m.version) + "'" + + if !hasChecksumColumn { + tuples = append(tuples, fmt.Sprintf("(%s)", quotedVersion)) + } else if m.checksum == nil { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", quotedVersion)) + } else { + quotedChecksum := "'" + quoter.Replace(*m.checksum) + "'" + tuples = append(tuples, fmt.Sprintf("(%s, %s)", quotedVersion, quotedChecksum)) + } + } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s.%s (version) VALUES\n (", drv.quotedDatabaseName(), migrationsTable) + - strings.Join(migrations, "),\n (") + - ");\n") + fmt.Sprintf("INSERT INTO %s.%s (%s) VALUES\n ", drv.quotedDatabaseName(), migrationsTable, columns) + + strings.Join(tuples, ",\n ") + + ";\n") } return nil @@ -255,7 +309,7 @@ func (drv *Driver) DatabaseExists() (bool, error) { defer dbutil.MustClose(db) exists := false - err = db.QueryRow("SELECT 1 FROM system.databases where name = ?", name). + err = db.QueryRow(fmt.Sprintf("EXISTS DATABASE %s", drv.quoteIdentifier(name))). Scan(&exists) if err == sql.ErrNoRows { return false, nil @@ -288,6 +342,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf(` create table if not exists %s%s ( version String, + checksum String, ts DateTime default now(), applied UInt8 default 1 ) engine = %s @@ -298,10 +353,39 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { return err } +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + var dummy int + err := db.QueryRow("SELECT 1 FROM system.columns WHERE database = ? AND table = ? AND name = 'checksum'", + drv.databaseName(), drv.migrationsTableName). + Scan(&dummy) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + qualifiedTableName := drv.quoteIdentifier(drv.databaseName()) + "." + drv.quotedMigrationsTableName() + query := fmt.Sprintf("ALTER TABLE %s%s ADD COLUMN checksum String", qualifiedTableName, drv.onClusterClause()) + _, err := db.Exec(query) + if err != nil { + // If column already exists (duplicate column error), ignore it + // This can happen in cluster setups due to race conditions + if chErr, ok := err.(*clickhouse.Exception); ok && chErr.Code == 15 { + return nil + } + return err + } + return nil +} + // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := fmt.Sprintf("select version from %s final where applied order by version desc", +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { + query := fmt.Sprintf("select version, checksum from %s final where applied order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { @@ -314,14 +398,20 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -332,10 +422,10 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, checksum) values (?, ?)", drv.quotedMigrationsTableName()), + version, checksum) return err } diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index 6b4c14e5..6397a39e 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -2,6 +2,7 @@ package clickhouse import ( "database/sql" + "fmt" "testing" "github.com/amacneil/dbmate/v2/pkg/dbmate" @@ -101,13 +102,13 @@ func TestClickHouseDumpSchemaOnCluster(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) tx, err = db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc2") + err = drv.InsertMigration(tx, "abc2", "checksum2") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -120,9 +121,9 @@ func TestClickHouseDumpSchemaOnCluster(t *testing.T) { require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO "+drv.databaseName()+".test_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO "+drv.databaseName()+".test_migrations (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -215,43 +216,43 @@ func TestClickHouseSelectMigrationsOnCluster(t *testing.T) { tx, err := db01.Begin() require.NoError(t, err) - stmt, err := tx.Prepare("insert into test_migrations (version) values (?)") + stmt, err := tx.Prepare("insert into test_migrations (version, checksum) values (?, ?)") require.NoError(t, err) - _, err = stmt.Exec("abc2") + _, err = stmt.Exec("abc2", nil) require.NoError(t, err) - _, err = stmt.Exec("abc1") + _, err = stmt.Exec("abc1", "checksum1") require.NoError(t, err) - _, err = stmt.Exec("abc3") + _, err = stmt.Exec("abc3", "checksum3") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) migrations01, err := drv01.SelectMigrations(db01, -1) require.NoError(t, err) - require.Equal(t, true, migrations01["abc1"]) - require.Equal(t, true, migrations01["abc2"]) - require.Equal(t, true, migrations01["abc3"]) + require.Equal(t, "checksum1", *migrations01["abc1"]) + require.Equal(t, "", *migrations01["abc2"]) + require.Equal(t, "checksum3", *migrations01["abc3"]) // Assert select on other node migrations02, err := drv02.SelectMigrations(db02, -1) require.NoError(t, err) - require.Equal(t, true, migrations02["abc1"]) - require.Equal(t, true, migrations02["abc2"]) - require.Equal(t, true, migrations02["abc3"]) + require.Equal(t, "checksum1", *migrations02["abc1"]) + require.Equal(t, "", *migrations02["abc2"]) + require.Equal(t, "checksum3", *migrations02["abc3"]) // test limit param migrations01, err = drv01.SelectMigrations(db01, 1) require.NoError(t, err) - require.Equal(t, true, migrations01["abc3"]) - require.Equal(t, false, migrations01["abc1"]) - require.Equal(t, false, migrations01["abc2"]) + require.Equal(t, "checksum3", *migrations01["abc3"]) + require.Equal(t, (*string)(nil), migrations01["abc2"]) + require.Equal(t, (*string)(nil), migrations01["abc1"]) // test limit param on other node migrations02, err = drv02.SelectMigrations(db02, 1) require.NoError(t, err) - require.Equal(t, true, migrations02["abc3"]) - require.Equal(t, false, migrations02["abc1"]) - require.Equal(t, false, migrations02["abc2"]) + require.Equal(t, "checksum3", *migrations02["abc3"]) + require.Equal(t, (*string)(nil), migrations02["abc2"]) + require.Equal(t, (*string)(nil), migrations02["abc1"]) } func TestClickHouseInsertMigrationOnCluster(t *testing.T) { @@ -282,7 +283,7 @@ func TestClickHouseInsertMigrationOnCluster(t *testing.T) { // insert migration tx, err := db01.Begin() require.NoError(t, err) - err = drv01.InsertMigration(tx, "abc1") + err = drv01.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -339,3 +340,64 @@ func TestClickHouseDeleteMigrationOnCluster(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, count02) } + +func TestClickHouseAddChecksumColumnOnClusterNonReplicated(t *testing.T) { + drv01 := testClickHouseDriverCluster01(t) + drv02 := testClickHouseDriverCluster02(t) + // Use a distinct table name to avoid collisions + tableName := "test_migrations_nonrepl" + drv01.migrationsTableName = tableName + drv02.migrationsTableName = tableName + + db01 := prepTestClickHouseDB(t, drv01) + defer dbutil.MustClose(db01) + + db02 := prepTestClickHouseDB(t, drv02) + defer dbutil.MustClose(db02) + + // Create migrations table WITHOUT checksum column, using non-replicated engine + // Even though OnCluster is true, we use ReplacingMergeTree (non-replicated) + // to test that ON CLUSTER clause is needed for DDL propagation across nodes. + engineClause := "ReplacingMergeTree(ts)" + createTableSQL := fmt.Sprintf(` + create table if not exists %s%s ( + version String, + ts DateTime default now(), + applied UInt8 default 1 + ) engine = %s + primary key version + order by version + `, drv01.quotedMigrationsTableName(), drv01.onClusterClause(), engineClause) + _, err := db01.Exec(createTableSQL) + require.NoError(t, err) + + // verify table exists on both nodes (because of ON CLUSTER clause) + exists, err := drv01.MigrationsTableExists(db01) + require.NoError(t, err) + require.True(t, exists) + exists, err = drv02.MigrationsTableExists(db02) + require.NoError(t, err) + require.True(t, exists) + + // verify checksum column does not exist initially + hasChecksum, err := drv01.HasChecksumColumn(db01) + require.NoError(t, err) + require.False(t, hasChecksum) + hasChecksum, err = drv02.HasChecksumColumn(db02) + require.NoError(t, err) + require.False(t, hasChecksum) + + // add checksum column + err = drv01.AddChecksumColumn(db01) + require.NoError(t, err) + + // verify checksum column exists on node1 (where ALTER TABLE executed) + hasChecksum, err = drv01.HasChecksumColumn(db01) + require.NoError(t, err) + require.True(t, hasChecksum, "checksum column should exist on node1") + + // verify checksum column exists on node2 (because of ON CLUSTER clause) + hasChecksum, err = drv02.HasChecksumColumn(db02) + require.NoError(t, err) + require.True(t, hasChecksum, "checksum column should exist on node2") +} diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index 1f4e2ad3..7011aa9a 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -105,13 +105,13 @@ func TestClickHouseDumpSchema(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) tx, err = db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc2") + err = drv.InsertMigration(tx, "abc2", "checksum2") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -123,9 +123,9 @@ func TestClickHouseDumpSchema(t *testing.T) { require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO "+drv.databaseName()+".test_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO "+drv.databaseName()+".test_migrations (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -264,29 +264,29 @@ func TestClickHouseSelectMigrations(t *testing.T) { tx, err := db.Begin() require.NoError(t, err) - stmt, err := tx.Prepare("insert into test_migrations (version) values (?)") + stmt, err := tx.Prepare("insert into test_migrations (version, checksum) values (?,?)") require.NoError(t, err) - _, err = stmt.Exec("abc2") + _, err = stmt.Exec("abc2", nil) require.NoError(t, err) - _, err = stmt.Exec("abc1") + _, err = stmt.Exec("abc1", nil) require.NoError(t, err) - _, err = stmt.Exec("abc3") + _, err = stmt.Exec("abc3", "checksum3") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestClickHouseInsertMigration(t *testing.T) { @@ -307,7 +307,7 @@ func TestClickHouseInsertMigration(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 63c92bc9..043218d9 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -245,12 +245,53 @@ func (drv *Driver) mysqldumpArgs(ver *mysqldumpVersion) []string { func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable) + } else { + query = fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable) + } + // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) + rows, err := db.Query(query) if err != nil { return nil, err } + defer dbutil.MustClose(rows) + + migrations := [][]string{} + for rows.Next() { + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } else { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } // build schema_migrations table data var buf bytes.Buffer @@ -258,10 +299,28 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { fmt.Sprintf("LOCK TABLES %s WRITE;\n", migrationsTable)) if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } + } else { + tuples = append(tuples, fmt.Sprintf("(%s)", v)) + } + } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + - strings.Join(migrations, "),\n (") + - ");\n") + fmt.Sprintf("INSERT INTO %s (%s) VALUES\n ", migrationsTable, columns) + + strings.Join(tuples, ",\n ") + + ";\n") } buf.WriteString("UNLOCK TABLES;\n") @@ -332,7 +391,27 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key)", + "create table if not exists %s (version varchar(128) primary key, checksum varchar(64))", + drv.quotedMigrationsTableName())) + + return err +} + +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + exists := false + err := db.QueryRow(fmt.Sprintf("SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = '%s' AND column_name = 'checksum'", + drv.migrationsTableName)). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN checksum VARCHAR(64)", drv.quotedMigrationsTableName())) return err @@ -340,8 +419,8 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { + query := fmt.Sprintf("select version, checksum from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -352,14 +431,20 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -370,10 +455,10 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, checksum) values (?, ?)", drv.quotedMigrationsTableName()), + version, checksum) return err } diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index 532ef8b9..a6d050ac 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -265,9 +265,9 @@ func TestMySQLDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema @@ -279,9 +279,9 @@ func TestMySQLDumpSchema(t *testing.T) { "-- Dbmate schema migrations\n"+ "--\n\n"+ "LOCK TABLES `test_migrations` WRITE;\n"+ - "INSERT INTO `test_migrations` (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n"+ + "INSERT INTO `test_migrations` (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n"+ "UNLOCK TABLES;\n") // DumpSchema should return error if command fails @@ -388,22 +388,22 @@ func TestMySQLSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestMySQLInsertMigration(t *testing.T) { @@ -422,7 +422,7 @@ func TestMySQLInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index f07a66e8..2df7dc24 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -225,21 +225,80 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { return nil, err } + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = "select quote_literal(version), quote_literal(checksum) from " + migrationsTable + " order by version asc" + } else { + query = "select quote_literal(version) from " + migrationsTable + " order by version asc" + } + // load applied migrations - migrations, err := dbutil.QueryColumn(db, - "select quote_literal(version) from "+migrationsTable+" order by version asc") + rows, err := db.Query(query) if err != nil { return nil, err } + defer dbutil.MustClose(rows) + + migrations := [][]string{} + for rows.Next() { + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } else { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } // build migrations table data var buf bytes.Buffer buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { - buf.WriteString("INSERT INTO " + migrationsTable + " (version) VALUES\n (" + - strings.Join(migrations, "),\n (") + - ");\n") + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } + } else { + tuples = append(tuples, fmt.Sprintf("(%s)", v)) + } + } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } + buf.WriteString("INSERT INTO " + migrationsTable + " (" + columns + ") VALUES\n " + + strings.Join(tuples, ",\n ") + + ";\n") } return buf.Bytes(), nil @@ -323,7 +382,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // first attempt at creating migrations table createTableStmt := fmt.Sprintf( - "create table if not exists %s.%s (version varchar primary key)", + "create table if not exists %s.%s (version varchar primary key, checksum varchar)", schema, migrationsTable) _, err = db.Exec(createTableStmt) if err == nil { @@ -351,15 +410,49 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { return err } -// SelectMigrations returns a list of applied migrations +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + schema, migrationsTableNameParts, err := drv.migrationsTableNameParts(db) + if err != nil { + return false, err + } + + migrationsTable := strings.Join(migrationsTableNameParts, ".") + exists := false + err = db.QueryRow("SELECT 1 FROM information_schema.columns "+ + "WHERE table_schema = $1 "+ + "AND table_name = $2 "+ + "AND column_name = 'checksum'", + schema, migrationsTable). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + schema, migrationsTable, err := drv.quotedMigrationsTableNameParts(db) + if err != nil { + return err + } + + addColumnStmt := fmt.Sprintf( + "ALTER TABLE %s.%s ADD COLUMN IF NOT EXISTS checksum VARCHAR", + schema, migrationsTable) + _, err = db.Exec(addColumnStmt) + return err +} + +// SelectMigrations returns a list of applied migrations and its checksum // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return nil, err } - query := "select version from " + migrationsTable + " order by version desc" + query := "select version, checksum from " + migrationsTable + " order by version desc" if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -370,14 +463,19 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -388,13 +486,13 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return err } - _, err = db.Exec("insert into "+migrationsTable+" (version) values ($1)", version) + _, err = db.Exec("insert into "+migrationsTable+" (version, checksum) values ($1, $2)", version, checksum) return err } diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 56d0ffcc..ef45b2fb 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -265,23 +265,23 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations") + require.Contains(t, string(schema), "\n--\n"+ "-- PostgreSQL database dump complete\n"+ "--\n\n") - require.Contains(t, string(schema), "-- Dbmate schema migrations\n"+ - "--\n\n"+ - "INSERT INTO public.schema_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + require.Contains(t, string(schema), + "INSERT INTO public.schema_migrations (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -302,23 +302,24 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE \"camelSchema\".\"testMigrations\"") + require.Contains(t, string(schema), "\n--\n"+ "-- PostgreSQL database dump complete\n"+ "--\n\n") require.Contains(t, string(schema), "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO \"camelSchema\".\"testMigrations\" (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO \"camelSchema\".\"testMigrations\" (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") }) } @@ -523,22 +524,22 @@ func TestPostgresSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into public.test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestPostgresInsertMigration(t *testing.T) { @@ -557,7 +558,7 @@ func TestPostgresInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from public.test_migrations where version = 'abc1'"). diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 12c19a4c..dd575af4 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -107,22 +107,81 @@ func (drv *Driver) DropDatabase() error { func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable) + } else { + query = fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable) + } + // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) + rows, err := db.Query(query) if err != nil { return nil, err } + defer dbutil.MustClose(rows) + + migrations := [][]string{} + for rows.Next() { + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } else { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } // build schema migrations table data var buf bytes.Buffer buf.WriteString("-- Dbmate schema migrations\n") if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } + } else { + tuples = append(tuples, fmt.Sprintf("(%s)", v)) + } + } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + - strings.Join(migrations, "),\n (") + - ");\n") + fmt.Sprintf("INSERT INTO %s (%s) VALUES\n ", migrationsTable, columns) + + strings.Join(tuples, ",\n ") + + ";\n") } return buf.Bytes(), nil @@ -175,7 +234,27 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key)", + "create table if not exists %s (version varchar(128) primary key, checksum varchar(64))", + drv.quotedMigrationsTableName())) + + return err +} + +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + exists := false + err := db.QueryRow("SELECT 1 FROM pragma_table_info(?) WHERE name = 'checksum'", + drv.migrationsTableName). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN checksum VARCHAR(64)", drv.quotedMigrationsTableName())) return err @@ -183,8 +262,8 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { + query := fmt.Sprintf("select version, checksum from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -195,14 +274,19 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -213,10 +297,10 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, checksum) values (?, ?)", drv.quotedMigrationsTableName()), + version, checksum) return err } diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index 87fffa6c..6ded6777 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -176,9 +176,9 @@ func TestSQLiteDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // create a table that will trigger `sqlite_sequence` system table @@ -191,9 +191,9 @@ func TestSQLiteDumpSchema(t *testing.T) { require.Contains(t, string(schema), "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT)") require.Contains(t, string(schema), "CREATE TABLE IF NOT EXISTS \"test_migrations\"") require.Contains(t, string(schema), ");\n-- Dbmate schema migrations\n"+ - "INSERT INTO \"test_migrations\" (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO \"test_migrations\" (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // sqlite_* tables should not be present in the dump (.schema --nosys) require.NotContains(t, string(schema), "sqlite_") @@ -290,22 +290,22 @@ func TestSQLiteSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestSQLiteInsertMigration(t *testing.T) { @@ -324,7 +324,7 @@ func TestSQLiteInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'").