From 143754d810ad1e57c08c6a164b141d62444afbdd Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Fri, 15 Aug 2025 20:57:54 +0000 Subject: [PATCH 01/15] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20checksum=20vali?= =?UTF-8?q?dation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 9 ++ pkg/dbmate/checksum.go | 52 ++++++++ pkg/dbmate/db.go | 30 ++++- pkg/dbmate/db_test.go | 117 +++++++++++++++++- pkg/dbmate/driver.go | 4 +- pkg/dbmate/migration.go | 1 + pkg/driver/bigquery/bigquery.go | 65 +++++++--- pkg/driver/bigquery/bigquery_test.go | 26 ++-- pkg/driver/clickhouse/clickhouse.go | 65 +++++++--- .../clickhouse/clickhouse_cluster_test.go | 44 +++---- pkg/driver/clickhouse/clickhouse_test.go | 32 ++--- pkg/driver/mysql/mysql.go | 66 +++++++--- pkg/driver/mysql/mysql_test.go | 28 ++--- pkg/driver/postgres/postgres.go | 61 ++++++--- pkg/driver/postgres/postgres_test.go | 38 +++--- pkg/driver/sqlite/sqlite.go | 61 ++++++--- pkg/driver/sqlite/sqlite_test.go | 28 ++--- 17 files changed, 549 insertions(+), 178 deletions(-) create mode 100644 pkg/dbmate/checksum.go diff --git a/main.go b/main.go index 4b4c9308..f5454eb3 100644 --- a/main.go +++ b/main.go @@ -96,6 +96,11 @@ func NewApp() *cli.App { Usage: "timeout for --wait flag", Value: defaultDB.WaitTimeout, }, + &cli.StringFlag{ + Name: "checksum-mode", + EnvVars: []string{"DBMATE_CHECKSUM_MODE"}, + Usage: "set the checksum mode used during local and applied migrations comparison", + }, } app.Commands = []*cli.Command{ @@ -293,6 +298,10 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { } db := dbmate.New(u) db.AutoDumpSchema = !c.Bool("no-dump-schema") + db.ChecksumMode, err = dbmate.ParseChecksumMode(c.String("checksum-mode")) + if err != nil { + return err + } db.MigrationsDir = c.StringSlice("migrations-dir") db.MigrationsTableName = c.String("migrations-table") db.SchemaFile = c.String("schema-file") diff --git a/pkg/dbmate/checksum.go b/pkg/dbmate/checksum.go new file mode 100644 index 00000000..a72c222e --- /dev/null +++ b/pkg/dbmate/checksum.go @@ -0,0 +1,52 @@ +package dbmate + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "strings" +) + +type ChecksumMode int + +const ( + ChecksumNone ChecksumMode = iota + ChecksumLenient + ChecksumStrict +) + +var ErrUnknownChecksumMode = errors.New("unknown checksum mode") + +// ParseChecksumMode parses environment/CLI strings to a ChecksumMode. +// Accepted strings (case-insensitive): "NONE", "LENIENT", "STRICT". +func ParseChecksumMode(s string) (ChecksumMode, error) { + switch strings.ToUpper(strings.TrimSpace(s)) { + case "", "NONE": + return ChecksumNone, nil + case "LENIENT": + return ChecksumLenient, nil + case "STRICT": + return ChecksumStrict, nil + default: + return ChecksumNone, ErrUnknownChecksumMode + } +} + +func ModeToString(m ChecksumMode) string { + switch m { + case ChecksumNone: + return "NONE" + case ChecksumLenient: + return "LENIENT" + case ChecksumStrict: + return "STRICT" + default: + return "UNKNOWN" + } +} + +// ComputeChecksum returns the hex SHA-256 of the supplied bytes. +func ComputeChecksum(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 15c0d2d8..51555126 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -60,6 +60,8 @@ type DB struct { WaitInterval time.Duration // WaitTimeout specifies maximum time for connection attempts WaitTimeout time.Duration + // ChecksumMode sepcifies migration checksum validation mode + ChecksumMode ChecksumMode } // StatusResult represents an available migration status @@ -83,6 +85,7 @@ func New(databaseURL *url.URL) *DB { WaitBefore: false, WaitInterval: time.Second, WaitTimeout: 60 * time.Second, + ChecksumMode: ChecksumNone, } } @@ -395,7 +398,7 @@ func (db *DB) Migrate() error { } // record migration - return drv.InsertMigration(tx, migration.Version) + return drv.InsertMigration(tx, migration.Version, migration.Checksum) } if migrationSection.UpOptions.Transaction() { @@ -461,7 +464,7 @@ func (db *DB) FindMigrations() ([]Migration, error) { defer dbutil.MustClose(sqlDB) // find applied migrations - appliedMigrations := map[string]bool{} + appliedMigrations := map[string]*string{} migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) if err != nil { return nil, err @@ -498,9 +501,30 @@ func (db *DB) FindMigrations() ([]Migration, error) { FilePath: path.Join(dir, matches[0]), FS: db.FS, Version: matches[1], + Checksum: "", } - if ok := appliedMigrations[migration.Version]; ok { + + contents, err := migration.readFile() + if err != nil { + return nil, err + } + + migration.Checksum = ComputeChecksum([]byte(contents)) + + if checksum, ok := appliedMigrations[migration.Version]; ok { migration.Applied = true + + if db.ChecksumMode != ChecksumNone && checksum != nil && *checksum != "" && migration.Checksum != *checksum { + errMsg := fmt.Sprintf("The migration file `%s` has been modified since it was applied. Please ensure that the applied migrations are not modified afterwards.", migration.FileName) + + if db.ChecksumMode == ChecksumStrict { + return nil, errors.New(errMsg) + } + + if db.ChecksumMode == ChecksumLenient { + fmt.Fprintf(db.Log, "Warning: %s\n", errMsg) + } + } } migrations = append(migrations, migration) diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 07d4e590..6298c564 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -1,6 +1,7 @@ package dbmate_test import ( + "bytes" "net/url" "os" "path/filepath" @@ -229,7 +230,10 @@ func TestLoadSchema(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users and posts tables have been created var count int @@ -368,7 +372,10 @@ func TestMigrate(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users table have records count := 0 @@ -400,7 +407,10 @@ func TestUp(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users table have records count := 0 @@ -439,7 +449,10 @@ func TestRollback(t *testing.T) { // check applied migrations appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) require.NoError(t, err) - require.Equal(t, map[string]bool{"20200227231541": true, "20151129054053": true}, appliedMigrations) + require.NotNil(t, appliedMigrations["20200227231541"]) + require.NotNil(t, appliedMigrations["20151129054053"]) + require.Equal(t, "f42c561983eab69a6d69984db98b23b432326acbd938d896687529933e29c54c", *appliedMigrations["20200227231541"]) + require.Equal(t, "96df8abff6662d519c1a6993483d36e2d35955fd557f25e903abe7bd3dc113f1", *appliedMigrations["20151129054053"]) // users and posts tables have been created var count int @@ -660,6 +673,102 @@ func TestFindMigrationsFSMultipleDirs(t *testing.T) { require.Equal(t, "db/migrations_c/006_test_migration_c.sql", actual[5].FilePath) } +func TestFindMigrationsChecksum(t *testing.T) { + testEachURL(t, func(t *testing.T, u *url.URL) { + db := dbmate.New(u) + db.AutoDumpSchema = false + drv, err := db.Driver() + require.NoError(t, err) + + db.ChecksumMode = dbmate.ChecksumLenient + + // prepare + relDir := "migrations" + fileName := "20250101000000_create_foo.sql" + fullKey := filepath.Join(relDir, fileName) + + // Original file content that will be used to compute the DB-stored checksum. + upSQLOriginal := `-- migrate:up +CREATE TABLE foo (id INTEGER PRIMARY KEY); + +-- migrate:down +DROP TABLE foo; +` + + mfs := fstest.MapFS{ + fullKey: &fstest.MapFile{Data: []byte(upSQLOriginal)}, + } + + db.FS = mfs + db.MigrationsDir = []string{relDir} + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + // one pending + results, err := db.FindMigrations() + require.NoError(t, err) + require.Len(t, results, 1) + require.False(t, results[0].Applied) + migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) + require.NoError(t, err) + require.False(t, migrationsTableExists) + + // run migrations + err = db.Migrate() + require.NoError(t, err) + + // one applied + results, err = db.FindMigrations() + require.NoError(t, err) + require.Len(t, results, 1) + require.True(t, results[0].Applied) + + // Different content to simulate a modified file (triggers mismatch) + upSQLModified := `-- migrate:up +CREATE TABLE foo (id INTEGER); + +-- migrate:down +DROP TABLE foo; +` + + mfs = fstest.MapFS{ + fullKey: &fstest.MapFile{Data: []byte(upSQLModified)}, + } + + db.FS = mfs + + // capture logs + buf := &bytes.Buffer{} + db.Log = buf + + err = db.Migrate() + require.NoError(t, err) + + out := buf.String() + require.Contains(t, out, "Warning: The migration file `20250101000000_create_foo.sql` has been modified since it was applied.") + + appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) + require.NoError(t, err) + // It should keep the original checksum + require.Equal(t, dbmate.ComputeChecksum([]byte(upSQLOriginal)), *appliedMigrations["20250101000000"]) + + // Test Strict mode + db.ChecksumMode = dbmate.ChecksumStrict + + err = db.Migrate() + require.Error(t, err) + }) +} + func TestMigrateUnrestrictedOrder(t *testing.T) { emptyMigration := []byte("-- migrate:up\n-- migrate:down") diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index b468bcef..61501b09 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -18,8 +18,8 @@ type Driver interface { DumpSchema(*sql.DB) ([]byte, error) MigrationsTableExists(*sql.DB) (bool, error) CreateMigrationsTable(*sql.DB) error - SelectMigrations(*sql.DB, int) (map[string]bool, error) - InsertMigration(dbutil.Transaction, string) error + SelectMigrations(*sql.DB, int) (map[string]*string, error) + InsertMigration(dbutil.Transaction, string, string) error DeleteMigration(dbutil.Transaction, string) error Ping() error QueryError(string, error) error diff --git a/pkg/dbmate/migration.go b/pkg/dbmate/migration.go index 478b560c..d154e24f 100644 --- a/pkg/dbmate/migration.go +++ b/pkg/dbmate/migration.go @@ -15,6 +15,7 @@ type Migration struct { FilePath string FS fs.FS Version string + Checksum string } func (m *Migration) readFile() (string, error) { diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 8c373cf0..0fb5cec2 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -97,6 +97,10 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { Name: "version", Type: bigquery.StringFieldType, }, + &bigquery.FieldSchema{ + Name: "checksum", + Type: bigquery.StringFieldType, + }, }, }) }) @@ -224,21 +228,49 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.migrationsTableName // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select version from %s order by version asc", migrationsTable)) + rows, err := db.Query(fmt.Sprintf("select version from %s order by version asc", migrationsTable)) if err != nil { return nil, err } + migrations := [][]string{} + for rows.Next() { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } + // build schema migrations table data var buf bytes.Buffer buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("('%s', NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("('%s','%s')", v, c)) + } + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version) VALUES\n ('", migrationsTable) + - strings.Join(migrations, "'),\n ('") + - "');\n") + fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + strings.Join(tuples, ",\n ") + + ";\n") } return buf.Bytes(), nil @@ -304,7 +336,7 @@ func (drv *Driver) DeleteMigration(util dbutil.Transaction, version string) erro return nil } -func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string, checksum string) error { db, err := drv.Open() if err != nil { return err @@ -316,9 +348,9 @@ func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string) error { return err } - queryTemplate := `INSERT INTO %s.%s (version) VALUES ('%s');` - queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName, version) - _, err = db.Exec(queryString, version) + queryTemplate := `INSERT INTO %s.%s (version, checksum) VALUES (?, ?);` + queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName) + _, err = db.Exec(queryString, version, checksum) if err != nil { return err } @@ -344,13 +376,13 @@ func (*Driver) QueryError(query string, err error) error { return &dbmate.QueryError{Err: err, Query: query} } -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { config, err := drv.getConfig(db) if err != nil { return nil, err } - query := fmt.Sprintf("SELECT version FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) + query := fmt.Sprintf("SELECT version, checksum FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -360,14 +392,19 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + migrations[version] = checksum } if err = rows.Err(); err != nil { diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index 03025a85..fbc780f4 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -233,22 +233,22 @@ func TestBigQuerySelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestBigQueryInsertMigration(t *testing.T) { @@ -267,7 +267,7 @@ func TestBigQueryInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). @@ -353,9 +353,9 @@ func TestGoogleBigQueryDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema @@ -369,7 +369,7 @@ func TestGoogleBigQueryDumpSchema(t *testing.T) { "-- Dbmate schema migrations\n"+ "--\n\n"+ "INSERT INTO schema_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") }) } diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index b1945c4d..64c99e69 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -200,27 +200,57 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { migrationsTable := drv.quotedMigrationsTableName() // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select version from %s final ", migrationsTable)+ + rows, err := db.Query( + fmt.Sprintf("select version, checksum from %s final ", migrationsTable) + "where applied order by version asc", ) if err != nil { return err } + migrations := [][]string{} + for rows.Next() { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return err + } + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } + + if err := rows.Err(); err != nil { + return err + } + quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`) for i := range migrations { - migrations[i] = "'" + quoter.Replace(migrations[i]) + "'" + for j := range migrations[i] { + migrations[i][j] = "'" + quoter.Replace(migrations[i][j]) + "'" + } } // build schema migrations table data buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + - strings.Join(migrations, "),\n (") + - ");\n") + fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + strings.Join(tuples, ",\n ") + + ";\n") } return nil @@ -288,6 +318,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf(` create table if not exists %s%s ( version String, + checksum String, ts DateTime default now(), applied UInt8 default 1 ) engine = %s @@ -300,8 +331,8 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := fmt.Sprintf("select version from %s final where applied order by version desc", +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { + query := fmt.Sprintf("select version, checksum from %s final where applied order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { @@ -314,14 +345,20 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -332,10 +369,10 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, checksum) values (?, ?)", drv.quotedMigrationsTableName()), + version, checksum) return err } diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index b956200c..8e4da076 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -101,13 +101,13 @@ func TestClickHouseDumpSchemaOnCluster(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) tx, err = db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc2") + err = drv.InsertMigration(tx, "abc2", "checksum2") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -120,9 +120,9 @@ func TestClickHouseDumpSchemaOnCluster(t *testing.T) { require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO test_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO test_migrations (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -215,43 +215,43 @@ func TestClickHouseSelectMigrationsOnCluster(t *testing.T) { tx, err := db01.Begin() require.NoError(t, err) - stmt, err := tx.Prepare("insert into test_migrations (version) values (?)") + stmt, err := tx.Prepare("insert into test_migrations (version, checksum) values (?, ?)") require.NoError(t, err) - _, err = stmt.Exec("abc2") + _, err = stmt.Exec("abc2", nil) require.NoError(t, err) - _, err = stmt.Exec("abc1") + _, err = stmt.Exec("abc1", "checksum1") require.NoError(t, err) - _, err = stmt.Exec("abc3") + _, err = stmt.Exec("abc3", "checksum3") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) migrations01, err := drv01.SelectMigrations(db01, -1) require.NoError(t, err) - require.Equal(t, true, migrations01["abc1"]) - require.Equal(t, true, migrations01["abc2"]) - require.Equal(t, true, migrations01["abc3"]) + require.Equal(t, "checksum1", *migrations01["abc1"]) + require.Equal(t, "", *migrations01["abc2"]) + require.Equal(t, "checksum3", *migrations01["abc3"]) // Assert select on other node migrations02, err := drv02.SelectMigrations(db02, -1) require.NoError(t, err) - require.Equal(t, true, migrations02["abc1"]) - require.Equal(t, true, migrations02["abc2"]) - require.Equal(t, true, migrations02["abc3"]) + require.Equal(t, "checksum1", *migrations02["abc1"]) + require.Equal(t, "", *migrations02["abc2"]) + require.Equal(t, "checksum3", *migrations02["abc3"]) // test limit param migrations01, err = drv01.SelectMigrations(db01, 1) require.NoError(t, err) - require.Equal(t, true, migrations01["abc3"]) - require.Equal(t, false, migrations01["abc1"]) - require.Equal(t, false, migrations01["abc2"]) + require.Equal(t, "checksum3", *migrations01["abc3"]) + require.Equal(t, (*string)(nil), migrations01["abc2"]) + require.Equal(t, (*string)(nil), migrations01["abc1"]) // test limit param on other node migrations02, err = drv02.SelectMigrations(db02, 1) require.NoError(t, err) - require.Equal(t, true, migrations02["abc3"]) - require.Equal(t, false, migrations02["abc1"]) - require.Equal(t, false, migrations02["abc2"]) + require.Equal(t, "checksum3", *migrations02["abc3"]) + require.Equal(t, (*string)(nil), migrations02["abc2"]) + require.Equal(t, (*string)(nil), migrations02["abc1"]) } func TestClickHouseInsertMigrationOnCluster(t *testing.T) { @@ -282,7 +282,7 @@ func TestClickHouseInsertMigrationOnCluster(t *testing.T) { // insert migration tx, err := db01.Begin() require.NoError(t, err) - err = drv01.InsertMigration(tx, "abc1") + err = drv01.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index da9912e4..64e4e5ee 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -105,13 +105,13 @@ func TestClickHouseDumpSchema(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) tx, err = db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc2") + err = drv.InsertMigration(tx, "abc2", "checksum2") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -123,9 +123,9 @@ func TestClickHouseDumpSchema(t *testing.T) { require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO test_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO test_migrations (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -264,29 +264,29 @@ func TestClickHouseSelectMigrations(t *testing.T) { tx, err := db.Begin() require.NoError(t, err) - stmt, err := tx.Prepare("insert into test_migrations (version) values (?)") + stmt, err := tx.Prepare("insert into test_migrations (version, checksum) values (?,?)") require.NoError(t, err) - _, err = stmt.Exec("abc2") + _, err = stmt.Exec("abc2", nil) require.NoError(t, err) - _, err = stmt.Exec("abc1") + _, err = stmt.Exec("abc1", nil) require.NoError(t, err) - _, err = stmt.Exec("abc3") + _, err = stmt.Exec("abc3", "checksum3") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestClickHouseInsertMigration(t *testing.T) { @@ -307,7 +307,7 @@ func TestClickHouseInsertMigration(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "checksum1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 9d6c3dde..030839bc 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -131,7 +131,9 @@ func (drv *Driver) mysqldumpArgs() []string { // generate CLI arguments args := []string{"--opt", "--routines", "--no-data", "--skip-dump-date", "--skip-add-drop-table"} - + if tls := drv.databaseURL.Query().Get("tls"); tls == "false" { + args = append(args, "--skip-ssl") + } socket := drv.databaseURL.Query().Get("socket") if socket != "" { args = append(args, "--socket="+socket) @@ -161,22 +163,50 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) + rows, err := db.Query(fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable)) if err != nil { return nil, err } + migrations := [][]string{} + for rows.Next() { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } + // build schema_migrations table data var buf bytes.Buffer buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n" + fmt.Sprintf("LOCK TABLES %s WRITE;\n", migrationsTable)) if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + - strings.Join(migrations, "),\n (") + - ");\n") + fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + strings.Join(tuples, ",\n ") + + ";\n") } buf.WriteString("UNLOCK TABLES;\n") @@ -246,7 +276,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key)", + "create table if not exists %s (version varchar(128) primary key, checksum varchar(64))", drv.quotedMigrationsTableName())) return err @@ -254,8 +284,8 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { + query := fmt.Sprintf("select version, checksum from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -266,14 +296,20 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -284,10 +320,10 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, checksum) values (?, ?)", drv.quotedMigrationsTableName()), + version, checksum) return err } diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index 048b205f..0e8756ba 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -192,9 +192,9 @@ func TestMySQLDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema @@ -206,9 +206,9 @@ func TestMySQLDumpSchema(t *testing.T) { "-- Dbmate schema migrations\n"+ "--\n\n"+ "LOCK TABLES `test_migrations` WRITE;\n"+ - "INSERT INTO `test_migrations` (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n"+ + "INSERT INTO `test_migrations` (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n"+ "UNLOCK TABLES;\n") // DumpSchema should return error if command fails @@ -315,22 +315,22 @@ func TestMySQLSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestMySQLInsertMigration(t *testing.T) { @@ -349,7 +349,7 @@ func TestMySQLInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 76cefc42..068d0bc0 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -177,20 +177,48 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { } // load applied migrations - migrations, err := dbutil.QueryColumn(db, - "select quote_literal(version) from "+migrationsTable+" order by version asc") + rows, err := db.Query("select quote_literal(version), quote_literal(checksum) from " + migrationsTable + " order by version asc") if err != nil { return nil, err } + migrations := [][]string{} + for rows.Next() { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } + // build migrations table data var buf bytes.Buffer buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { - buf.WriteString("INSERT INTO " + migrationsTable + " (version) VALUES\n (" + - strings.Join(migrations, "),\n (") + - ");\n") + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } + } + buf.WriteString("INSERT INTO " + migrationsTable + " (version, checksum) VALUES\n " + + strings.Join(tuples, ",\n ") + + ";\n") } return buf.Bytes(), nil @@ -265,7 +293,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // first attempt at creating migrations table createTableStmt := fmt.Sprintf( - "create table if not exists %s.%s (version varchar primary key)", + "create table if not exists %s.%s (version varchar primary key, checksum varchar)", schema, migrationsTable) _, err = db.Exec(createTableStmt) if err == nil { @@ -293,15 +321,15 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { return err } -// SelectMigrations returns a list of applied migrations +// SelectMigrations returns a list of applied migrations and its checksum // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return nil, err } - query := "select version from " + migrationsTable + " order by version desc" + query := "select version, checksum from " + migrationsTable + " order by version desc" if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -312,14 +340,19 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -330,13 +363,13 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return err } - _, err = db.Exec("insert into "+migrationsTable+" (version) values ($1)", version) + _, err = db.Exec("insert into "+migrationsTable+" (version, checksum) values ($1, $2)", version, checksum) return err } diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index d2dc4709..74ff1231 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -217,9 +217,9 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema @@ -232,9 +232,9 @@ func TestPostgresDumpSchema(t *testing.T) { "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO public.schema_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO public.schema_migrations (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -255,9 +255,9 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // DumpSchema should return schema @@ -270,9 +270,9 @@ func TestPostgresDumpSchema(t *testing.T) { "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO \"camelSchema\".\"testMigrations\" (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO \"camelSchema\".\"testMigrations\" (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") }) } @@ -477,22 +477,22 @@ func TestPostgresSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into public.test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestPostgresInsertMigration(t *testing.T) { @@ -511,7 +511,7 @@ func TestPostgresInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from public.test_migrations where version = 'abc1'"). diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 12c19a4c..7b7ad0b4 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -108,21 +108,49 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() // load applied migrations - migrations, err := dbutil.QueryColumn(db, - fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) + rows, err := db.Query(fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable)) if err != nil { return nil, err } + migrations := [][]string{} + for rows.Next() { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } + + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } + } + + if err := rows.Err(); err != nil { + return nil, err + } + // build schema migrations table data var buf bytes.Buffer buf.WriteString("-- Dbmate schema migrations\n") if len(migrations) > 0 { + tuples := make([]string, 0, len(migrations)) + for _, m := range migrations { + v := m[0] + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + - strings.Join(migrations, "),\n (") + - ");\n") + fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + strings.Join(tuples, ",\n ") + + ";\n") } return buf.Bytes(), nil @@ -175,7 +203,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key)", + "create table if not exists %s (version varchar(128) primary key, checksum varchar(64))", drv.quotedMigrationsTableName())) return err @@ -183,8 +211,8 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { + query := fmt.Sprintf("select version, checksum from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -195,14 +223,19 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err defer dbutil.MustClose(rows) - migrations := map[string]bool{} + migrations := map[string]*string{} for rows.Next() { var version string - if err := rows.Scan(&version); err != nil { + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { return nil, err } - migrations[version] = true + if checksum == nil { + empty := "" + checksum = &empty + } + migrations[version] = checksum } if err = rows.Err(); err != nil { @@ -213,10 +246,10 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, checksum string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, checksum) values (?, ?)", drv.quotedMigrationsTableName()), + version, checksum) return err } diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index 87fffa6c..6ded6777 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -176,9 +176,9 @@ func TestSQLiteDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "checksum2") require.NoError(t, err) // create a table that will trigger `sqlite_sequence` system table @@ -191,9 +191,9 @@ func TestSQLiteDumpSchema(t *testing.T) { require.Contains(t, string(schema), "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT)") require.Contains(t, string(schema), "CREATE TABLE IF NOT EXISTS \"test_migrations\"") require.Contains(t, string(schema), ");\n-- Dbmate schema migrations\n"+ - "INSERT INTO \"test_migrations\" (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO \"test_migrations\" (version, checksum) VALUES\n"+ + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // sqlite_* tables should not be present in the dump (.schema --nosys) require.NotContains(t, string(schema), "sqlite_") @@ -290,22 +290,22 @@ func TestSQLiteSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, checksum) + values ('abc2', null), ('abc1', null), ('abc3', 'checksum3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) require.NoError(t, err) - require.Equal(t, true, migrations["abc1"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, true, migrations["abc2"]) + require.Equal(t, "", *migrations["abc1"]) + require.Equal(t, "", *migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) // test limit param migrations, err = drv.SelectMigrations(db, 1) require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, false, migrations["abc1"]) - require.Equal(t, false, migrations["abc2"]) + require.Equal(t, "checksum3", *migrations["abc3"]) + require.Equal(t, (*string)(nil), migrations["abc1"]) + require.Equal(t, (*string)(nil), migrations["abc2"]) } func TestSQLiteInsertMigration(t *testing.T) { @@ -324,7 +324,7 @@ func TestSQLiteInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "checksum1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). From b641e43744862987d9a0fcacc32c67d44482cfe8 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 16 Aug 2025 14:06:16 +0000 Subject: [PATCH 02/15] Add retro-compatibility --- pkg/dbmate/db.go | 16 ++++- pkg/dbmate/db_test.go | 43 ++++++++++++ pkg/dbmate/driver.go | 2 + pkg/driver/bigquery/bigquery.go | 102 +++++++++++++++++++++++++++- pkg/driver/clickhouse/clickhouse.go | 16 +++++ pkg/driver/mysql/mysql.go | 20 ++++++ pkg/driver/postgres/postgres.go | 34 ++++++++++ pkg/driver/sqlite/sqlite.go | 21 ++++++ 8 files changed, 250 insertions(+), 4 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 51555126..63bb47d6 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -471,6 +471,18 @@ func (db *DB) FindMigrations() ([]Migration, error) { } if migrationsTableExists { + hasChecksumColumn, err := drv.HasChecksumColumn(sqlDB) + if err != nil { + return nil, err + } + + if !hasChecksumColumn { + err = drv.AddChecksumColumn(sqlDB) + if err != nil { + return nil, err + } + } + appliedMigrations, err = drv.SelectMigrations(sqlDB, -1) if err != nil { return nil, err @@ -518,11 +530,11 @@ func (db *DB) FindMigrations() ([]Migration, error) { errMsg := fmt.Sprintf("The migration file `%s` has been modified since it was applied. Please ensure that the applied migrations are not modified afterwards.", migration.FileName) if db.ChecksumMode == ChecksumStrict { - return nil, errors.New(errMsg) + return nil, fmt.Errorf("%s%s%s", "\x1b[31m", errMsg, "\x1b[0m") } if db.ChecksumMode == ChecksumLenient { - fmt.Fprintf(db.Log, "Warning: %s\n", errMsg) + fmt.Fprintf(db.Log, "%sWarning: %s%s\n", "\x1b[33m", errMsg, "\x1b[0m") } } } diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 6298c564..71cbd3b4 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -2,6 +2,7 @@ package dbmate_test import ( "bytes" + "fmt" "net/url" "os" "path/filepath" @@ -769,6 +770,48 @@ DROP TABLE foo; }) } +func TestChecksumFeatureRetroCompatibility(t *testing.T) { + testEachURL(t, func(t *testing.T, u *url.URL) { + db := newTestDB(t, u) + db.AutoDumpSchema = false + + drv, err := db.Driver() + require.NoError(t, err) + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + err = drv.CreateMigrationsTable(sqlDB) + require.NoError(t, err) + + err = db.Migrate() + require.NoError(t, err) + + _, err = sqlDB.Exec(fmt.Sprintf("alter table %s drop column checksum", db.MigrationsTableName)) + require.NoError(t, err) + + pending, err := db.Status(false) + require.NoError(t, err) + require.Equal(t, 0, pending) + + migrations, err := drv.SelectMigrations(sqlDB, -1) + require.NoError(t, err) + require.NotNil(t, migrations["20200227231541"]) + require.NotNil(t, migrations["20151129054053"]) + // old migrations have no checksum set + require.Equal(t, "", *migrations["20200227231541"]) + require.Equal(t, "", *migrations["20151129054053"]) + }) +} + func TestMigrateUnrestrictedOrder(t *testing.T) { emptyMigration := []byte("-- migrate:up\n-- migrate:down") diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index 61501b09..988ea592 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -18,6 +18,8 @@ type Driver interface { DumpSchema(*sql.DB) ([]byte, error) MigrationsTableExists(*sql.DB) (bool, error) CreateMigrationsTable(*sql.DB) error + HasChecksumColumn(*sql.DB) (bool, error) + AddChecksumColumn(*sql.DB) error SelectMigrations(*sql.DB, int) (map[string]*string, error) InsertMigration(dbutil.Transaction, string, string) error DeleteMigration(dbutil.Transaction, string) error diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 0fb5cec2..415bc18e 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "errors" "fmt" "io" "net/url" @@ -98,14 +99,111 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { Type: bigquery.StringFieldType, }, &bigquery.FieldSchema{ - Name: "checksum", - Type: bigquery.StringFieldType, + Name: "checksum", + Type: bigquery.StringFieldType, + Required: false, }, }, }) }) } +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return false, err + } + defer conn.Close() + + err = conn.Raw(func(driverConn any) error { + client := getClient(driverConn) + config := getConfig(driverConn) + table := client.Dataset(config.dataSet).Table(drv.migrationsTableName) + meta, err := table.Metadata(context.Background()) + if err != nil { + return err + } + + for _, field := range meta.Schema { + if field.Name == "checksum" { + return nil + } + } + + return errors.New("column not found in table") + }) + + if err != nil { + if err.Error() != "column not found in table" { + return false, err + } + return false, nil + } + + return true, nil +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + + return conn.Raw(func(driverConn any) error { + client := getClient(driverConn) + config := getConfig(driverConn) + + exists, err := tableExists(client, config.dataSet, drv.migrationsTableName) + if err != nil { + return err + } + if !exists { + return errors.New("migrations table not found") + } + + table := client.Dataset(config.dataSet).Table(drv.migrationsTableName) + + meta, err := table.Metadata(context.Background()) + if err != nil { + return fmt.Errorf("failed to get table metadata: %w", err) + } + + for _, f := range meta.Schema { + if strings.EqualFold(f.Name, "checksum") { + return nil + } + } + + checksumField := &bigquery.FieldSchema{ + Name: "checksum", + Type: bigquery.StringFieldType, + Required: false, + } + + newSchema := append(meta.Schema, checksumField) + _, err = table.Update(ctx, bigquery.TableMetadataToUpdate{Schema: newSchema}, meta.ETag) + if err != nil { + return fmt.Errorf("table update failed: %w", err) + } + + meta2, merr := client.Dataset(config.dataSet).Table(drv.migrationsTableName).Metadata(ctx) + if merr == nil { + for _, f := range meta2.Schema { + if strings.EqualFold(f.Name, "checksum") { + return nil // success + } + } + return errors.New("new column not found") + } + + fmt.Printf("Column %q added successfully to table %s.%s\n", checksumField.Name, config.dataSet, drv.migrationsTableName) + return nil + }) +} + func (drv *Driver) DatabaseExists() (bool, error) { db, err := drv.Open() if err != nil { diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 64c99e69..2b5e3492 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -329,6 +329,22 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { return err } +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + exists := false + err := db.QueryRow(fmt.Sprintf("SELECT 1 FROM system.columns WHERE database = '%s' AND table = '%s' AND name = 'checksum'", drv.databaseName(), drv.quotedMigrationsTableName())). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s.%s ADD COLUMN checksum String", drv.databaseName(), drv.migrationsTableName)) + return err +} + // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 030839bc..eb2f6404 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -282,6 +282,26 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { return err } +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + exists := false + err := db.QueryRow(fmt.Sprintf("SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = '%s' AND column_name = 'checksum'", + drv.migrationsTableName)). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN checksum VARCHAR(64)", + drv.quotedMigrationsTableName())) + + return err +} + // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 068d0bc0..f7dcb729 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -321,6 +321,40 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { return err } +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + schema, migrationsTableNameParts, err := drv.migrationsTableNameParts(db) + if err != nil { + return false, err + } + + migrationsTable := strings.Join(migrationsTableNameParts, ".") + exists := false + err = db.QueryRow("SELECT 1 FROM information_schema.columns "+ + "WHERE table_schema = $1 "+ + "AND table_name = $2 "+ + "AND column_name = 'checksum'", + schema, migrationsTable). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + schema, migrationsTable, err := drv.quotedMigrationsTableNameParts(db) + if err != nil { + return err + } + + addColumnStmt := fmt.Sprintf( + "ALTER TABLE %s.%s ADD COLUMN IF NOT EXISTS checksum VARCHAR", + schema, migrationsTable) + _, err = db.Exec(addColumnStmt) + return err +} + // SelectMigrations returns a list of applied migrations and its checksum // with an optional limit (in descending order) func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 7b7ad0b4..87cefd74 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -209,6 +209,27 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { return err } +func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { + exists := false + err := db.QueryRow("SELECT 1 FROM sqlite_master "+ + "WHERE type='table' AND name=$1 AND sql LIKE '%%checksum%%'", + drv.migrationsTableName). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +func (drv *Driver) AddChecksumColumn(db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN checksum VARCHAR(64)", + drv.quotedMigrationsTableName())) + + return err +} + // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]*string, error) { From 47da91f53f271602c21e8493804c6714fd31657e Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 16 Aug 2025 15:20:00 +0000 Subject: [PATCH 03/15] Update README file --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 12721284..3fc6e144 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ The following options are available with all commands. You must use command line - `--strict` - fail if migrations would be applied out of order _(env: `DBMATE_STRICT`)_ - `--wait` - wait for the db to become available before executing the subsequent command _(env: `DBMATE_WAIT`)_ - `--wait-timeout 60s` - timeout for --wait flag _(env: `DBMATE_WAIT_TIMEOUT`)_ +- `--checksum-mode lenient` - specify checksum mode on applied migrations comparison, see [details about concept](#checksum-validation) _(env: `DBMATE_CHECKSUM_MODE`)_ ## Usage @@ -602,12 +603,26 @@ The table is very simple: ```sql CREATE TABLE IF NOT EXISTS schema_migrations ( - version VARCHAR(255) PRIMARY KEY + version VARCHAR(128) PRIMARY KEY + checksum VARCHAR(64) ) ``` You can customize the name of this table using the `--migrations-table` flag or `DBMATE_MIGRATIONS_TABLE` environment variable. +### Checksum validation + +Dbmate supports validating migration file integrity using checksums. When enabled, each migration's contents are hashed (SHA-256) and stored in the `schema_migrations` table. On subsequent runs, dbmate compares the current migration file's checksum with the stored value to detect any changes. + +You can configure checksum validation using the `DBMATE_CHECKSUM_MODE` environment variable or the `--checksum-mode` command-line option. Supported modes are: +- `NONE`: Disable checksum validation (default). +- `LENIENT`: Warn if a migration file has changed after being applied. +- `STRICT`: Fail if a migration file has changed after being applied. + +Whatever mode is defined, Dbmate will still record each migration file's hash in database. + +This feature helps ensure that applied migrations remain unchanged, improving database integrity and team collaboration by explicitly raising bad practices using database schema migration tool. + ## Alternatives Why another database schema migration tool? Dbmate was inspired by many other tools, primarily [Active Record Migrations](http://guides.rubyonrails.org/active_record_migrations.html), with the goals of being trivial to configure, and language & framework independent. Here is a comparison between dbmate and other popular migration tools. From 1961335af4a9fcadb442e797b0680416771c815d Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 16 Aug 2025 17:28:14 +0000 Subject: [PATCH 04/15] Make it platform resilient --- pkg/dbmate/checksum.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/dbmate/checksum.go b/pkg/dbmate/checksum.go index a72c222e..2f928fa0 100644 --- a/pkg/dbmate/checksum.go +++ b/pkg/dbmate/checksum.go @@ -1,6 +1,7 @@ package dbmate import ( + "bytes" "crypto/sha256" "encoding/hex" "errors" @@ -46,7 +47,9 @@ func ModeToString(m ChecksumMode) string { } // ComputeChecksum returns the hex SHA-256 of the supplied bytes. +// It is platform resilient, normalizing CRLF to LF func ComputeChecksum(b []byte) string { + b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) sum := sha256.Sum256(b) return hex.EncodeToString(sum[:]) } From c8ad9d762b4dd6de7d995d518da1679117c52a99 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 16 Aug 2025 17:53:12 +0000 Subject: [PATCH 05/15] Fix path definition for windows platform --- pkg/dbmate/db_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 71cbd3b4..fc21b0f1 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "os" + "path" "path/filepath" "regexp" "strings" @@ -686,7 +687,7 @@ func TestFindMigrationsChecksum(t *testing.T) { // prepare relDir := "migrations" fileName := "20250101000000_create_foo.sql" - fullKey := filepath.Join(relDir, fileName) + fullKey := path.Join(relDir, fileName) // Original file content that will be used to compute the DB-stored checksum. upSQLOriginal := `-- migrate:up From c709768b8dab40f8bd623c025258e86d103dce35 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Fri, 3 Oct 2025 20:26:13 +0000 Subject: [PATCH 06/15] Strip BOMs and replace stabilize checksum computation accross platforms --- pkg/dbmate/checksum.go | 11 ++++++++-- pkg/dbmate/checksum_test.go | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 pkg/dbmate/checksum_test.go diff --git a/pkg/dbmate/checksum.go b/pkg/dbmate/checksum.go index 2f928fa0..3aac500d 100644 --- a/pkg/dbmate/checksum.go +++ b/pkg/dbmate/checksum.go @@ -18,6 +18,8 @@ const ( var ErrUnknownChecksumMode = errors.New("unknown checksum mode") +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + // ParseChecksumMode parses environment/CLI strings to a ChecksumMode. // Accepted strings (case-insensitive): "NONE", "LENIENT", "STRICT". func ParseChecksumMode(s string) (ChecksumMode, error) { @@ -46,9 +48,14 @@ func ModeToString(m ChecksumMode) string { } } -// ComputeChecksum returns the hex SHA-256 of the supplied bytes. -// It is platform resilient, normalizing CRLF to LF +// ComputeChecksum computes a SHA256 checksum of the given bytes after +// canonicalizing text. We strip a leading UTF-8 BOM (if present) and normalize +// CRLF -> LF so checksums are stable across platforms. func ComputeChecksum(b []byte) string { + // strip UTF-8 BOM if present + b = bytes.TrimPrefix(b, utf8BOM) + + // normalize CRLF -> LF b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) sum := sha256.Sum256(b) return hex.EncodeToString(sum[:]) diff --git a/pkg/dbmate/checksum_test.go b/pkg/dbmate/checksum_test.go new file mode 100644 index 00000000..5d4ec804 --- /dev/null +++ b/pkg/dbmate/checksum_test.go @@ -0,0 +1,44 @@ +package dbmate + +import ( + "bytes" + "testing" +) + +func TestComputeChecksum_LFvsCRLF(t *testing.T) { + a := []byte("-- migrate:up\nCREATE TABLE foo (id INTEGER);\n") + b := []byte("-- migrate:up\r\nCREATE TABLE foo (id INTEGER);\r\n") + ha := ComputeChecksum(a) + hb := ComputeChecksum(b) + if ha != hb { + t.Fatalf("checksums differ for LF vs CRLF: %s != %s", ha, hb) + } +} + +func TestComputeChecksum_BOMStripped(t *testing.T) { + bom := []byte{0xEF, 0xBB, 0xBF} + body := []byte("-- migrate:up\nCREATE TABLE foo (id INTEGER);\n") + withBOM := append(bom, body...) + h1 := ComputeChecksum(body) + h2 := ComputeChecksum(withBOM) + if h1 != h2 { + t.Fatalf("checksums differ with/without BOM: %s != %s", h1, h2) + } +} + +func TestComputeChecksum_CRLFandBOM(t *testing.T) { + bom := []byte{0xEF, 0xBB, 0xBF} + lf := []byte("-- migrate:up\nCREATE TABLE foo (id INTEGER);\n") + crlf := bytes.ReplaceAll(lf, []byte("\n"), []byte("\r\n")) + hlf := ComputeChecksum(lf) + hcrlf := ComputeChecksum(crlf) + if hlf != hcrlf { + t.Fatalf("checksums differ for CRLF vs LF: %s != %s", hlf, hcrlf) + } + // BOM CRLF + withBOM := append(bom, crlf...) + h3 := ComputeChecksum(withBOM) + if h3 != hlf { + t.Fatalf("checksums differ for BOMCRLF vs LF: %s != %s", h3, hlf) + } +} From d50efe7827b4fe06ab963b1af8750a2ea4aa084f Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Fri, 3 Oct 2025 21:01:19 +0000 Subject: [PATCH 07/15] Replace clickhouse use of system database --- pkg/driver/clickhouse/clickhouse.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 2b5e3492..73832497 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -285,7 +285,7 @@ func (drv *Driver) DatabaseExists() (bool, error) { defer dbutil.MustClose(db) exists := false - err = db.QueryRow("SELECT 1 FROM system.databases where name = ?", name). + err = db.QueryRow(fmt.Sprintf("EXISTS DATABASE %s", name)). Scan(&exists) if err == sql.ErrNoRows { return false, nil @@ -331,7 +331,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { exists := false - err := db.QueryRow(fmt.Sprintf("SELECT 1 FROM system.columns WHERE database = '%s' AND table = '%s' AND name = 'checksum'", drv.databaseName(), drv.quotedMigrationsTableName())). + err := db.QueryRow(fmt.Sprintf("SHOW COLUMNS FROM %s.%s WHERE field = 'checksum'", drv.databaseName(), drv.migrationsTableName)). Scan(&exists) if err == sql.ErrNoRows { return false, nil From ffa7420a28f697223ee36ac8acc231a5ced23584 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 4 Oct 2025 08:00:59 +0000 Subject: [PATCH 08/15] Fix sonar issues on dockerfile --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0d1feee5..cc1e0458 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,19 @@ # development stage -FROM golang:1.24.6 as dev +FROM golang:1.24.6 AS dev WORKDIR /src ENV PATH="/src/typescript/node_modules/.bin:${PATH}" RUN git config --global --add safe.directory /src # install development tools RUN apt-get update \ - && apt-get install -qq --no-install-recommends \ + && apt-get -y install -qq --no-install-recommends \ curl \ file \ mariadb-client \ - postgresql-client \ - sqlite3 \ nodejs \ npm \ + postgresql-client \ + sqlite3 \ && rm -rf /var/lib/apt/lists/* # golangci-lint @@ -27,7 +27,7 @@ COPY . /src/ RUN make build # release stage -FROM alpine:3.22.1 as release +FROM alpine:3.22.1 AS release RUN apk add --no-cache \ mariadb-client \ mariadb-connector-c \ From 17c4a6c32210352e4b09f3ea1fc127324da52126 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 4 Oct 2025 09:48:20 +0000 Subject: [PATCH 09/15] Fix postgresql driver tests to ignore newer PostgreSQL toolchains' entries --- pkg/driver/postgres/postgres_test.go | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 74ff1231..cfe8ae39 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -226,15 +226,10 @@ func TestPostgresDumpSchema(t *testing.T) { schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations") - require.Contains(t, string(schema), "\n--\n"+ - "-- PostgreSQL database dump complete\n"+ - "--\n\n\n"+ - "--\n"+ - "-- Dbmate schema migrations\n"+ - "--\n\n"+ + require.Contains(t, string(schema), "INSERT INTO public.schema_migrations (version, checksum) VALUES\n"+ - " ('abc1', 'checksum1'),\n"+ - " ('abc2', 'checksum2');\n") + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -264,15 +259,10 @@ func TestPostgresDumpSchema(t *testing.T) { schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE \"camelSchema\".\"testMigrations\"") - require.Contains(t, string(schema), "\n--\n"+ - "-- PostgreSQL database dump complete\n"+ - "--\n\n\n"+ - "--\n"+ - "-- Dbmate schema migrations\n"+ - "--\n\n"+ + require.Contains(t, string(schema), "INSERT INTO \"camelSchema\".\"testMigrations\" (version, checksum) VALUES\n"+ - " ('abc1', 'checksum1'),\n"+ - " ('abc2', 'checksum2');\n") + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") }) } From 2800ac6b8269c468d336a7f52d7f3f68861603b0 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 4 Oct 2025 09:53:39 +0000 Subject: [PATCH 10/15] Move lenient as the default checksum mode --- README.md | 4 ++-- pkg/dbmate/checksum.go | 6 +++--- pkg/dbmate/db.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3fc6e144..fbd0b7f4 100644 --- a/README.md +++ b/README.md @@ -615,8 +615,8 @@ You can customize the name of this table using the `--migrations-table` flag or Dbmate supports validating migration file integrity using checksums. When enabled, each migration's contents are hashed (SHA-256) and stored in the `schema_migrations` table. On subsequent runs, dbmate compares the current migration file's checksum with the stored value to detect any changes. You can configure checksum validation using the `DBMATE_CHECKSUM_MODE` environment variable or the `--checksum-mode` command-line option. Supported modes are: -- `NONE`: Disable checksum validation (default). -- `LENIENT`: Warn if a migration file has changed after being applied. +- `NONE`: Disable checksum validation. +- `LENIENT`: Warn if a migration file has changed after being applied (default). - `STRICT`: Fail if a migration file has changed after being applied. Whatever mode is defined, Dbmate will still record each migration file's hash in database. diff --git a/pkg/dbmate/checksum.go b/pkg/dbmate/checksum.go index 3aac500d..72f4072a 100644 --- a/pkg/dbmate/checksum.go +++ b/pkg/dbmate/checksum.go @@ -24,14 +24,14 @@ var utf8BOM = []byte{0xEF, 0xBB, 0xBF} // Accepted strings (case-insensitive): "NONE", "LENIENT", "STRICT". func ParseChecksumMode(s string) (ChecksumMode, error) { switch strings.ToUpper(strings.TrimSpace(s)) { - case "", "NONE": + case "NONE": return ChecksumNone, nil - case "LENIENT": + case "", "LENIENT": return ChecksumLenient, nil case "STRICT": return ChecksumStrict, nil default: - return ChecksumNone, ErrUnknownChecksumMode + return ChecksumLenient, ErrUnknownChecksumMode } } diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 63bb47d6..9fe23b09 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -85,7 +85,7 @@ func New(databaseURL *url.URL) *DB { WaitBefore: false, WaitInterval: time.Second, WaitTimeout: 60 * time.Second, - ChecksumMode: ChecksumNone, + ChecksumMode: ChecksumLenient, } } From ee627bc1672e4f394f272791c7f5fbc6fd5f2f8d Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 31 Jan 2026 10:29:45 +0000 Subject: [PATCH 11/15] fix issues reported by Cursor Bugbot --- pkg/driver/bigquery/bigquery.go | 66 +++++++++++++++++++++------- pkg/driver/bigquery/bigquery_test.go | 6 +-- pkg/driver/clickhouse/clickhouse.go | 43 +++++++++--------- pkg/driver/mysql/mysql.go | 1 + pkg/driver/postgres/postgres.go | 1 + pkg/driver/postgres/postgres_test.go | 9 ++-- pkg/driver/sqlite/sqlite.go | 1 + 7 files changed, 83 insertions(+), 44 deletions(-) diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 415bc18e..9aad469b 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -199,7 +199,12 @@ func (drv *Driver) AddChecksumColumn(db *sql.DB) error { return errors.New("new column not found") } - fmt.Printf("Column %q added successfully to table %s.%s\n", checksumField.Name, config.dataSet, drv.migrationsTableName) + // Verification failed, but table update succeeded. + // Log success message with warning about verification failure. + if drv.log != nil { + fmt.Fprintf(drv.log, "Column %q added successfully to table %s.%s (verification failed: %v)\n", + checksumField.Name, config.dataSet, drv.migrationsTableName, merr) + } return nil }) } @@ -325,24 +330,47 @@ func (drv *Driver) schemaDump(db *sql.DB) ([]byte, error) { func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.migrationsTableName + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select version, checksum from %s order by version asc", migrationsTable) + } else { + query = fmt.Sprintf("select version from %s order by version asc", migrationsTable) + } + // load applied migrations - rows, err := db.Query(fmt.Sprintf("select version from %s order by version asc", migrationsTable)) + rows, err := db.Query(query) if err != nil { return nil, err } + defer dbutil.MustClose(rows) migrations := [][]string{} for rows.Next() { - var version string - var checksum *string - if err := rows.Scan(&version, &checksum); err != nil { - return nil, err - } + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } - if checksum == nil { - migrations = append(migrations, []string{version, ""}) + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } } else { - migrations = append(migrations, []string{version, *checksum}) + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) } } @@ -358,15 +386,23 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { tuples := make([]string, 0, len(migrations)) for _, m := range migrations { v := m[0] - c := m[1] - if c == "" { - tuples = append(tuples, fmt.Sprintf("('%s', NULL)", v)) + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("('%s', NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("('%s','%s')", v, c)) + } } else { - tuples = append(tuples, fmt.Sprintf("('%s','%s')", v, c)) + tuples = append(tuples, fmt.Sprintf("('%s')", v)) } } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + fmt.Sprintf("INSERT INTO %s (%s) VALUES\n ", migrationsTable, columns) + strings.Join(tuples, ",\n ") + ";\n") } diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index fbc780f4..a318052a 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -368,8 +368,8 @@ func TestGoogleBigQueryDumpSchema(t *testing.T) { require.Contains(t, string(schema), "\n--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO schema_migrations (version) VALUES\n"+ - " ('abc1', 'checksum1'),\n"+ - " ('abc2', 'checksum2');\n") + "INSERT INTO schema_migrations (version, checksum) VALUES\n"+ + " ('abc1','checksum1'),\n"+ + " ('abc2','checksum2');\n") }) } diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 73832497..97cb6056 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -207,19 +207,20 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { if err != nil { return err } + defer dbutil.MustClose(rows) - migrations := [][]string{} + type migration struct { + version string + checksum *string + } + migrations := []migration{} for rows.Next() { var version string var checksum *string if err := rows.Scan(&version, &checksum); err != nil { return err } - if checksum == nil { - migrations = append(migrations, []string{version, ""}) - } else { - migrations = append(migrations, []string{version, *checksum}) - } + migrations = append(migrations, migration{version, checksum}) } if err := rows.Err(); err != nil { @@ -227,24 +228,20 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { } quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`) - for i := range migrations { - for j := range migrations[i] { - migrations[i][j] = "'" + quoter.Replace(migrations[i][j]) + "'" - } - } - // build schema migrations table data buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { tuples := make([]string, 0, len(migrations)) for _, m := range migrations { - v := m[0] - c := m[1] - if c == "" { - tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + // quote version (always non-NULL) + quotedVersion := "'" + quoter.Replace(m.version) + "'" + + if m.checksum == nil { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", quotedVersion)) } else { - tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + quotedChecksum := "'" + quoter.Replace(*m.checksum) + "'" + tuples = append(tuples, fmt.Sprintf("(%s, %s)", quotedVersion, quotedChecksum)) } } buf.WriteString( @@ -330,14 +327,16 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { } func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { - exists := false - err := db.QueryRow(fmt.Sprintf("SHOW COLUMNS FROM %s.%s WHERE field = 'checksum'", drv.databaseName(), drv.migrationsTableName)). - Scan(&exists) + var dummy int + err := db.QueryRow(fmt.Sprintf("SELECT 1 FROM system.columns WHERE database = '%s' AND table = '%s' AND name = 'checksum'", drv.databaseName(), drv.migrationsTableName)). + Scan(&dummy) if err == sql.ErrNoRows { return false, nil } - - return exists, err + if err != nil { + return false, err + } + return true, nil } func (drv *Driver) AddChecksumColumn(db *sql.DB) error { diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 336a7f98..f0dd64c8 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -250,6 +250,7 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { if err != nil { return nil, err } + defer dbutil.MustClose(rows) migrations := [][]string{} for rows.Next() { diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 8685bd5e..9cfee6c7 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -230,6 +230,7 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { if err != nil { return nil, err } + defer dbutil.MustClose(rows) migrations := [][]string{} for rows.Next() { diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 10970fa6..0c948256 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -274,7 +274,7 @@ func TestPostgresDumpSchema(t *testing.T) { schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations") - + require.Contains(t, string(schema), "\n--\n"+ "-- PostgreSQL database dump complete\n"+ "--\n\n") @@ -315,10 +315,11 @@ func TestPostgresDumpSchema(t *testing.T) { require.Contains(t, string(schema), "\n--\n"+ "-- PostgreSQL database dump complete\n"+ "--\n\n") - require.Contains(t, string(schema), + require.Contains(t, string(schema), "-- Dbmate schema migrations\n"+ + "--\n\n"+ "INSERT INTO \"camelSchema\".\"testMigrations\" (version, checksum) VALUES\n"+ - " ('abc1', 'checksum1'),\n"+ - " ('abc2', 'checksum2');\n") + " ('abc1', 'checksum1'),\n"+ + " ('abc2', 'checksum2');\n") }) } diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 87cefd74..0b1bf192 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -112,6 +112,7 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { if err != nil { return nil, err } + defer dbutil.MustClose(rows) migrations := [][]string{} for rows.Next() { From 0366dae4f0ae81fcae29e1e4c1f3b1c6943b98d9 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 31 Jan 2026 10:40:43 +0000 Subject: [PATCH 12/15] fix format --- pkg/driver/clickhouse/clickhouse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 97cb6056..296f8964 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -210,7 +210,7 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { defer dbutil.MustClose(rows) type migration struct { - version string + version string checksum *string } migrations := []migration{} From 41c7fecf3b4b2bd7a0a25a8e83bbef84ec64d82f Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 31 Jan 2026 10:51:17 +0000 Subject: [PATCH 13/15] Fix sync-go-toolchain workflow for fork PRs - Checkout using PR head SHA instead of head_ref to work with forks - Only push changes when PR is from same repository (internal) - This should allow the sync job to run without checkout errors --- .github/workflows/sync-go-toolchain.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-go-toolchain.yml b/.github/workflows/sync-go-toolchain.yml index 083a4b4a..b507c13d 100644 --- a/.github/workflows/sync-go-toolchain.yml +++ b/.github/workflows/sync-go-toolchain.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v6 with: - ref: ${{ github.head_ref }} + ref: ${{ github.event.pull_request.head.sha }} - uses: actions/setup-go@v6 with: @@ -43,7 +43,7 @@ jobs: go mod edit -toolchain="go${NEW_VERSION}" - name: Commit and push changes - if: steps.gomod.outputs.toolchain != format('go{0}', steps.dockerfile.outputs.version) + if: steps.gomod.outputs.toolchain != format('go{0}', steps.dockerfile.outputs.version) && github.event.pull_request.head.repo.full_name == github.repository env: NEW_VERSION: ${{ steps.dockerfile.outputs.version }} run: | From bf22061eb8f9e3806dcd6fcffa918bc5c8595837 Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 31 Jan 2026 11:46:48 +0000 Subject: [PATCH 14/15] fix issues reported by Cursor Bugbot --- main.go | 9 +++++ pkg/driver/clickhouse/clickhouse.go | 52 ++++++++++++++++++------ pkg/driver/mysql/mysql.go | 58 ++++++++++++++++++++------- pkg/driver/postgres/postgres.go | 58 ++++++++++++++++++++------- pkg/driver/sqlite/sqlite.go | 61 +++++++++++++++++++++-------- 5 files changed, 182 insertions(+), 56 deletions(-) diff --git a/main.go b/main.go index f5454eb3..cb1550dc 100644 --- a/main.go +++ b/main.go @@ -132,6 +132,9 @@ func NewApp() *cli.App { Action: action(func(db *dbmate.DB, c *cli.Context) error { db.Strict = c.Bool("strict") db.Verbose = c.Bool("verbose") + if db.Verbose { + fmt.Fprintf(db.Log, "Checksum mode: %s\n", dbmate.ModeToString(db.ChecksumMode)) + } return db.CreateAndMigrate() }), }, @@ -168,6 +171,9 @@ func NewApp() *cli.App { Action: action(func(db *dbmate.DB, c *cli.Context) error { db.Strict = c.Bool("strict") db.Verbose = c.Bool("verbose") + if db.Verbose { + fmt.Fprintf(db.Log, "Checksum mode: %s\n", dbmate.ModeToString(db.ChecksumMode)) + } return db.Migrate() }), }, @@ -185,6 +191,9 @@ func NewApp() *cli.App { }, Action: action(func(db *dbmate.DB, c *cli.Context) error { db.Verbose = c.Bool("verbose") + if db.Verbose { + fmt.Fprintf(db.Log, "Checksum mode: %s\n", dbmate.ModeToString(db.ChecksumMode)) + } return db.Rollback() }), }, diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 296f8964..a4c418f0 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -199,11 +199,24 @@ func (drv *Driver) schemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { migrationsTable := drv.quotedMigrationsTableName() + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select version, checksum from %s final ", migrationsTable) + + "where applied order by version asc" + } else { + query = fmt.Sprintf("select version from %s final ", migrationsTable) + + "where applied order by version asc" + } + // load applied migrations - rows, err := db.Query( - fmt.Sprintf("select version, checksum from %s final ", migrationsTable) + - "where applied order by version asc", - ) + rows, err := db.Query(query) if err != nil { return err } @@ -215,12 +228,20 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { } migrations := []migration{} for rows.Next() { - var version string - var checksum *string - if err := rows.Scan(&version, &checksum); err != nil { - return err + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return err + } + migrations = append(migrations, migration{version, checksum}) + } else { + var version string + if err := rows.Scan(&version); err != nil { + return err + } + migrations = append(migrations, migration{version, nil}) } - migrations = append(migrations, migration{version, checksum}) } if err := rows.Err(); err != nil { @@ -237,15 +258,21 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { // quote version (always non-NULL) quotedVersion := "'" + quoter.Replace(m.version) + "'" - if m.checksum == nil { + if !hasChecksumColumn { + tuples = append(tuples, fmt.Sprintf("(%s)", quotedVersion)) + } else if m.checksum == nil { tuples = append(tuples, fmt.Sprintf("(%s, NULL)", quotedVersion)) } else { quotedChecksum := "'" + quoter.Replace(*m.checksum) + "'" tuples = append(tuples, fmt.Sprintf("(%s, %s)", quotedVersion, quotedChecksum)) } } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + fmt.Sprintf("INSERT INTO %s (%s) VALUES\n ", migrationsTable, columns) + strings.Join(tuples, ",\n ") + ";\n") } @@ -328,7 +355,8 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { var dummy int - err := db.QueryRow(fmt.Sprintf("SELECT 1 FROM system.columns WHERE database = '%s' AND table = '%s' AND name = 'checksum'", drv.databaseName(), drv.migrationsTableName)). + err := db.QueryRow("SELECT 1 FROM system.columns WHERE database = ? AND table = ? AND name = 'checksum'", + drv.databaseName(), drv.migrationsTableName). Scan(&dummy) if err == sql.ErrNoRows { return false, nil diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index f0dd64c8..043218d9 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -245,8 +245,22 @@ func (drv *Driver) mysqldumpArgs(ver *mysqldumpVersion) []string { func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable) + } else { + query = fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable) + } + // load applied migrations - rows, err := db.Query(fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable)) + rows, err := db.Query(query) if err != nil { return nil, err } @@ -254,16 +268,24 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrations := [][]string{} for rows.Next() { - var version string - var checksum *string - if err := rows.Scan(&version, &checksum); err != nil { - return nil, err - } + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } - if checksum == nil { - migrations = append(migrations, []string{version, ""}) + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } } else { - migrations = append(migrations, []string{version, *checksum}) + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) } } @@ -280,15 +302,23 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { tuples := make([]string, 0, len(migrations)) for _, m := range migrations { v := m[0] - c := m[1] - if c == "" { - tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } } else { - tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + tuples = append(tuples, fmt.Sprintf("(%s)", v)) } } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + fmt.Sprintf("INSERT INTO %s (%s) VALUES\n ", migrationsTable, columns) + strings.Join(tuples, ",\n ") + ";\n") } diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 9cfee6c7..2df7dc24 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -225,8 +225,22 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { return nil, err } + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = "select quote_literal(version), quote_literal(checksum) from " + migrationsTable + " order by version asc" + } else { + query = "select quote_literal(version) from " + migrationsTable + " order by version asc" + } + // load applied migrations - rows, err := db.Query("select quote_literal(version), quote_literal(checksum) from " + migrationsTable + " order by version asc") + rows, err := db.Query(query) if err != nil { return nil, err } @@ -234,16 +248,24 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrations := [][]string{} for rows.Next() { - var version string - var checksum *string - if err := rows.Scan(&version, &checksum); err != nil { - return nil, err - } + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } - if checksum == nil { - migrations = append(migrations, []string{version, ""}) + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } } else { - migrations = append(migrations, []string{version, *checksum}) + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) } } @@ -259,14 +281,22 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { tuples := make([]string, 0, len(migrations)) for _, m := range migrations { v := m[0] - c := m[1] - if c == "" { - tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } } else { - tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + tuples = append(tuples, fmt.Sprintf("(%s)", v)) } } - buf.WriteString("INSERT INTO " + migrationsTable + " (version, checksum) VALUES\n " + + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } + buf.WriteString("INSERT INTO " + migrationsTable + " (" + columns + ") VALUES\n " + strings.Join(tuples, ",\n ") + ";\n") } diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 0b1bf192..dd575af4 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -107,8 +107,22 @@ func (drv *Driver) DropDatabase() error { func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() + // check if checksum column exists + hasChecksumColumn, err := drv.HasChecksumColumn(db) + if err != nil { + return nil, err + } + + // build query based on column existence + var query string + if hasChecksumColumn { + query = fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable) + } else { + query = fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable) + } + // load applied migrations - rows, err := db.Query(fmt.Sprintf("select quote(version), quote(checksum) from %s order by version asc", migrationsTable)) + rows, err := db.Query(query) if err != nil { return nil, err } @@ -116,16 +130,24 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrations := [][]string{} for rows.Next() { - var version string - var checksum *string - if err := rows.Scan(&version, &checksum); err != nil { - return nil, err - } + if hasChecksumColumn { + var version string + var checksum *string + if err := rows.Scan(&version, &checksum); err != nil { + return nil, err + } - if checksum == nil { - migrations = append(migrations, []string{version, ""}) + if checksum == nil { + migrations = append(migrations, []string{version, ""}) + } else { + migrations = append(migrations, []string{version, *checksum}) + } } else { - migrations = append(migrations, []string{version, *checksum}) + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + migrations = append(migrations, []string{version}) } } @@ -141,15 +163,23 @@ func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { tuples := make([]string, 0, len(migrations)) for _, m := range migrations { v := m[0] - c := m[1] - if c == "" { - tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + if hasChecksumColumn { + c := m[1] + if c == "" { + tuples = append(tuples, fmt.Sprintf("(%s, NULL)", v)) + } else { + tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + } } else { - tuples = append(tuples, fmt.Sprintf("(%s, %s)", v, c)) + tuples = append(tuples, fmt.Sprintf("(%s)", v)) } } + columns := "version" + if hasChecksumColumn { + columns = "version, checksum" + } buf.WriteString( - fmt.Sprintf("INSERT INTO %s (version, checksum) VALUES\n ", migrationsTable) + + fmt.Sprintf("INSERT INTO %s (%s) VALUES\n ", migrationsTable, columns) + strings.Join(tuples, ",\n ") + ";\n") } @@ -212,8 +242,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { exists := false - err := db.QueryRow("SELECT 1 FROM sqlite_master "+ - "WHERE type='table' AND name=$1 AND sql LIKE '%%checksum%%'", + err := db.QueryRow("SELECT 1 FROM pragma_table_info(?) WHERE name = 'checksum'", drv.migrationsTableName). Scan(&exists) if err == sql.ErrNoRows { From c58a3a9ff56806671e606de1e2bfa0927c0ab50e Mon Sep 17 00:00:00 2001 From: Thomas VOISIN Date: Sat, 31 Jan 2026 13:54:57 +0000 Subject: [PATCH 15/15] Fix lack of ON CLUSTER clause in AddChecksumColumn for clickhouse --- pkg/driver/clickhouse/clickhouse.go | 16 ++++- .../clickhouse/clickhouse_cluster_test.go | 62 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index a4c418f0..7c575c4f 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -309,7 +309,7 @@ func (drv *Driver) DatabaseExists() (bool, error) { defer dbutil.MustClose(db) exists := false - err = db.QueryRow(fmt.Sprintf("EXISTS DATABASE %s", name)). + err = db.QueryRow(fmt.Sprintf("EXISTS DATABASE %s", drv.quoteIdentifier(name))). Scan(&exists) if err == sql.ErrNoRows { return false, nil @@ -368,8 +368,18 @@ func (drv *Driver) HasChecksumColumn(db *sql.DB) (bool, error) { } func (drv *Driver) AddChecksumColumn(db *sql.DB) error { - _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s.%s ADD COLUMN checksum String", drv.databaseName(), drv.migrationsTableName)) - return err + qualifiedTableName := drv.quoteIdentifier(drv.databaseName()) + "." + drv.quotedMigrationsTableName() + query := fmt.Sprintf("ALTER TABLE %s%s ADD COLUMN checksum String", qualifiedTableName, drv.onClusterClause()) + _, err := db.Exec(query) + if err != nil { + // If column already exists (duplicate column error), ignore it + // This can happen in cluster setups due to race conditions + if chErr, ok := err.(*clickhouse.Exception); ok && chErr.Code == 15 { + return nil + } + return err + } + return nil } // SelectMigrations returns a list of applied migrations diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index 8e4da076..ca961366 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -2,6 +2,7 @@ package clickhouse import ( "database/sql" + "fmt" "testing" "github.com/amacneil/dbmate/v2/pkg/dbmate" @@ -339,3 +340,64 @@ func TestClickHouseDeleteMigrationOnCluster(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, count02) } + +func TestClickHouseAddChecksumColumnOnClusterNonReplicated(t *testing.T) { + drv01 := testClickHouseDriverCluster01(t) + drv02 := testClickHouseDriverCluster02(t) + // Use a distinct table name to avoid collisions + tableName := "test_migrations_nonrepl" + drv01.migrationsTableName = tableName + drv02.migrationsTableName = tableName + + db01 := prepTestClickHouseDB(t, drv01) + defer dbutil.MustClose(db01) + + db02 := prepTestClickHouseDB(t, drv02) + defer dbutil.MustClose(db02) + + // Create migrations table WITHOUT checksum column, using non-replicated engine + // Even though OnCluster is true, we use ReplacingMergeTree (non-replicated) + // to test that ON CLUSTER clause is needed for DDL propagation across nodes. + engineClause := "ReplacingMergeTree(ts)" + createTableSQL := fmt.Sprintf(` + create table if not exists %s%s ( + version String, + ts DateTime default now(), + applied UInt8 default 1 + ) engine = %s + primary key version + order by version + `, drv01.quotedMigrationsTableName(), drv01.onClusterClause(), engineClause) + _, err := db01.Exec(createTableSQL) + require.NoError(t, err) + + // verify table exists on both nodes (because of ON CLUSTER clause) + exists, err := drv01.MigrationsTableExists(db01) + require.NoError(t, err) + require.True(t, exists) + exists, err = drv02.MigrationsTableExists(db02) + require.NoError(t, err) + require.True(t, exists) + + // verify checksum column does not exist initially + hasChecksum, err := drv01.HasChecksumColumn(db01) + require.NoError(t, err) + require.False(t, hasChecksum) + hasChecksum, err = drv02.HasChecksumColumn(db02) + require.NoError(t, err) + require.False(t, hasChecksum) + + // add checksum column + err = drv01.AddChecksumColumn(db01) + require.NoError(t, err) + + // verify checksum column exists on node1 (where ALTER TABLE executed) + hasChecksum, err = drv01.HasChecksumColumn(db01) + require.NoError(t, err) + require.True(t, hasChecksum, "checksum column should exist on node1") + + // verify checksum column exists on node2 (because of ON CLUSTER clause) + hasChecksum, err = drv02.HasChecksumColumn(db02) + require.NoError(t, err) + require.True(t, hasChecksum, "checksum column should exist on node2") +}