From e2f749b8fc21cfa0e7b7e8a5bb664a839c2a1ddc Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 10:43:55 +0100 Subject: [PATCH 01/46] Add dumping the entire applied migrations as a second column of schema_migrations table. --- pkg/dbmate/db.go | 2 +- pkg/dbmate/driver.go | 2 +- pkg/driver/bigquery/bigquery.go | 2 +- pkg/driver/bigquery/bigquery_test.go | 6 +++--- pkg/driver/clickhouse/clickhouse.go | 2 +- pkg/driver/clickhouse/clickhouse_cluster_test.go | 6 +++--- pkg/driver/clickhouse/clickhouse_test.go | 6 +++--- pkg/driver/mysql/mysql.go | 6 +++--- pkg/driver/mysql/mysql_test.go | 6 +++--- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/postgres/postgres_test.go | 10 +++++----- pkg/driver/sqlite/sqlite.go | 2 +- pkg/driver/sqlite/sqlite_test.go | 2 +- 13 files changed, 27 insertions(+), 27 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 4f5c39e2..bc4d7a22 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -394,7 +394,7 @@ func (db *DB) Migrate() error { } // record migration - return drv.InsertMigration(tx, migration.Version) + return drv.InsertMigration(tx, migration.Version, parsed.Up) } if parsed.UpOptions.Transaction() { diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index b468bcef..9f642ddf 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -19,7 +19,7 @@ type Driver interface { MigrationsTableExists(*sql.DB) (bool, error) CreateMigrationsTable(*sql.DB) error SelectMigrations(*sql.DB, int) (map[string]bool, error) - InsertMigration(dbutil.Transaction, string) error + InsertMigration(dbutil.Transaction, string, string) error DeleteMigration(dbutil.Transaction, string) error Ping() error QueryError(string, error) error diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index c477baca..1a859181 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -304,7 +304,7 @@ func (drv *Driver) DeleteMigration(util dbutil.Transaction, version string) erro return nil } -func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string, dump string) error { db, err := drv.Open() if err != nil { return err diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index 03025a85..bcd0854a 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -267,7 +267,7 @@ func TestBigQueryInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). @@ -353,9 +353,9 @@ func TestGoogleBigQueryDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "abc2") require.NoError(t, err) // DumpSchema should return schema diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 1878f5bd..59a1b759 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -332,7 +332,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), version) diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index b956200c..87dbe54e 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -101,13 +101,13 @@ func TestClickHouseDumpSchemaOnCluster(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "abc1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) tx, err = db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc2") + err = drv.InsertMigration(tx, "abc2", "abc2") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -282,7 +282,7 @@ func TestClickHouseInsertMigrationOnCluster(t *testing.T) { // insert migration tx, err := db01.Begin() require.NoError(t, err) - err = drv01.InsertMigration(tx, "abc1") + err = drv01.InsertMigration(tx, "abc1", "abc1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index da9912e4..08d263cb 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -105,13 +105,13 @@ func TestClickHouseDumpSchema(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "abc1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) tx, err = db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc2") + err = drv.InsertMigration(tx, "abc2", "abc2") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -307,7 +307,7 @@ func TestClickHouseInsertMigration(t *testing.T) { // insert migration tx, err := db.Begin() require.NoError(t, err) - err = drv.InsertMigration(tx, "abc1") + err = drv.InsertMigration(tx, "abc1", "abc1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index f2d508d5..32b55236 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,7 +246,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key)", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext)", drv.quotedMigrationsTableName())) return err @@ -284,10 +284,10 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + version, dump) return err } diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index 048b205f..bf1a7e80 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -192,9 +192,9 @@ func TestMySQLDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "abc2") require.NoError(t, err) // DumpSchema should return schema @@ -349,7 +349,7 @@ func TestMySQLInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 7b3e4293..4cba4006 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -330,7 +330,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return err diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index d2dc4709..0e7ccdab 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -217,9 +217,9 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "abc2") require.NoError(t, err) // DumpSchema should return schema @@ -255,9 +255,9 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "abc2") require.NoError(t, err) // DumpSchema should return schema @@ -511,7 +511,7 @@ func TestPostgresInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) err = db.QueryRow("select count(*) from public.test_migrations where version = 'abc1'"). diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 12c19a4c..66559947 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -213,7 +213,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err } // InsertMigration adds a new migration record -func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), version) diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index 87fffa6c..b9571cd8 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -324,7 +324,7 @@ func TestSQLiteInsertMigration(t *testing.T) { require.Equal(t, 0, count) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). From ce6d66a4b8edbcfcb70318019f00f8d7e2c742fd Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 10:48:47 +0100 Subject: [PATCH 02/46] Add missing arguments in sql lite testing. --- pkg/driver/sqlite/sqlite_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index b9571cd8..5381852f 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -176,9 +176,9 @@ func TestSQLiteDumpSchema(t *testing.T) { require.NoError(t, err) // insert migration - err = drv.InsertMigration(db, "abc1") + err = drv.InsertMigration(db, "abc1", "abc1") require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") + err = drv.InsertMigration(db, "abc2", "abc2") require.NoError(t, err) // create a table that will trigger `sqlite_sequence` system table From eb35cf4cf815f729bc10b24678843c0f647da542 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 11:05:34 +0100 Subject: [PATCH 03/46] Add missing argument handling in sql. --- pkg/driver/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 32b55236..3148a3cb 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -286,7 +286,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // InsertMigration adds a new migration record func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), + fmt.Sprintf("insert into %s (version, dump) values (?, ?)", drv.quotedMigrationsTableName()), version, dump) return err From c757b90e89551b12af6dabf6661334788220dc73 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 11:24:38 +0100 Subject: [PATCH 04/46] Add recording both up and down migrations in dump column. --- pkg/dbmate/db.go | 2 +- pkg/driver/bigquery/bigquery.go | 4 ++-- pkg/driver/clickhouse/clickhouse.go | 4 ++-- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/sqlite/sqlite.go | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index bc4d7a22..7d264f42 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -394,7 +394,7 @@ func (db *DB) Migrate() error { } // record migration - return drv.InsertMigration(tx, migration.Version, parsed.Up) + return drv.InsertMigration(tx, migration.Version, fmt.Sprintf("%s\n\n%s", parsed.Up, parsed.Down)) } if parsed.UpOptions.Transaction() { diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 1a859181..fd30eebf 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -316,8 +316,8 @@ func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string, dump st return err } - queryTemplate := `INSERT INTO %s.%s (version) VALUES ('%s');` - queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName, version) + queryTemplate := `INSERT INTO %s.%s (version, dump) VALUES ('%s','%s');` + queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName, version, dump) _, err = db.Exec(queryString, version) if err != nil { return err diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 59a1b759..537f943a 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -334,8 +334,8 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // InsertMigration adds a new migration record func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, dump) values (?, ?)", drv.quotedMigrationsTableName()), + version, dump) return err } diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 4cba4006..ef30febe 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -336,7 +336,7 @@ func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump s return err } - _, err = db.Exec("insert into "+migrationsTable+" (version) values ($1)", version) + _, err = db.Exec("insert into "+migrationsTable+" (version, dump) values ($1, $2)", version, dump) return err } diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 66559947..2fbe1d2c 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -215,8 +215,8 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // InsertMigration adds a new migration record func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), - version) + fmt.Sprintf("insert into %s (version, dump) values (?, ?)", drv.quotedMigrationsTableName()), + version, dump) return err } From 097126047e3f1b33dbdd2a6d919ec0c5a8d9ca2b Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 14:51:32 +0100 Subject: [PATCH 05/46] Add sync command line argument to synchronize existing database. --- main.go | 16 +++ pkg/dbmate/db.go | 135 +++++++++++++++++- pkg/dbmate/driver.go | 1 + pkg/driver/bigquery/bigquery.go | 33 +++++ pkg/driver/bigquery/bigquery_test.go | 7 + pkg/driver/clickhouse/clickhouse.go | 31 ++++ .../clickhouse/clickhouse_cluster_test.go | 7 + pkg/driver/clickhouse/clickhouse_test.go | 7 + pkg/driver/mysql/mysql.go | 29 ++++ pkg/driver/mysql/mysql_test.go | 7 + pkg/driver/postgres/postgres.go | 34 +++++ pkg/driver/postgres/postgres_test.go | 7 + pkg/driver/sqlite/sqlite.go | 29 ++++ pkg/driver/sqlite/sqlite_test.go | 7 + 14 files changed, 349 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 4b4c9308..b754aa57 100644 --- a/main.go +++ b/main.go @@ -183,6 +183,22 @@ func NewApp() *cli.App { return db.Rollback() }), }, + { + Name: "sync", + Usage: "Strips eventual later migrations or migrate up in case the database is older", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + EnvVars: []string{"DBMATE_VERBOSE"}, + Usage: "print the result of each statement execution", + }, + }, + Action: action(func(db *dbmate.DB, c *cli.Context) error { + db.Verbose = c.Bool("verbose") + return db.Synchronize() + }), + }, { Name: "status", Usage: "List applied and pending migrations", diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 7d264f42..64c88977 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -394,7 +394,7 @@ func (db *DB) Migrate() error { } // record migration - return drv.InsertMigration(tx, migration.Version, fmt.Sprintf("%s\n\n%s", parsed.Up, parsed.Down)) + return drv.InsertMigration(tx, migration.Version, parsed.Down) } if parsed.UpOptions.Transaction() { @@ -589,6 +589,139 @@ func (db *DB) Rollback() error { return nil } +// Strips eventual later migrations or migrate up in case the database is older. +func (db *DB) Synchronize() error { + drv, err := db.Driver() + if err != nil { + return err + } + + sqlDB, err := db.openDatabaseForMigration(drv) + if err != nil { + return err + } + defer dbutil.MustClose(sqlDB) + + // find last applied migration + var latest *Migration + migrations, err := db.FindMigrations() + if err != nil { + return err + } + + highestAppliedMigrationVersion := "" + pendingMigrations := []Migration{} + for _, migration := range migrations { + if migration.Applied { + if db.Strict && highestAppliedMigrationVersion <= migration.Version { + highestAppliedMigrationVersion = migration.Version + } + } else { + pendingMigrations = append(pendingMigrations, migration) + } + } + + if len(pendingMigrations) > 0 && db.Strict && pendingMigrations[0].Version <= highestAppliedMigrationVersion { + return fmt.Errorf( + "migration `%s` is out of order with already applied migrations, the version number has to be higher than the applied migration `%s` in --strict mode", + pendingMigrations[0].Version, + highestAppliedMigrationVersion, + ) + } + + // If there is at least one pending up migration we apply them all. + if len(pendingMigrations) > 0 { + + sqlDB, err := db.openDatabaseForMigration(drv) + if err != nil { + return err + } + defer dbutil.MustClose(sqlDB) + + for _, migration := range pendingMigrations { + fmt.Fprintf(db.Log, "Applying: %s\n", migration.FileName) + + start := time.Now() + + parsed, err := migration.Parse() + if err != nil { + return err + } + + execMigration := func(tx dbutil.Transaction) error { + // run actual migration + result, err := tx.Exec(parsed.Up) + if err != nil { + return drv.QueryError(parsed.Up, err) + } else if db.Verbose { + db.printVerbose(result) + } + + // record migration + return drv.InsertMigration(tx, migration.Version, parsed.Down) + } + + if parsed.UpOptions.Transaction() { + // begin transaction + err = doTransaction(sqlDB, execMigration) + } else { + // run outside of transaction + err = execMigration(sqlDB) + } + + elapsed := time.Since(start) + fmt.Fprintf(db.Log, "Applied: %s in %s\n", migration.FileName, elapsed) + + if err != nil { + return err + } + } + } else { + // Otherwise we need to check if we need to rollback some newer migration that the db have. + appliedMigrations := map[string]string{} + if migrationsTableExists { + migrationsToBeRolledBack, err = drv.SelectMigrationsFromVersion(sqlDB, highestAppliedMigrationVersion) + if err != nil { + return nil, err + } + } + for _, migration := range migrationsToBeRolledBack { + fmt.Fprintf(db.Log, "Applying: %s\n", migration.version) + + start := time.Now() + + // run actual migration dump rollback + result, err := tx.Exec(migration.dump) + if err != nil { + return drv.QueryError(migration.dump, err) + } else if db.Verbose { + db.printVerbose(result) + } + + // record migration + err = drv.DeleteMigration(sqlDB, migration.version) + + if err != nil { + return err + } + + elapsed := time.Since(start) + fmt.Fprintf(db.Log, "Applied: %s in %s\n", migration.version, elapsed) + + if err != nil { + return err + } + } + } + + // automatically update schema file, silence errors + if db.AutoDumpSchema { + _ = db.DumpSchema() + } + + return nil +} + // Status shows the status of all migrations func (db *DB) Status(quiet bool) (int, error) { results, err := db.FindMigrations() diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index 9f642ddf..c5f846e0 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -19,6 +19,7 @@ type Driver interface { MigrationsTableExists(*sql.DB) (bool, error) CreateMigrationsTable(*sql.DB) error SelectMigrations(*sql.DB, int) (map[string]bool, error) + SelectMigrationsFromVersion(*sql.DB, int) (map[string]string, error) InsertMigration(dbutil.Transaction, string, string) error DeleteMigration(dbutil.Transaction, string) error Ping() error diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index fd30eebf..61ca9e7a 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -377,6 +377,39 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err return migrations, nil } +// SelectMigrationsFromVersion returns a list of applied migrations +// newer than a specified version +func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { + config, err := drv.getConfig(db) + if err != nil { + return nil, err + } + + query := fmt.Sprintf("SELECT version FROM %s.%s WHERE version > %s ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer dbutil.MustClose(rows) + + migrations := map[string]string{} + for rows.Next() { + var version string + var dump string + if err := rows.Scan(&version, &dump); err != nil { + return nil, err + } + + migrations[version] = dump + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return migrations, nil +} + // Helper function to check whether a table exists or not in a dataset func tableExists(client *bigquery.Client, datasetID, tableName string) (bool, error) { table := client.Dataset(datasetID).Table(tableName) diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index bcd0854a..bb91a47c 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -249,6 +249,13 @@ func TestBigQuerySelectMigrations(t *testing.T) { require.Equal(t, true, migrations["abc3"]) require.Equal(t, false, migrations["abc1"]) require.Equal(t, false, migrations["abc2"]) + + // test migration from + migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, false, migrations["abc1"]) } func TestBigQueryInsertMigration(t *testing.T) { diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 537f943a..9259265e 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -331,6 +331,37 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err return migrations, nil } +// SelectMigrationsFromVersion returns a list of applied migrations +// newer than a specified version +func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { + query := fmt.Sprintf("select version from %s final where version > %s applied order by version desc", + drv.quotedMigrationsTableName(), version_from) + + rows, err := db.Query(query) + if err != nil { + return nil, err + } + + defer dbutil.MustClose(rows) + + migrations := map[string]string{} + for rows.Next() { + var version string + var dump string + if err := rows.Scan(&version, &dump); err != nil { + return nil, err + } + + migrations[version] = dump + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return migrations, nil +} + // InsertMigration adds a new migration record func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index 87dbe54e..b0e9b64e 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -246,6 +246,13 @@ func TestClickHouseSelectMigrationsOnCluster(t *testing.T) { require.Equal(t, false, migrations01["abc1"]) require.Equal(t, false, migrations01["abc2"]) + // test migration from + migrations, err = drv.SelectMigrationsFromVersion(db01, "abc1") + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, false, migrations["abc1"]) + // test limit param on other node migrations02, err = drv02.SelectMigrations(db02, 1) require.NoError(t, err) diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index 08d263cb..0e3ffdcd 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -287,6 +287,13 @@ func TestClickHouseSelectMigrations(t *testing.T) { require.Equal(t, true, migrations["abc3"]) require.Equal(t, false, migrations["abc1"]) require.Equal(t, false, migrations["abc2"]) + + // test migration from + migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, false, migrations["abc1"]) } func TestClickHouseInsertMigration(t *testing.T) { diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 3148a3cb..7e930e48 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -283,6 +283,35 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err return migrations, nil } +// SelectMigrationsFromVersion returns a list of applied migrations +// newer than a specified version +func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { + query := fmt.Sprintf("select version from %s where version > %s order by version desc", drv.quotedMigrationsTableName(), version_from) + rows, err := db.Query(query) + if err != nil { + return nil, err + } + + defer dbutil.MustClose(rows) + + migrations := map[string]string{} + for rows.Next() { + var version string + var dump string + if err := rows.Scan(&version, &dump); err != nil { + return nil, err + } + + migrations[version] = dump + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return migrations, nil +} + // InsertMigration adds a new migration record func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index bf1a7e80..a8e6dc57 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -331,6 +331,13 @@ func TestMySQLSelectMigrations(t *testing.T) { require.Equal(t, true, migrations["abc3"]) require.Equal(t, false, migrations["abc1"]) require.Equal(t, false, migrations["abc2"]) + + // test migration from + migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, false, migrations["abc1"]) } func TestMySQLInsertMigration(t *testing.T) { diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index ef30febe..f420bef2 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -329,6 +329,40 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err return migrations, nil } +// SelectMigrationsFromVersion returns a list of applied migrations +// newer than a specified version +func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { + migrationsTable, err := drv.quotedMigrationsTableName(db) + if err != nil { + return nil, err + } + + query := "select version from " + migrationsTable + " where version > " + version_from + " order by version desc" + rows, err := db.Query(query) + if err != nil { + return nil, err + } + + defer dbutil.MustClose(rows) + + migrations := map[string]string{} + for rows.Next() { + var version string + var dump string + if err := rows.Scan(&version, &dump); err != nil { + return nil, err + } + + migrations[version] = dump + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return migrations, nil +} + // InsertMigration adds a new migration record func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { migrationsTable, err := drv.quotedMigrationsTableName(db) diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 0e7ccdab..24534dce 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -493,6 +493,13 @@ func TestPostgresSelectMigrations(t *testing.T) { require.Equal(t, true, migrations["abc3"]) require.Equal(t, false, migrations["abc1"]) require.Equal(t, false, migrations["abc2"]) + + // test migration from + migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, false, migrations["abc1"]) } func TestPostgresInsertMigration(t *testing.T) { diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 2fbe1d2c..b9deaf8f 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -212,6 +212,35 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err return migrations, nil } +// SelectMigrationsFromVersion returns a list of applied migrations +// newer than a specified version +func (drv *Driver) SelectMigrations(db *sql.DB, version_from string) (map[string]string, error) { + query := fmt.Sprintf("select version from %s where version > %s order by version desc", drv.quotedMigrationsTableName(), version_from) + rows, err := db.Query(query) + if err != nil { + return nil, err + } + + defer dbutil.MustClose(rows) + + migrations := map[string]string{} + for rows.Next() { + var version string + var dump string + if err := rows.Scan(&version, &dump); err != nil { + return nil, err + } + + migrations[version] = dump + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return migrations, nil +} + // InsertMigration adds a new migration record func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump string) error { _, err := db.Exec( diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index 5381852f..587793b5 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -306,6 +306,13 @@ func TestSQLiteSelectMigrations(t *testing.T) { require.Equal(t, true, migrations["abc3"]) require.Equal(t, false, migrations["abc1"]) require.Equal(t, false, migrations["abc2"]) + + // test migration from + migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, false, migrations["abc1"]) } func TestSQLiteInsertMigration(t *testing.T) { From 3a1201db0ea50a4c2ba9e2df2f07e1df106a9bff Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 14:55:37 +0100 Subject: [PATCH 06/46] Edit few type mistakes. --- pkg/dbmate/driver.go | 2 +- pkg/driver/sqlite/sqlite.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index c5f846e0..a8114f1d 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -19,7 +19,7 @@ type Driver interface { MigrationsTableExists(*sql.DB) (bool, error) CreateMigrationsTable(*sql.DB) error SelectMigrations(*sql.DB, int) (map[string]bool, error) - SelectMigrationsFromVersion(*sql.DB, int) (map[string]string, error) + SelectMigrationsFromVersion(*sql.DB, string) (map[string]string, error) InsertMigration(dbutil.Transaction, string, string) error DeleteMigration(dbutil.Transaction, string) error Ping() error diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index b9deaf8f..5a2e147d 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -214,7 +214,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version -func (drv *Driver) SelectMigrations(db *sql.DB, version_from string) (map[string]string, error) { +func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { query := fmt.Sprintf("select version from %s where version > %s order by version desc", drv.quotedMigrationsTableName(), version_from) rows, err := db.Query(query) if err != nil { From a55ec8d4280f8b76ce4d5b47d03317fdab59e1b9 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 15:01:19 +0100 Subject: [PATCH 07/46] Cut some unused variables. --- pkg/dbmate/db.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 64c88977..0c297296 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -603,7 +603,6 @@ func (db *DB) Synchronize() error { defer dbutil.MustClose(sqlDB) // find last applied migration - var latest *Migration migrations, err := db.FindMigrations() if err != nil { return err @@ -678,9 +677,14 @@ func (db *DB) Synchronize() error { } } else { // Otherwise we need to check if we need to rollback some newer migration that the db have. - appliedMigrations := map[string]string{} + migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) + if err != nil { + return nil, err + } + + migrationsToBeRolledBack := map[string]string{} if migrationsTableExists { - migrationsToBeRolledBack, err = drv.SelectMigrationsFromVersion(sqlDB, highestAppliedMigrationVersion) + migrationsToBeRolledBack, err := drv.SelectMigrationsFromVersion(sqlDB, highestAppliedMigrationVersion) if err != nil { return nil, err } @@ -691,7 +695,7 @@ func (db *DB) Synchronize() error { start := time.Now() // run actual migration dump rollback - result, err := tx.Exec(migration.dump) + result, err := sqlDB.Exec(migration.dump) if err != nil { return drv.QueryError(migration.dump, err) } else if db.Verbose { From 57cb532fc625d5500e9c4215767f4f876d4bd432 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 15:04:05 +0100 Subject: [PATCH 08/46] Edit wrong return value and for loop. --- pkg/dbmate/db.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 0c297296..0c6ad73b 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -679,17 +679,17 @@ func (db *DB) Synchronize() error { // Otherwise we need to check if we need to rollback some newer migration that the db have. migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) if err != nil { - return nil, err + return err } migrationsToBeRolledBack := map[string]string{} if migrationsTableExists { migrationsToBeRolledBack, err := drv.SelectMigrationsFromVersion(sqlDB, highestAppliedMigrationVersion) if err != nil { - return nil, err + return err } } - for _, migration := range migrationsToBeRolledBack { + for migration := range migrationsToBeRolledBack { fmt.Fprintf(db.Log, "Applying: %s\n", migration.version) start := time.Now() From ba8356bcf35f4618cd44118bebc913652e920d46 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 15:08:56 +0100 Subject: [PATCH 09/46] Cut mistake in for loop over the map. --- pkg/dbmate/db.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 0c6ad73b..3641c41d 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -689,28 +689,28 @@ func (db *DB) Synchronize() error { return err } } - for migration := range migrationsToBeRolledBack { - fmt.Fprintf(db.Log, "Applying: %s\n", migration.version) + for version, dump := range migrationsToBeRolledBack { + fmt.Fprintf(db.Log, "Applying: %s\n", version) start := time.Now() // run actual migration dump rollback - result, err := sqlDB.Exec(migration.dump) + result, err := sqlDB.Exec(dump) if err != nil { - return drv.QueryError(migration.dump, err) + return drv.QueryError(dump, err) } else if db.Verbose { db.printVerbose(result) } // record migration - err = drv.DeleteMigration(sqlDB, migration.version) + err = drv.DeleteMigration(sqlDB, version) if err != nil { return err } elapsed := time.Since(start) - fmt.Fprintf(db.Log, "Applied: %s in %s\n", migration.version, elapsed) + fmt.Fprintf(db.Log, "Applied: %s in %s\n", version, elapsed) if err != nil { return err From 1dc390d9aced6549062a7f3ea088ff7b48dc5098 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 15:14:19 +0100 Subject: [PATCH 10/46] Update db.go --- pkg/dbmate/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 3641c41d..068e04e0 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -684,7 +684,7 @@ func (db *DB) Synchronize() error { migrationsToBeRolledBack := map[string]string{} if migrationsTableExists { - migrationsToBeRolledBack, err := drv.SelectMigrationsFromVersion(sqlDB, highestAppliedMigrationVersion) + migrationsToBeRolledBack, err = drv.SelectMigrationsFromVersion(sqlDB, highestAppliedMigrationVersion) if err != nil { return err } From d7c8d02624bdafb6cb7305e81ee20567810d9324 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 15:35:41 +0100 Subject: [PATCH 11/46] Edit tests that had errors. --- pkg/driver/bigquery/bigquery_test.go | 7 +++---- pkg/driver/clickhouse/clickhouse_cluster_test.go | 7 +++---- pkg/driver/clickhouse/clickhouse_test.go | 7 +++---- pkg/driver/mysql/mysql_test.go | 7 +++---- pkg/driver/postgres/postgres_test.go | 7 +++---- pkg/driver/sqlite/sqlite_test.go | 7 +++---- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index bb91a47c..f8516f16 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -251,11 +251,10 @@ func TestBigQuerySelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, false, migrations["abc1"]) + require.Equal(t, "abc3", migrations_dump["abc3"]) + require.Equal(t, "abc2", migrations_dump["abc2"]) } func TestBigQueryInsertMigration(t *testing.T) { diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index b0e9b64e..ddf4e420 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -247,11 +247,10 @@ func TestClickHouseSelectMigrationsOnCluster(t *testing.T) { require.Equal(t, false, migrations01["abc2"]) // test migration from - migrations, err = drv.SelectMigrationsFromVersion(db01, "abc1") + migrations01_dump, err = drv01.SelectMigrationsFromVersion(db01, "abc1") require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, false, migrations["abc1"]) + require.Equal(t, "abc3", migrations_dump["abc3"]) + require.Equal(t, "abc2", migrations_dump["abc2"]) // test limit param on other node migrations02, err = drv02.SelectMigrations(db02, 1) diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index 0e3ffdcd..403fe614 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -289,11 +289,10 @@ func TestClickHouseSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, false, migrations["abc1"]) + require.Equal(t, "abc3", migrations_dump["abc3"]) + require.Equal(t, "abc2", migrations_dump["abc2"]) } func TestClickHouseInsertMigration(t *testing.T) { diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index a8e6dc57..7cd61a6c 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -333,11 +333,10 @@ func TestMySQLSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, false, migrations["abc1"]) + require.Equal(t, "abc3", migrations_dump["abc3"]) + require.Equal(t, "abc2", migrations_dump["abc2"]) } func TestMySQLInsertMigration(t *testing.T) { diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 24534dce..3305c9a8 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -495,11 +495,10 @@ func TestPostgresSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, false, migrations["abc1"]) + require.Equal(t, "abc3", migrations_dump["abc3"]) + require.Equal(t, "abc2", migrations_dump["abc2"]) } func TestPostgresInsertMigration(t *testing.T) { diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index 587793b5..dda4e5d1 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -308,11 +308,10 @@ func TestSQLiteSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) - require.Equal(t, true, migrations["abc3"]) - require.Equal(t, true, migrations["abc2"]) - require.Equal(t, false, migrations["abc1"]) + require.Equal(t, "abc3", migrations_dump["abc3"]) + require.Equal(t, "abc2", migrations_dump["abc2"]) } func TestSQLiteInsertMigration(t *testing.T) { From 2905a946ebd50babb67ec552e628608b5b96f136 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 15:44:45 +0100 Subject: [PATCH 12/46] Add two points for create variables. --- pkg/driver/bigquery/bigquery_test.go | 2 +- pkg/driver/clickhouse/clickhouse_cluster_test.go | 2 +- pkg/driver/clickhouse/clickhouse_test.go | 2 +- pkg/driver/mysql/mysql_test.go | 2 +- pkg/driver/postgres/postgres_test.go | 2 +- pkg/driver/sqlite/sqlite_test.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index f8516f16..3da9ec32 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -251,7 +251,7 @@ func TestBigQuerySelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err := drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) require.Equal(t, "abc3", migrations_dump["abc3"]) require.Equal(t, "abc2", migrations_dump["abc2"]) diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index ddf4e420..55f39958 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -247,7 +247,7 @@ func TestClickHouseSelectMigrationsOnCluster(t *testing.T) { require.Equal(t, false, migrations01["abc2"]) // test migration from - migrations01_dump, err = drv01.SelectMigrationsFromVersion(db01, "abc1") + migrations01_dump, err := drv01.SelectMigrationsFromVersion(db01, "abc1") require.NoError(t, err) require.Equal(t, "abc3", migrations_dump["abc3"]) require.Equal(t, "abc2", migrations_dump["abc2"]) diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index 403fe614..c104d2dc 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -289,7 +289,7 @@ func TestClickHouseSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err := drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) require.Equal(t, "abc3", migrations_dump["abc3"]) require.Equal(t, "abc2", migrations_dump["abc2"]) diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index 7cd61a6c..23a0cb46 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -333,7 +333,7 @@ func TestMySQLSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err := drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) require.Equal(t, "abc3", migrations_dump["abc3"]) require.Equal(t, "abc2", migrations_dump["abc2"]) diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 3305c9a8..46ed43be 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -495,7 +495,7 @@ func TestPostgresSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err := drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) require.Equal(t, "abc3", migrations_dump["abc3"]) require.Equal(t, "abc2", migrations_dump["abc2"]) diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index dda4e5d1..9a853524 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -308,7 +308,7 @@ func TestSQLiteSelectMigrations(t *testing.T) { require.Equal(t, false, migrations["abc2"]) // test migration from - migrations_dump, err = drv.SelectMigrationsFromVersion(db, "abc1") + migrations_dump, err := drv.SelectMigrationsFromVersion(db, "abc1") require.NoError(t, err) require.Equal(t, "abc3", migrations_dump["abc3"]) require.Equal(t, "abc2", migrations_dump["abc2"]) From de518780c1e93f5b353819c799430c193323515c Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 15:59:21 +0100 Subject: [PATCH 13/46] Add missing columns in schema creation of some sql drivers. --- pkg/driver/clickhouse/clickhouse.go | 1 + pkg/driver/clickhouse/clickhouse_cluster_test.go | 4 ++-- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/sqlite/sqlite.go | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 9259265e..120a5f30 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -292,6 +292,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { applied UInt8 default 1 ) engine = %s primary key version + mediumtext dump order by version `, drv.quotedMigrationsTableName(), drv.onClusterClause(), engineClause)) diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index 55f39958..b5594f43 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -249,8 +249,8 @@ func TestClickHouseSelectMigrationsOnCluster(t *testing.T) { // test migration from migrations01_dump, err := drv01.SelectMigrationsFromVersion(db01, "abc1") require.NoError(t, err) - require.Equal(t, "abc3", migrations_dump["abc3"]) - require.Equal(t, "abc2", migrations_dump["abc2"]) + require.Equal(t, "abc3", migrations01_dump["abc3"]) + require.Equal(t, "abc2", migrations01_dump["abc2"]) // test limit param on other node migrations02, err = drv02.SelectMigrations(db02, 1) diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index f420bef2..02549e3c 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -265,7 +265,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // first attempt at creating migrations table createTableStmt := fmt.Sprintf( - "create table if not exists %s.%s (version varchar(128) primary key)", + "create table if not exists %s.%s (version varchar(128) primary key, dump mediumtext)", schema, migrationsTable) _, err = db.Exec(createTableStmt) if err == nil { diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 5a2e147d..e38c16e5 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -175,7 +175,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key)", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext)", drv.quotedMigrationsTableName())) return err From a14242973a10035a791d805cdb9d1647c24fb533 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 16:06:46 +0100 Subject: [PATCH 14/46] Add quotes for from version string in query. --- pkg/driver/bigquery/bigquery.go | 2 +- pkg/driver/clickhouse/clickhouse.go | 2 +- pkg/driver/mysql/mysql.go | 2 +- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/sqlite/sqlite.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 61ca9e7a..bde7ff03 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -385,7 +385,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - query := fmt.Sprintf("SELECT version FROM %s.%s WHERE version > %s ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) + query := fmt.Sprintf("SELECT version FROM %s.%s WHERE version > \"%s\" ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) rows, err := db.Query(query) if err != nil { return nil, err diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 120a5f30..bcd01472 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -335,7 +335,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select version from %s final where version > %s applied order by version desc", + query := fmt.Sprintf("select version from %s final where version > '%s' applied order by version desc", drv.quotedMigrationsTableName(), version_from) rows, err := db.Query(query) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 7e930e48..6955fa04 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -286,7 +286,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select version from %s where version > %s order by version desc", drv.quotedMigrationsTableName(), version_from) + query := fmt.Sprintf("select version from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) rows, err := db.Query(query) if err != nil { return nil, err diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 02549e3c..276ae782 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -337,7 +337,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - query := "select version from " + migrationsTable + " where version > " + version_from + " order by version desc" + query := "select version from " + migrationsTable + " where version > '" + version_from + "' order by version desc" rows, err := db.Query(query) if err != nil { return nil, err diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index e38c16e5..a12e839a 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -215,7 +215,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select version from %s where version > %s order by version desc", drv.quotedMigrationsTableName(), version_from) + query := fmt.Sprintf("select version from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) rows, err := db.Query(query) if err != nil { return nil, err From de366c3de3ac6eb910185d92e12ddf9f5a37ec95 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 16:14:24 +0100 Subject: [PATCH 15/46] Add selecting all with select query from version. --- pkg/driver/bigquery/bigquery.go | 2 +- pkg/driver/clickhouse/clickhouse.go | 2 +- pkg/driver/mysql/mysql.go | 2 +- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/sqlite/sqlite.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index bde7ff03..0bd0f405 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -385,7 +385,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - query := fmt.Sprintf("SELECT version FROM %s.%s WHERE version > \"%s\" ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) + query := fmt.Sprintf("SELECT * FROM %s.%s WHERE version > '%s' ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) rows, err := db.Query(query) if err != nil { return nil, err diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index bcd01472..b9c22467 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -335,7 +335,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select version from %s final where version > '%s' applied order by version desc", + query := fmt.Sprintf("select * from %s final where version > '%s' applied order by version desc", drv.quotedMigrationsTableName(), version_from) rows, err := db.Query(query) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 6955fa04..4c81beb5 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -286,7 +286,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select version from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) rows, err := db.Query(query) if err != nil { return nil, err diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 276ae782..74a16d72 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -337,7 +337,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - query := "select version from " + migrationsTable + " where version > '" + version_from + "' order by version desc" + query := "select * from " + migrationsTable + " where version > '" + version_from + "' order by version desc" rows, err := db.Query(query) if err != nil { return nil, err diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index a12e839a..f29cf83c 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -215,7 +215,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select version from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) rows, err := db.Query(query) if err != nil { return nil, err From b9f5796297ddda0c78c885f6facbf2447197f2e6 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 16:26:02 +0100 Subject: [PATCH 16/46] Add correct creation of new migration schema. --- pkg/driver/bigquery/bigquery_test.go | 14 ++++++------- .../clickhouse/clickhouse_cluster_test.go | 6 +++--- pkg/driver/clickhouse/clickhouse_test.go | 6 +++--- pkg/driver/mysql/mysql_test.go | 14 ++++++------- pkg/driver/postgres/postgres_test.go | 20 +++++++++---------- pkg/driver/sqlite/sqlite_test.go | 4 ++-- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index 3da9ec32..7efbe3bd 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -233,8 +233,8 @@ func TestBigQuerySelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc2','abc2'), ('abc1','abc1'), ('abc3','abc3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) @@ -292,8 +292,8 @@ func TestBigQueryDeleteMigration(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc1'), ('abc2')`) + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc1','abc1'), ('abc2','abc2')`) require.NoError(t, err) err = drv.DeleteMigration(db, "abc2") @@ -374,8 +374,8 @@ func TestGoogleBigQueryDumpSchema(t *testing.T) { require.Contains(t, string(schema), "\n--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO schema_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO schema_migrations (version, dump) VALUES\n"+ + " ('abc1','abc1'),\n"+ + " ('abc2','abc2');\n") }) } diff --git a/pkg/driver/clickhouse/clickhouse_cluster_test.go b/pkg/driver/clickhouse/clickhouse_cluster_test.go index b5594f43..59d53ea8 100644 --- a/pkg/driver/clickhouse/clickhouse_cluster_test.go +++ b/pkg/driver/clickhouse/clickhouse_cluster_test.go @@ -120,9 +120,9 @@ func TestClickHouseDumpSchemaOnCluster(t *testing.T) { require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO test_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO test_migrations (version, dump) VALUES\n"+ + " ('abc1','abc1'),\n"+ + " ('abc2','abc2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index c104d2dc..7081a13b 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -123,9 +123,9 @@ func TestClickHouseDumpSchema(t *testing.T) { require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO test_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO test_migrations (version, dump) VALUES\n"+ + " ('abc1','abc1'),\n"+ + " ('abc2','abc2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index 23a0cb46..0a89148c 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -206,9 +206,9 @@ func TestMySQLDumpSchema(t *testing.T) { "-- Dbmate schema migrations\n"+ "--\n\n"+ "LOCK TABLES `test_migrations` WRITE;\n"+ - "INSERT INTO `test_migrations` (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n"+ + "INSERT INTO `test_migrations` (version, dump) VALUES\n"+ + " ('abc1','abc1'),\n"+ + " ('abc2','abc2');\n"+ "UNLOCK TABLES;\n") // DumpSchema should return error if command fails @@ -315,8 +315,8 @@ func TestMySQLSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc2','abc2'), ('abc1','abc1'), ('abc3','abc3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) @@ -374,8 +374,8 @@ func TestMySQLDeleteMigration(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc1'), ('abc2')`) + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc1','abc1'), ('abc2','abc2')`) require.NoError(t, err) err = drv.DeleteMigration(db, "abc2") diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 46ed43be..09427591 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -232,9 +232,9 @@ func TestPostgresDumpSchema(t *testing.T) { "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO public.schema_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO public.schema_migrations (version, dump) VALUES\n"+ + " ('abc1','abc1'),\n"+ + " ('abc2','abc2');\n") // DumpSchema should return error if command fails drv.databaseURL.Path = "/fakedb" @@ -270,9 +270,9 @@ func TestPostgresDumpSchema(t *testing.T) { "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO \"camelSchema\".\"testMigrations\" (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + "INSERT INTO \"camelSchema\".\"testMigrations\" (version, dump) VALUES\n"+ + " ('abc1','abc1'),\n"+ + " ('abc2','abc2');\n") }) } @@ -477,8 +477,8 @@ func TestPostgresSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into public.test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into public.test_migrations (version, dump) + values ('abc2','abc2'), ('abc1','abc1'), ('abc3','abc3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) @@ -536,8 +536,8 @@ func TestPostgresDeleteMigration(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into public.test_migrations (version) - values ('abc1'), ('abc2')`) + _, err = db.Exec(`insert into public.test_migrations (version, dump) + values ('abc1','abc1'), ('abc2','abc2')`) require.NoError(t, err) err = drv.DeleteMigration(db, "abc2") diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index 9a853524..6eb1b76d 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -290,8 +290,8 @@ func TestSQLiteSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(db) require.NoError(t, err) - _, err = db.Exec(`insert into test_migrations (version) - values ('abc2'), ('abc1'), ('abc3')`) + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc2','abc2'), ('abc1','abc1'), ('abc3','abc3')`) require.NoError(t, err) migrations, err := drv.SelectMigrations(db, -1) From b423bb97e2f0a3b44626ac8a7f379a85d9ff89e8 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Tue, 11 Mar 2025 19:30:47 +0100 Subject: [PATCH 17/46] Cut strict condition. --- pkg/dbmate/db.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 068e04e0..317f1939 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -612,7 +612,7 @@ func (db *DB) Synchronize() error { pendingMigrations := []Migration{} for _, migration := range migrations { if migration.Applied { - if db.Strict && highestAppliedMigrationVersion <= migration.Version { + if highestAppliedMigrationVersion <= migration.Version { highestAppliedMigrationVersion = migration.Version } } else { @@ -682,6 +682,8 @@ func (db *DB) Synchronize() error { return err } + fmt.Fprintf(db.Log, "Latest available migration: %s\n", highestAppliedMigrationVersion) + migrationsToBeRolledBack := map[string]string{} if migrationsTableExists { migrationsToBeRolledBack, err = drv.SelectMigrationsFromVersion(sqlDB, highestAppliedMigrationVersion) From 1bd4eed08f22cee2b799ecd59cbf7c003ab75bde Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 12:38:55 +0100 Subject: [PATCH 18/46] Add UpdateEmptyDump() function. --- pkg/dbmate/db.go | 95 +++++++++++++++++++++--- pkg/dbmate/driver.go | 1 + pkg/driver/bigquery/bigquery.go | 48 ++++++++++-- pkg/driver/bigquery/bigquery_test.go | 25 +++++++ pkg/driver/clickhouse/clickhouse.go | 35 +++++++-- pkg/driver/clickhouse/clickhouse_test.go | 40 +++++++++- pkg/driver/mysql/mysql.go | 26 ++++++- pkg/driver/mysql/mysql_test.go | 25 +++++++ pkg/driver/postgres/postgres.go | 29 +++++++- pkg/driver/postgres/postgres_test.go | 25 +++++++ pkg/driver/sqlite/sqlite.go | 26 ++++++- pkg/driver/sqlite/sqlite_test.go | 25 +++++++ 12 files changed, 361 insertions(+), 39 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 317f1939..f9a0bb2a 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -514,6 +514,84 @@ func (db *DB) FindMigrations() ([]Migration, error) { return migrations, nil } +// UpdateEmptyDumps lists all available migrations +func (db *DB) UpdateEmptyDumps() (error) { + drv, err := db.Driver() + if err != nil { + return err + } + + sqlDB, err := drv.Open() + if err != nil { + return nil, err + } + defer dbutil.MustClose(sqlDB) + + // find applied migrations + appliedMigrations := map[string]string{} + migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) + if err != nil { + return err + } + + // Get all migrations dump from database. + if migrationsTableExists { + appliedMigrations, err = drv.SelectMigrationsFromVersion(sqlDB, "") + if err != nil { + return err + } + } + + for _, dir := range db.MigrationsDir { + // find filesystem migrations + files, err := db.readMigrationsDir(dir) + if err != nil { + return fmt.Errorf("%w `%s`", ErrMigrationDirNotFound, dir) + } + + for _, file := range files { + if file.IsDir() { + continue + } + + matches := migrationFileRegexp.FindStringSubmatch(file.Name()) + if len(matches) < 2 { + continue + } + + migration := Migration{ + Applied: false, + FileName: matches[0], + FilePath: path.Join(dir, matches[0]), + FS: db.FS, + Version: matches[1], + } + if ok := appliedMigrations[migration.Version]; ok { + // Migration already applied, check if the dump column in db is eventually empty. + if isEmpty(appliedMigrations[migration.Version]) { + fmt.Fprintf(db.Log, "Updating dump column of: %s\n", migration.FileName) + + start := time.Now() + + parsed, err := migration.Parse() + if err != nil { + return err + } + + appliedMigrations[migration.Version] = parsed.Down + + err = drv.UpdateMigrationDump(tx, migration.Version, parsed.Down) + if err != nil { + return err + } + } + } + } + } + + return nil +} + // Rollback rolls back the most recent migration func (db *DB) Rollback() error { drv, err := db.Driver() @@ -608,6 +686,11 @@ func (db *DB) Synchronize() error { return err } + err := db.UpdateEmptyDumps() + if err != nil { + return err + } + highestAppliedMigrationVersion := "" pendingMigrations := []Migration{} for _, migration := range migrations { @@ -631,12 +714,6 @@ func (db *DB) Synchronize() error { // If there is at least one pending up migration we apply them all. if len(pendingMigrations) > 0 { - sqlDB, err := db.openDatabaseForMigration(drv) - if err != nil { - return err - } - defer dbutil.MustClose(sqlDB) - for _, migration := range pendingMigrations { fmt.Fprintf(db.Log, "Applying: %s\n", migration.FileName) @@ -682,7 +759,7 @@ func (db *DB) Synchronize() error { return err } - fmt.Fprintf(db.Log, "Latest available migration: %s\n", highestAppliedMigrationVersion) + fmt.Fprintf(db.Log, "Latest available migration file: %s.sql\n", highestAppliedMigrationVersion) migrationsToBeRolledBack := map[string]string{} if migrationsTableExists { @@ -692,7 +769,7 @@ func (db *DB) Synchronize() error { } } for version, dump := range migrationsToBeRolledBack { - fmt.Fprintf(db.Log, "Applying: %s\n", version) + fmt.Fprintf(db.Log, "Rolling back later db migration: %s\n", version) start := time.Now() @@ -712,7 +789,7 @@ func (db *DB) Synchronize() error { } elapsed := time.Since(start) - fmt.Fprintf(db.Log, "Applied: %s in %s\n", version, elapsed) + fmt.Fprintf(db.Log, "Rolled back: %s in %s\n", version, elapsed) if err != nil { return err diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index a8114f1d..ed497590 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -22,6 +22,7 @@ type Driver interface { SelectMigrationsFromVersion(*sql.DB, string) (map[string]string, error) InsertMigration(dbutil.Transaction, string, string) error DeleteMigration(dbutil.Transaction, string) error + UpdateMigrationDump(dbutil.Transaction, string, string) error Ping() error QueryError(string, error) error } diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 0bd0f405..0f007e8c 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -97,6 +97,10 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { Name: "version", Type: bigquery.StringFieldType, }, + { + Name: "dump", + Type: bigquery.StringFieldType, + }, }, }) }) @@ -283,6 +287,28 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { return exists, nil } +func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string, dump string) error { + db, err := drv.Open() + if err != nil { + return err + } + defer dbutil.MustClose(db) + + config, err := drv.getConfig(db) + if err != nil { + return err + } + + queryTemplate := `INSERT INTO %s.%s (version, dump) VALUES ('%s','%s');` + queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName, version, dump) + _, err = db.Exec(queryString, version) + if err != nil { + return err + } + + return nil +} + func (drv *Driver) DeleteMigration(util dbutil.Transaction, version string) error { db, err := drv.Open() if err != nil { @@ -304,7 +330,7 @@ func (drv *Driver) DeleteMigration(util dbutil.Transaction, version string) erro return nil } -func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string, dump string) error { +func (drv *Driver) UpdateMigrationDump(util dbutil.Transaction, version string, dump string) error { db, err := drv.Open() if err != nil { return err @@ -316,9 +342,8 @@ func (drv *Driver) InsertMigration(_ dbutil.Transaction, version string, dump st return err } - queryTemplate := `INSERT INTO %s.%s (version, dump) VALUES ('%s','%s');` - queryString := fmt.Sprintf(queryTemplate, config.dataSet, drv.migrationsTableName, version, dump) - _, err = db.Exec(queryString, version) + query := fmt.Sprintf("UPDATE %s.%s SET dump = '%s' WHERE version = '%s';", config.dataSet, drv.migrationsTableName, dump, version) + _, err = util.Exec(query) if err != nil { return err } @@ -385,7 +410,12 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - query := fmt.Sprintf("SELECT * FROM %s.%s WHERE version > '%s' ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) + if isEmpty(version_from){ + query := fmt.Sprintf("SELECT * FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) + } else { + query := fmt.Sprintf("SELECT * FROM %s.%s WHERE version > '%s' ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) + } + rows, err := db.Query(query) if err != nil { return nil, err @@ -395,12 +425,16 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) migrations := map[string]string{} for rows.Next() { var version string - var dump string + var dump sql.NullString if err := rows.Scan(&version, &dump); err != nil { return nil, err } - migrations[version] = dump + if dump.Valid { + migrations[version] = dump + } else { + migrations[version] = "" + } } if err = rows.Err(); err != nil { diff --git a/pkg/driver/bigquery/bigquery_test.go b/pkg/driver/bigquery/bigquery_test.go index 7efbe3bd..dcd524f2 100644 --- a/pkg/driver/bigquery/bigquery_test.go +++ b/pkg/driver/bigquery/bigquery_test.go @@ -305,6 +305,31 @@ func TestBigQueryDeleteMigration(t *testing.T) { require.Equal(t, 1, count) } +func TestBigQueryUpdateMigrationDump(t *testing.T) { + drv := testBigQueryDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestBigQueryDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc1','')`) + require.NoError(t, err) + + err = drv.UpdateMigrationDump(db, "abc1", "abc1") + require.NoError(t, err) + + var version string + var dump string + err = db.QueryRow("select * from test_migrations").Scan(&version, &dump) + require.NoError(t, err) + require.Equal(t, "abc1", version) + require.Equal(t, "abc1", dump) +} + func TestBigQueryPingError(t *testing.T) { drv := testBigQueryDriver(t) diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index b9c22467..cee94815 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -288,12 +288,14 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf(` create table if not exists %s%s ( version String, + dump String, ts DateTime default now(), applied UInt8 default 1 ) engine = %s primary key version - mediumtext dump - order by version + order by version; + alter table users + add column if not exists dump String; `, drv.quotedMigrationsTableName(), drv.onClusterClause(), engineClause)) return err @@ -335,8 +337,12 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select * from %s final where version > '%s' applied order by version desc", - drv.quotedMigrationsTableName(), version_from) + + if isEmpty(version_from){ + query := fmt.Sprintf("select * from %s final applied order by version desc", drv.quotedMigrationsTableName()) + } else { + query := fmt.Sprintf("select * from %s final where version > '%s' applied order by version desc", drv.quotedMigrationsTableName(), version_from) + } rows, err := db.Query(query) if err != nil { @@ -348,12 +354,16 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) migrations := map[string]string{} for rows.Next() { var version string - var dump string + var dump sql.NullString if err := rows.Scan(&version, &dump); err != nil { return nil, err } - migrations[version] = dump + if dump.Valid { + migrations[version] = dump + } else { + migrations[version] = "" + } } if err = rows.Err(); err != nil { @@ -375,14 +385,23 @@ func (drv *Driver) InsertMigration(db dbutil.Transaction, version string, dump s // DeleteMigration removes a migration record func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error { _, err := db.Exec( - fmt.Sprintf("insert into %s (version, applied) values (?, ?)", + fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()), - version, false, + version ) return err } +// UpdateMigrationDump updates the dump column of a specific migration record +func (drv *Driver) UpdateMigrationDump(db dbutil.Transaction, version string, dump string) error { + _, err := db.Exec( + fmt.Sprintf("update %s set dump = ? where version = ?", drv.quotedMigrationsTableName()), + dump, version) + + return err +} + // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. func (drv *Driver) Ping() error { diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index 7081a13b..faca1f63 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -335,11 +335,11 @@ func TestClickHouseDeleteMigration(t *testing.T) { tx, err := db.Begin() require.NoError(t, err) - stmt, err := tx.Prepare("insert into test_migrations (version) values (?)") + stmt, err := tx.Prepare("insert into test_migrations (version, dump) values (?, ?)") require.NoError(t, err) - _, err = stmt.Exec("abc2") + _, err = stmt.Exec("abc2","abc2") require.NoError(t, err) - _, err = stmt.Exec("abc1") + _, err = stmt.Exec("abc1","abc1") require.NoError(t, err) err = tx.Commit() require.NoError(t, err) @@ -357,6 +357,40 @@ func TestClickHouseDeleteMigration(t *testing.T) { require.Equal(t, 1, count) } +func TestClickHouseUpdateMigrationDump(t *testing.T) { + drv := testClickHouseDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestClickHouseDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc1','')`) + require.NoError(t, err) + + tx, err := db.Begin() + require.NoError(t, err) + stmt, err := tx.Prepare("insert into test_migrations (version, dump) values (?,?)") + require.NoError(t, err) + _, err = stmt.Exec("abc1","") + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) + + err = drv.UpdateMigrationDump(db, "abc1", "abc1") + require.NoError(t, err) + + var version string + var dump string + err = db.QueryRow("select * from test_migrations").Scan(&version, &dump) + require.NoError(t, err) + require.Equal(t, "abc1", version) + require.Equal(t, "abc1", dump) +} + func TestClickHousePing(t *testing.T) { drv := testClickHouseDriver(t) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 4c81beb5..f1239ce3 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,7 +246,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext)", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", drv.quotedMigrationsTableName())) return err @@ -286,7 +286,12 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + if isEmpty(version_from){ + query := fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) + } else { + query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + } + rows, err := db.Query(query) if err != nil { return nil, err @@ -297,12 +302,16 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) migrations := map[string]string{} for rows.Next() { var version string - var dump string + var dump sql.NullString if err := rows.Scan(&version, &dump); err != nil { return nil, err } - migrations[version] = dump + if dump.Valid { + migrations[version] = dump + } else { + migrations[version] = "" + } } if err = rows.Err(); err != nil { @@ -330,6 +339,15 @@ func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error return err } +// UpdateMigrationDump updates the dump column of a specific migration record +func (drv *Driver) UpdateMigrationDump(db dbutil.Transaction, version string, dump string) error { + _, err := db.Exec( + fmt.Sprintf("update %s set dump = ? where version = ?", drv.quotedMigrationsTableName()), + dump, version) + + return err +} + // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. func (drv *Driver) Ping() error { diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index 0a89148c..180479d4 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -387,6 +387,31 @@ func TestMySQLDeleteMigration(t *testing.T) { require.Equal(t, 1, count) } +func TestMySQLUpdateMigrationDump(t *testing.T) { + drv := testMySQLDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestMySQLDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc1','')`) + require.NoError(t, err) + + err = drv.UpdateMigrationDump(db, "abc1", "abc1") + require.NoError(t, err) + + var version string + var dump string + err = db.QueryRow("select * from test_migrations").Scan(&version, &dump) + require.NoError(t, err) + require.Equal(t, "abc1", version) + require.Equal(t, "abc1", dump) +} + func TestMySQLPing(t *testing.T) { drv := testMySQLDriver(t) diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 74a16d72..b6bae279 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -265,7 +265,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // first attempt at creating migrations table createTableStmt := fmt.Sprintf( - "create table if not exists %s.%s (version varchar(128) primary key, dump mediumtext)", + "create table if not exists %s.%s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", schema, migrationsTable) _, err = db.Exec(createTableStmt) if err == nil { @@ -337,7 +337,12 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - query := "select * from " + migrationsTable + " where version > '" + version_from + "' order by version desc" + if isEmpty(version_from){ + query := "select * from " + migrationsTable + " order by version desc" + } else { + query := "select * from " + migrationsTable + " where version > '" + version_from + "' order by version desc" + } + rows, err := db.Query(query) if err != nil { return nil, err @@ -348,12 +353,16 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) migrations := map[string]string{} for rows.Next() { var version string - var dump string + var dump sql.NullString if err := rows.Scan(&version, &dump); err != nil { return nil, err } - migrations[version] = dump + if dump.Valid { + migrations[version] = dump + } else { + migrations[version] = "" + } } if err = rows.Err(); err != nil { @@ -387,6 +396,18 @@ func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error return err } +// UpdateMigrationDump updates the dump column of a specific migration record +func (drv *Driver) UpdateMigrationDump(db dbutil.Transaction, version string, dump string) error { + migrationsTable, err := drv.quotedMigrationsTableName(db) + if err != nil { + return err + } + + _, err = db.Exec("update from "+migrationsTable+" set dump = $1 where version = $2", dump, version) + + return err +} + // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. func (drv *Driver) Ping() error { diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 09427591..ba55220f 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -549,6 +549,31 @@ func TestPostgresDeleteMigration(t *testing.T) { require.Equal(t, 1, count) } +func TestPostgresUpdateMigrationDump(t *testing.T) { + drv := testPostgresDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + _, err = db.Exec(`insert into public.test_migrations (version, dump) + values ('abc1','')`) + require.NoError(t, err) + + err = drv.UpdateMigrationDump(db, "abc1", "abc1") + require.NoError(t, err) + + var version string + var dump string + err = db.QueryRow("select * from public.test_migrations").Scan(&version, &dump) + require.NoError(t, err) + require.Equal(t, "abc1", version) + require.Equal(t, "abc1", dump) +} + func TestPostgresPing(t *testing.T) { drv := testPostgresDriver(t) diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index f29cf83c..3c1d747c 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -175,7 +175,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext)", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", drv.quotedMigrationsTableName())) return err @@ -215,7 +215,12 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + if isEmpty(version_from){ + query := fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) + } else { + query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + } + rows, err := db.Query(query) if err != nil { return nil, err @@ -226,12 +231,16 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) migrations := map[string]string{} for rows.Next() { var version string - var dump string + var dump sql.NullString if err := rows.Scan(&version, &dump); err != nil { return nil, err } - migrations[version] = dump + if dump.Valid { + migrations[version] = dump + } else { + migrations[version] = "" + } } if err = rows.Err(); err != nil { @@ -259,6 +268,15 @@ func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error return err } +// UpdateMigrationDump updates the dump column of a specific migration record +func (drv *Driver) UpdateMigrationDump(db dbutil.Transaction, version string, dump string) error { + _, err := db.Exec( + fmt.Sprintf("update %s set dump = ? where version = ?", drv.quotedMigrationsTableName()), + dump, version) + + return err +} + // Ping verifies a connection to the database. Due to the way SQLite works, by // testing whether the database is valid, it will automatically create the database // if it does not already exist. diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index 6eb1b76d..bac53397 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -362,6 +362,31 @@ func TestSQLiteDeleteMigration(t *testing.T) { require.Equal(t, 1, count) } +func TestSQLLiteUpdateMigrationDump(t *testing.T) { + drv := testMySQLDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestMySQLDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + _, err = db.Exec(`insert into test_migrations (version, dump) + values ('abc1','')`) + require.NoError(t, err) + + err = drv.UpdateMigrationDump(db, "abc1", "abc1") + require.NoError(t, err) + + var version string + var dump string + err = db.QueryRow("select * from test_migrations").Scan(&version, &dump) + require.NoError(t, err) + require.Equal(t, "abc1", version) + require.Equal(t, "abc1", dump) +} + func TestSQLitePing(t *testing.T) { drv := testSQLiteDriver(t) path := ConnectionString(drv.databaseURL) From ecc0af1e9326bfafae12f16385edb4f7467921aa Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 12:42:29 +0100 Subject: [PATCH 19/46] Cut isEmpty function that doesn't exists. --- pkg/dbmate/db.go | 2 +- pkg/driver/bigquery/bigquery.go | 2 +- pkg/driver/clickhouse/clickhouse.go | 2 +- pkg/driver/mysql/mysql.go | 2 +- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/sqlite/sqlite.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index f9a0bb2a..613a233c 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -568,7 +568,7 @@ func (db *DB) UpdateEmptyDumps() (error) { } if ok := appliedMigrations[migration.Version]; ok { // Migration already applied, check if the dump column in db is eventually empty. - if isEmpty(appliedMigrations[migration.Version]) { + if appliedMigrations[migration.Version] == "" { fmt.Fprintf(db.Log, "Updating dump column of: %s\n", migration.FileName) start := time.Now() diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index 0f007e8c..e9238578 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -410,7 +410,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - if isEmpty(version_from){ + if version_from == "" { query := fmt.Sprintf("SELECT * FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) } else { query := fmt.Sprintf("SELECT * FROM %s.%s WHERE version > '%s' ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index cee94815..2d22ec13 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -338,7 +338,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - if isEmpty(version_from){ + if version_from == "" { query := fmt.Sprintf("select * from %s final applied order by version desc", drv.quotedMigrationsTableName()) } else { query := fmt.Sprintf("select * from %s final where version > '%s' applied order by version desc", drv.quotedMigrationsTableName(), version_from) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index f1239ce3..304b48de 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -286,7 +286,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - if isEmpty(version_from){ + if version_from == "" { query := fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) } else { query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index b6bae279..0c31b1c4 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -337,7 +337,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } - if isEmpty(version_from){ + if version_from == "" { query := "select * from " + migrationsTable + " order by version desc" } else { query := "select * from " + migrationsTable + " where version > '" + version_from + "' order by version desc" diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 3c1d747c..214264e9 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -215,7 +215,7 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { - if isEmpty(version_from){ + if version_from == "" { query := fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) } else { query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) From 0d192a14d2c36ff04dbd885805657ddcf2621a5d Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 12:50:43 +0100 Subject: [PATCH 20/46] Edit some mistakes. --- pkg/dbmate/db.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 613a233c..28e06e92 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -523,18 +523,18 @@ func (db *DB) UpdateEmptyDumps() (error) { sqlDB, err := drv.Open() if err != nil { - return nil, err + return err } defer dbutil.MustClose(sqlDB) // find applied migrations - appliedMigrations := map[string]string{} migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) if err != nil { return err } // Get all migrations dump from database. + dumpMigrations := map[string]string{} if migrationsTableExists { appliedMigrations, err = drv.SelectMigrationsFromVersion(sqlDB, "") if err != nil { @@ -542,6 +542,15 @@ func (db *DB) UpdateEmptyDumps() (error) { } } + // Get all migrations dump from database. + appliedMigrations := map[string]bool{} + if migrationsTableExists { + appliedMigrations, err = drv.SelectMigrations(sqlDB, -1) + if err != nil { + return err + } + } + for _, dir := range db.MigrationsDir { // find filesystem migrations files, err := db.readMigrationsDir(dir) @@ -568,7 +577,7 @@ func (db *DB) UpdateEmptyDumps() (error) { } if ok := appliedMigrations[migration.Version]; ok { // Migration already applied, check if the dump column in db is eventually empty. - if appliedMigrations[migration.Version] == "" { + if dumpMigrations[migration.Version] == "" { fmt.Fprintf(db.Log, "Updating dump column of: %s\n", migration.FileName) start := time.Now() @@ -580,7 +589,7 @@ func (db *DB) UpdateEmptyDumps() (error) { appliedMigrations[migration.Version] = parsed.Down - err = drv.UpdateMigrationDump(tx, migration.Version, parsed.Down) + err = drv.UpdateMigrationDump(sqlDB, migration.Version, parsed.Down) if err != nil { return err } @@ -686,7 +695,7 @@ func (db *DB) Synchronize() error { return err } - err := db.UpdateEmptyDumps() + err = db.UpdateEmptyDumps() if err != nil { return err } From 84894bec1ff39fe472433912ed303b1db15d53ed Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 12:53:19 +0100 Subject: [PATCH 21/46] Cut another mistake. --- pkg/dbmate/db.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 28e06e92..f01c3db8 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -536,7 +536,7 @@ func (db *DB) UpdateEmptyDumps() (error) { // Get all migrations dump from database. dumpMigrations := map[string]string{} if migrationsTableExists { - appliedMigrations, err = drv.SelectMigrationsFromVersion(sqlDB, "") + dumpMigrations, err = drv.SelectMigrationsFromVersion(sqlDB, "") if err != nil { return err } @@ -587,7 +587,7 @@ func (db *DB) UpdateEmptyDumps() (error) { return err } - appliedMigrations[migration.Version] = parsed.Down + dumpMigrations[migration.Version] = parsed.Down err = drv.UpdateMigrationDump(sqlDB, migration.Version, parsed.Down) if err != nil { From 4908a895b31b425f31bd00843cac7091d95f3e51 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 12:56:40 +0100 Subject: [PATCH 22/46] Add forgotten print log. --- pkg/dbmate/db.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index f01c3db8..21afe549 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -578,7 +578,7 @@ func (db *DB) UpdateEmptyDumps() (error) { if ok := appliedMigrations[migration.Version]; ok { // Migration already applied, check if the dump column in db is eventually empty. if dumpMigrations[migration.Version] == "" { - fmt.Fprintf(db.Log, "Updating dump column of: %s\n", migration.FileName) + fmt.Fprintf(db.Log, "Updating dump column of: %s\n", migration.Version) start := time.Now() @@ -593,6 +593,9 @@ func (db *DB) UpdateEmptyDumps() (error) { if err != nil { return err } + + elapsed := time.Since(start) + fmt.Fprintf(db.Log, "Updated dump column of: %s in %s\n", migration.Version, elapsed) } } } From b12155e12e6974e4a3d8c8a5ea6122b76f434d97 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 13:02:34 +0100 Subject: [PATCH 23/46] Edit a compile error. --- README.md | 1 + pkg/driver/bigquery/bigquery.go | 7 ++++--- pkg/driver/clickhouse/clickhouse.go | 10 +++++----- pkg/driver/mysql/mysql.go | 7 ++++--- pkg/driver/postgres/postgres.go | 7 ++++--- pkg/driver/sqlite/sqlite.go | 7 ++++--- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 923c2a64..9e8fdab8 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ dbmate drop # drop the database dbmate migrate # run any pending migrations dbmate rollback # roll back the most recent migration dbmate down # alias for rollback +dbmate sync # sync the database to most recent migration or downgrade the database in case the available file migrations are older dbmate status # show the status of all migrations (supports --exit-code and --quiet) dbmate dump # write the database schema.sql file dbmate load # load schema.sql file to the database diff --git a/pkg/driver/bigquery/bigquery.go b/pkg/driver/bigquery/bigquery.go index e9238578..e4674761 100644 --- a/pkg/driver/bigquery/bigquery.go +++ b/pkg/driver/bigquery/bigquery.go @@ -410,10 +410,11 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } + var query string if version_from == "" { - query := fmt.Sprintf("SELECT * FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) + query = fmt.Sprintf("SELECT * FROM %s.%s ORDER BY version DESC", config.dataSet, drv.migrationsTableName) } else { - query := fmt.Sprintf("SELECT * FROM %s.%s WHERE version > '%s' ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) + query = fmt.Sprintf("SELECT * FROM %s.%s WHERE version > '%s' ORDER BY version DESC", config.dataSet, drv.migrationsTableName, version_from) } rows, err := db.Query(query) @@ -431,7 +432,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) } if dump.Valid { - migrations[version] = dump + migrations[version] = dump.String } else { migrations[version] = "" } diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 2d22ec13..97700730 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -338,10 +338,11 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { + var query string if version_from == "" { - query := fmt.Sprintf("select * from %s final applied order by version desc", drv.quotedMigrationsTableName()) + query = fmt.Sprintf("select * from %s final applied order by version desc", drv.quotedMigrationsTableName()) } else { - query := fmt.Sprintf("select * from %s final where version > '%s' applied order by version desc", drv.quotedMigrationsTableName(), version_from) + query = fmt.Sprintf("select * from %s final where version > '%s' applied order by version desc", drv.quotedMigrationsTableName(), version_from) } rows, err := db.Query(query) @@ -360,7 +361,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) } if dump.Valid { - migrations[version] = dump + migrations[version] = dump.String } else { migrations[version] = "" } @@ -387,8 +388,7 @@ func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error _, err := db.Exec( fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()), - version - ) + version) return err } diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 304b48de..16bc71cb 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -286,10 +286,11 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { + var query string if version_from == "" { - query := fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) + query = fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) } else { - query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + query = fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) } rows, err := db.Query(query) @@ -308,7 +309,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) } if dump.Valid { - migrations[version] = dump + migrations[version] = dump.String } else { migrations[version] = "" } diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 0c31b1c4..b85275e9 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -337,10 +337,11 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) return nil, err } + var query string if version_from == "" { - query := "select * from " + migrationsTable + " order by version desc" + query = "select * from " + migrationsTable + " order by version desc" } else { - query := "select * from " + migrationsTable + " where version > '" + version_from + "' order by version desc" + query = "select * from " + migrationsTable + " where version > '" + version_from + "' order by version desc" } rows, err := db.Query(query) @@ -359,7 +360,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) } if dump.Valid { - migrations[version] = dump + migrations[version] = dump.String } else { migrations[version] = "" } diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 214264e9..e7b089fd 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -215,10 +215,11 @@ func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, err // SelectMigrationsFromVersion returns a list of applied migrations // newer than a specified version func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) (map[string]string, error) { + var query string if version_from == "" { - query := fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) + query = fmt.Sprintf("select * from %s order by version desc", drv.quotedMigrationsTableName()) } else { - query := fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) + query = fmt.Sprintf("select * from %s where version > '%s' order by version desc", drv.quotedMigrationsTableName(), version_from) } rows, err := db.Query(query) @@ -237,7 +238,7 @@ func (drv *Driver) SelectMigrationsFromVersion(db *sql.DB, version_from string) } if dump.Valid { - migrations[version] = dump + migrations[version] = dump.String } else { migrations[version] = "" } From 55371bd361879b2453e33feba1a1be49f9451e43 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 13:08:35 +0100 Subject: [PATCH 24/46] Add missing sprintf argumetns. --- pkg/driver/clickhouse/clickhouse.go | 4 ++-- pkg/driver/mysql/mysql.go | 2 +- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/sqlite/sqlite.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 97700730..9eb0c17b 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -294,9 +294,9 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { ) engine = %s primary key version order by version; - alter table users + alter table %s%s add column if not exists dump String; - `, drv.quotedMigrationsTableName(), drv.onClusterClause(), engineClause)) + `, drv.quotedMigrationsTableName(), drv.onClusterClause(), engineClause, drv.quotedMigrationsTableName(), drv.onClusterClause())) return err } diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 16bc71cb..aea76fc6 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -247,7 +247,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", - drv.quotedMigrationsTableName())) + drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err } diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index b85275e9..4b2ddd98 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -266,7 +266,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // first attempt at creating migrations table createTableStmt := fmt.Sprintf( "create table if not exists %s.%s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", - schema, migrationsTable) + schema, migrationsTable, schema, migrationsTable) _, err = db.Exec(createTableStmt) if err == nil { // table exists or created successfully diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index e7b089fd..1d5a1c7a 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -176,7 +176,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", - drv.quotedMigrationsTableName())) + drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err } From ce53c29f60db53431c19f2bfa7fbb47c0b7a755a Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 13:11:59 +0100 Subject: [PATCH 25/46] Edit error in postgres and sqlite test. --- pkg/driver/postgres/postgres.go | 2 +- pkg/driver/sqlite/sqlite_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 4b2ddd98..6e068f77 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -265,7 +265,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // first attempt at creating migrations table createTableStmt := fmt.Sprintf( - "create table if not exists %s.%s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", + "create table if not exists %s.%s (version varchar(128) primary key, dump mediumtext); alter table %s.%s add column if not exists dump mediumtext;", schema, migrationsTable, schema, migrationsTable) _, err = db.Exec(createTableStmt) if err == nil { diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index bac53397..fbb15e22 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -362,11 +362,11 @@ func TestSQLiteDeleteMigration(t *testing.T) { require.Equal(t, 1, count) } -func TestSQLLiteUpdateMigrationDump(t *testing.T) { - drv := testMySQLDriver(t) +func TestSQLiteUpdateMigrationDump(t *testing.T) { + drv := testSQLiteDriver(t) drv.migrationsTableName = "test_migrations" - db := prepTestMySQLDB(t) + db := prepTestSQLiteDB(t) defer dbutil.MustClose(db) err := drv.CreateMigrationsTable(db) From c0fbc206d6664f099856ed4146d0879bb74d3360 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 14:13:48 +0100 Subject: [PATCH 26/46] Add missing quotes needed in sqlite. --- pkg/driver/sqlite/sqlite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 1d5a1c7a..68d6c250 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -175,7 +175,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists 'dump' mediumtext;", drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err From acdaefe2f29effbc98ec8a5f04eed4e642ff0aa8 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 14:23:23 +0100 Subject: [PATCH 27/46] Cut adding dump column in existing sqlite db because it doesn't work. --- pkg/driver/sqlite/sqlite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 68d6c250..b902bbc4 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -175,7 +175,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists 'dump' mediumtext;", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext)", drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err From 88869609f2cb6eb457cd0e11b6aa0c891bc10a9f Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 14:28:42 +0100 Subject: [PATCH 28/46] Update sqlite.go --- pkg/driver/sqlite/sqlite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index b902bbc4..e5aaea57 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -176,7 +176,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( "create table if not exists %s (version varchar(128) primary key, dump mediumtext)", - drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) + drv.quotedMigrationsTableName())) return err } From 7107dd9745546f6d1e1e76de6b9948bc1a57e2ed Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 14:32:19 +0100 Subject: [PATCH 29/46] Update clickhouse_test.go --- pkg/driver/clickhouse/clickhouse_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/clickhouse/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go index faca1f63..d214f2b2 100644 --- a/pkg/driver/clickhouse/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -361,7 +361,7 @@ func TestClickHouseUpdateMigrationDump(t *testing.T) { drv := testClickHouseDriver(t) drv.migrationsTableName = "test_migrations" - db := prepTestClickHouseDB(t) + db := prepTestClickHouseDB(t, drv) defer dbutil.MustClose(db) err := drv.CreateMigrationsTable(db) From 5c21229104f918439deaee1f70bf42fce4657da0 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 14:48:13 +0100 Subject: [PATCH 30/46] Edit sql procedure to add column because it was wrong. --- pkg/driver/mysql/mysql.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index aea76fc6..ffb75664 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,8 +246,16 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); alter table %s add column if not exists dump mediumtext;", - drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); + IF NOT EXISTS ( + SELECT * FROM information_schema.COLUMNS + WHERE column_name='dump' + and table_name='%s' + ) + THEN + alter table %s add column if not exists dump mediumtext; + END IF;", + drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err } From a72133443302a2fbad952faeb4b057454149aabd Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 14:51:49 +0100 Subject: [PATCH 31/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index ffb75664..7a7f7c66 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,15 +246,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); - IF NOT EXISTS ( - SELECT * FROM information_schema.COLUMNS - WHERE column_name='dump' - and table_name='%s' - ) - THEN - alter table %s add column if not exists dump mediumtext; - END IF;", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); IF NOT EXISTS ( SELECT * FROM information_schema.COLUMNS WHERE column_name='dump' and table_name='%s' ) THEN alter table %s add column if not exists dump mediumtext; END IF;", drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err From 9855666ca743e5e4196d81929f74a4815ad3cbef Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 15:26:53 +0100 Subject: [PATCH 32/46] Edit mysql add column if exists that should now work. --- pkg/driver/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 7a7f7c66..177c9dd7 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,7 +246,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); IF NOT EXISTS ( SELECT * FROM information_schema.COLUMNS WHERE column_name='dump' and table_name='%s' ) THEN alter table %s add column if not exists dump mediumtext; END IF;", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); DELIMITER // CREATE PROCEDURE AddColumnIfNotExists() BEGIN IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s' AND COLUMN_NAME = 'dump') THEN ALTER TABLE %s ADD COLUMN dump mediumtext; END IF; END // CALL AddColumnIfNotExists(); DROP PROCEDURE AddColumnIfNotExists; // DELIMITER ;", drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err From 3a76bff1ca516b93b08a76a4006a1e61eb290d13 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 15:35:05 +0100 Subject: [PATCH 33/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 177c9dd7..8e10ed5a 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,8 +246,8 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); DELIMITER // CREATE PROCEDURE AddColumnIfNotExists() BEGIN IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s' AND COLUMN_NAME = 'dump') THEN ALTER TABLE %s ADD COLUMN dump mediumtext; END IF; END // CALL AddColumnIfNotExists(); DROP PROCEDURE AddColumnIfNotExists; // DELIMITER ;", - drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); DELIMITER // CREATE PROCEDURE AddColumnIfNotExists() BEGIN IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = %s AND COLUMN_NAME = 'dump') THEN ALTER TABLE %s ADD COLUMN dump mediumtext; END IF; END // CALL AddColumnIfNotExists(); DROP PROCEDURE AddColumnIfNotExists; // DELIMITER ;", + drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.migrationsTableName())) return err } From cea6db32aecec9652f113af153e16f3a388fe663 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 15:36:43 +0100 Subject: [PATCH 34/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 8e10ed5a..735b1147 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -247,7 +247,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( "create table if not exists %s (version varchar(128) primary key, dump mediumtext); DELIMITER // CREATE PROCEDURE AddColumnIfNotExists() BEGIN IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = %s AND COLUMN_NAME = 'dump') THEN ALTER TABLE %s ADD COLUMN dump mediumtext; END IF; END // CALL AddColumnIfNotExists(); DROP PROCEDURE AddColumnIfNotExists; // DELIMITER ;", - drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.migrationsTableName())) + drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err } From a34d444c12f860e67d1f93b8e2415e2e759f171a Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 15:46:41 +0100 Subject: [PATCH 35/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 735b1147..07b8e8ac 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,7 +246,16 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); DELIMITER // CREATE PROCEDURE AddColumnIfNotExists() BEGIN IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = %s AND COLUMN_NAME = 'dump') THEN ALTER TABLE %s ADD COLUMN dump mediumtext; END IF; END // CALL AddColumnIfNotExists(); DROP PROCEDURE AddColumnIfNotExists; // DELIMITER ;", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); DELIMITER // + CREATE PROCEDURE AddColumnIfNotExists() + BEGIN + IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = %s AND COLUMN_NAME = 'dump') THEN + ALTER TABLE %s ADD COLUMN dump mediumtext; + END IF; + END // + CALL AddColumnIfNotExists(); + DROP PROCEDURE AddColumnIfNotExists; // + DELIMITER ;", drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err From 634bc05b065e3476965f375ba8aeac42927ef95b Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 15:48:10 +0100 Subject: [PATCH 36/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 07b8e8ac..fcc77658 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,16 +246,16 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); DELIMITER // - CREATE PROCEDURE AddColumnIfNotExists() - BEGIN - IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = %s AND COLUMN_NAME = 'dump') THEN - ALTER TABLE %s ADD COLUMN dump mediumtext; - END IF; - END // - CALL AddColumnIfNotExists(); - DROP PROCEDURE AddColumnIfNotExists; // - DELIMITER ;", + "create table if not exists %s (version varchar(128) primary key, dump mediumtext); delimiter // + create procedure AddColumnIfNotExists() + begin + if not exists ( select * from information_schema.columns where table_name = %s and column_name = 'dump') then + alter table %s add column dump mediumtext; + end if; + end // + call AddColumnIfNotExists(); + drop procedure AddColumnIfNotExists; // + delimiter ;", drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err From 10eab036dba94c9c73ac9a5cac49a5b0587d2f98 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 15:50:08 +0100 Subject: [PATCH 37/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index fcc77658..38b13826 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -246,7 +246,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf( - "create table if not exists %s (version varchar(128) primary key, dump mediumtext); delimiter // + `create table if not exists %s (version varchar(128) primary key, dump mediumtext); delimiter // create procedure AddColumnIfNotExists() begin if not exists ( select * from information_schema.columns where table_name = %s and column_name = 'dump') then @@ -255,7 +255,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { end // call AddColumnIfNotExists(); drop procedure AddColumnIfNotExists; // - delimiter ;", + delimiter ;`, drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) return err From 1014657c3c2044c9bdff0768891a647c7f962e45 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 16:29:00 +0100 Subject: [PATCH 38/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 38b13826..3118fb00 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -245,20 +245,20 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { - _, err := db.Exec(fmt.Sprintf( - `create table if not exists %s (version varchar(128) primary key, dump mediumtext); delimiter // - create procedure AddColumnIfNotExists() - begin - if not exists ( select * from information_schema.columns where table_name = %s and column_name = 'dump') then - alter table %s add column dump mediumtext; - end if; - end // - call AddColumnIfNotExists(); - drop procedure AddColumnIfNotExists; // - delimiter ;`, - drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName(), drv.quotedMigrationsTableName())) + _, err := db.Exec(fmt.Sprintf("create table if not exists %s (version varchar(128) primary key, dump mediumtext);"), drv.quotedMigrationsTableName()) + if err != nil { + return err + } - return err + // The add column query can fail because the column could be already there. + _, err := db.Exec(fmt.Sprintf("alter table %s add column dump mediumtext;"), drv.quotedMigrationsTableName()) + if err != nil { + fmt.Fprintf(db.Log, "Dump column already there.\n") + } else { + fmt.Fprintf(db.Log, "Dump column added.\n") + } + + return nil } // SelectMigrations returns a list of applied migrations From 7859ad1b6e3084abf299554699a1f96a5e5e9f76 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 16:32:23 +0100 Subject: [PATCH 39/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 3118fb00..19406686 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -251,7 +251,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { } // The add column query can fail because the column could be already there. - _, err := db.Exec(fmt.Sprintf("alter table %s add column dump mediumtext;"), drv.quotedMigrationsTableName()) + _, err = db.Exec(fmt.Sprintf("alter table %s add column dump mediumtext;"), drv.quotedMigrationsTableName()) if err != nil { fmt.Fprintf(db.Log, "Dump column already there.\n") } else { From 78d842329a28cd87a52f8560083da97faf1781dc Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 16:34:27 +0100 Subject: [PATCH 40/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 19406686..f6dc0c79 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -253,9 +253,9 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // The add column query can fail because the column could be already there. _, err = db.Exec(fmt.Sprintf("alter table %s add column dump mediumtext;"), drv.quotedMigrationsTableName()) if err != nil { - fmt.Fprintf(db.Log, "Dump column already there.\n") + fmt.Fprintf(drv.Log, "Dump column already there.\n") } else { - fmt.Fprintf(db.Log, "Dump column added.\n") + fmt.Fprintf(drv.Log, "Dump column added.\n") } return nil From b14a0452672922cbc8041bc27409b7ec6da4862f Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 16:37:50 +0100 Subject: [PATCH 41/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index f6dc0c79..c0e8d575 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -253,9 +253,9 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // The add column query can fail because the column could be already there. _, err = db.Exec(fmt.Sprintf("alter table %s add column dump mediumtext;"), drv.quotedMigrationsTableName()) if err != nil { - fmt.Fprintf(drv.Log, "Dump column already there.\n") + fmt.Fprintf(drv.log, "Dump column already there.\n") } else { - fmt.Fprintf(drv.Log, "Dump column added.\n") + fmt.Fprintf(drv.log, "Dump column added.\n") } return nil From 0e59417e0fce9092203e26a94a7fcff21dda5609 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 16:43:39 +0100 Subject: [PATCH 42/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index c0e8d575..7204b7e7 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -251,7 +251,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { } // The add column query can fail because the column could be already there. - _, err = db.Exec(fmt.Sprintf("alter table %s add column dump mediumtext;"), drv.quotedMigrationsTableName()) + _, err = db.Exec(fmt.Sprintf("alter table %s add column dump mediumtext;", drv.quotedMigrationsTableName())) if err != nil { fmt.Fprintf(drv.log, "Dump column already there.\n") } else { From 15591f0e55a419712eef6e3f2b0714f454abb524 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 16:46:24 +0100 Subject: [PATCH 43/46] Update mysql.go --- pkg/driver/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index 7204b7e7..f6411657 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -245,7 +245,7 @@ func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { - _, err := db.Exec(fmt.Sprintf("create table if not exists %s (version varchar(128) primary key, dump mediumtext);"), drv.quotedMigrationsTableName()) + _, err := db.Exec(fmt.Sprintf("create table if not exists %s (version varchar(128) primary key, dump mediumtext);", drv.quotedMigrationsTableName())) if err != nil { return err } From bce201110e90405776d3d1bc34aa5b760b5a953d Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Wed, 12 Mar 2025 17:09:32 +0100 Subject: [PATCH 44/46] Edit version to v1.0.0. --- pkg/dbmate/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/version.go b/pkg/dbmate/version.go index 62a2cc7e..13c32e02 100644 --- a/pkg/dbmate/version.go +++ b/pkg/dbmate/version.go @@ -1,4 +1,4 @@ package dbmate // Version of dbmate -const Version = "2.26.0" +const Version = "1.0.0" From 48c270061472241cc7b2c186021eb58628b808dc Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Thu, 21 May 2026 16:29:20 +0200 Subject: [PATCH 45/46] Add migrations-url argument for getting migrations from another database. --- main.go | 21 ++- pkg/dbmate/db.go | 372 +++++++++++++++++++++++++++++----------- pkg/dbmate/migration.go | 106 ++++++++---- 3 files changed, 364 insertions(+), 135 deletions(-) diff --git a/main.go b/main.go index b754aa57..f3763c86 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,11 @@ func NewApp() *cli.App { Value: cli.NewStringSlice(defaultDB.MigrationsDir[0]), Usage: "specify the directory containing migration files", }, + &cli.StringFlag{ + Name: "migrations-url", + EnvVars: []string{"DBMATE_MIGRATIONS_URL"}, + Usage: "specify a database URL to read migrations from (sync only). If set, sync reads migrations from this DB instead of filesystem", + }, &cli.StringFlag{ Name: "migrations-table", EnvVars: []string{"DBMATE_MIGRATIONS_TABLE"}, @@ -184,8 +189,8 @@ func NewApp() *cli.App { }), }, { - Name: "sync", - Usage: "Strips eventual later migrations or migrate up in case the database is older", + Name: "sync", + Usage: "Strips eventual later migrations or migrate up in case the database is older. Uses filesystem migrations by default; if --migrations-url is set, reads migrations from that database.", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "verbose", @@ -196,6 +201,12 @@ func NewApp() *cli.App { }, Action: action(func(db *dbmate.DB, c *cli.Context) error { db.Verbose = c.Bool("verbose") + + // check ambiguità + if db.MigrationsUrl != "" && c.IsSet("migrations-dir") { + return fmt.Errorf("cannot use both --migrations-url and --migrations-dir with sync: choose one source") + } + return db.Synchronize() }), }, @@ -314,10 +325,16 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { db.SchemaFile = c.String("schema-file") db.WaitBefore = c.Bool("wait") waitTimeout := c.Duration("wait-timeout") + db.MigrationsUrl = c.String("migrations-url") // ✅ AGGIUNTO + if waitTimeout != 0 { db.WaitTimeout = waitTimeout } + if c.String("migrations-url") != "" && c.Command.Name != "sync" { + return fmt.Errorf("--migrations-url is supported only for the sync command") + } + return f(db, c) } } diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 21afe549..3298ef0e 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -46,6 +46,8 @@ type DB struct { Log io.Writer // MigrationsDir specifies the directory or directories to find migration files MigrationsDir []string + // MigrationsUrl specifies the db url to find migrations table + MigrationsUrl string // MigrationsTableName specifies the database table to record migrations in MigrationsTableName string // SchemaFile specifies the location for schema.sql file @@ -332,6 +334,25 @@ func (db *DB) openDatabaseForMigration(drv Driver) (*sql.DB, error) { return sqlDB, nil } +func (db *DB) EnsureDumpColumnEncoding(sqlDB *sql.DB) error { + query := fmt.Sprintf(` + ALTER TABLE %s + MODIFY dump LONGTEXT + CHARACTER SET utf8mb4 + COLLATE utf8mb4_unicode_ci + `, db.MigrationsTableName) + + _, err := sqlDB.Exec(query) + if err != nil { + // non bloccare se già ok o se non serve + fmt.Fprintf(db.Log, "Warning: cannot alter dump column charset: %v\n", err) + return nil + } + + fmt.Fprintf(db.Log, "Dump column charset ensured (utf8mb4)\n") + return nil +} + // Migrate migrates database to the latest version func (db *DB) Migrate() error { drv, err := db.Driver() @@ -394,7 +415,7 @@ func (db *DB) Migrate() error { } // record migration - return drv.InsertMigration(tx, migration.Version, parsed.Down) + return drv.InsertMigration(tx, migration.Version, parsed.Full) } if parsed.UpOptions.Transaction() { @@ -514,94 +535,228 @@ func (db *DB) FindMigrations() ([]Migration, error) { return migrations, nil } -// UpdateEmptyDumps lists all available migrations -func (db *DB) UpdateEmptyDumps() (error) { - drv, err := db.Driver() - if err != nil { - return err - } - - sqlDB, err := drv.Open() - if err != nil { - return err - } - defer dbutil.MustClose(sqlDB) - - // find applied migrations - migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) - if err != nil { - return err - } - - // Get all migrations dump from database. - dumpMigrations := map[string]string{} - if migrationsTableExists { - dumpMigrations, err = drv.SelectMigrationsFromVersion(sqlDB, "") - if err != nil { - return err - } - } - - // Get all migrations dump from database. - appliedMigrations := map[string]bool{} - if migrationsTableExists { - appliedMigrations, err = drv.SelectMigrations(sqlDB, -1) - if err != nil { - return err - } - } - - for _, dir := range db.MigrationsDir { - // find filesystem migrations - files, err := db.readMigrationsDir(dir) - if err != nil { - return fmt.Errorf("%w `%s`", ErrMigrationDirNotFound, dir) - } - - for _, file := range files { - if file.IsDir() { - continue - } - - matches := migrationFileRegexp.FindStringSubmatch(file.Name()) - if len(matches) < 2 { - continue - } - - migration := Migration{ - Applied: false, - FileName: matches[0], - FilePath: path.Join(dir, matches[0]), - FS: db.FS, - Version: matches[1], - } - if ok := appliedMigrations[migration.Version]; ok { - // Migration already applied, check if the dump column in db is eventually empty. - if dumpMigrations[migration.Version] == "" { - fmt.Fprintf(db.Log, "Updating dump column of: %s\n", migration.Version) - - start := time.Now() - - parsed, err := migration.Parse() - if err != nil { - return err - } - - dumpMigrations[migration.Version] = parsed.Down - - err = drv.UpdateMigrationDump(sqlDB, migration.Version, parsed.Down) - if err != nil { - return err - } - - elapsed := time.Since(start) - fmt.Fprintf(db.Log, "Updated dump column of: %s in %s\n", migration.Version, elapsed) - } - } - } - } +// FindMigrationsFromDB lists all available migrations reading them from a DB specified by db.MigrationsUrl. +// It marks migrations as Applied based on the target database (db.DatabaseURL), exactly like FindMigrations(). +func (db *DB) FindMigrationsFromDB() ([]Migration, error) { + // migrations-url must be provided + if db.MigrationsUrl == "" { + return db.FindMigrations() + } + + // ------------------------------------------------------------ + // 1) Open TARGET DB (where we apply migrations) to know what is already applied + // ------------------------------------------------------------ + drv, err := db.Driver() + if err != nil { + return nil, err + } + + targetSQL, err := drv.Open() + if err != nil { + return nil, err + } + defer dbutil.MustClose(targetSQL) + + // find applied migrations on TARGET + appliedMigrations := map[string]bool{} + migrationsTableExists, err := drv.MigrationsTableExists(targetSQL) + if err != nil { + return nil, err + } + if migrationsTableExists { + appliedMigrations, err = drv.SelectMigrations(targetSQL, -1) + if err != nil { + return nil, err + } + } + + // ------------------------------------------------------------ + // 2) Open SOURCE DB (db.MigrationsUrl) to read available migrations + their dumps + // ------------------------------------------------------------ + migURL, err := url.Parse(db.MigrationsUrl) + if err != nil { + return nil, fmt.Errorf("invalid migrations-url: %w", err) + } + + // Create a lightweight DB instance for SOURCE, reusing same configuration where sensible + srcDB := *db + srcDB.DatabaseURL = migURL + + srcDrv, err := srcDB.Driver() + if err != nil { + return nil, err + } + + sourceSQL, err := srcDrv.Open() + if err != nil { + return nil, err + } + defer dbutil.MustClose(sourceSQL) + + // Ensure migrations table exists on SOURCE + sourceTableExists, err := srcDrv.MigrationsTableExists(sourceSQL) + if err != nil { + return nil, err + } + if !sourceTableExists { + // no migrations available on source + return []Migration{}, nil + } + + // ------------------------------------------------------------ + // 3) Read (version, dump) from SOURCE migrations table + // ------------------------------------------------------------ + // NOTE: assumes your fork added a "dump" column to the migrations table + // and that it contains the FULL text with both -- migrate:up and -- migrate:down blocks. + query := fmt.Sprintf( + "SELECT version, dump FROM %s ORDER BY version", + db.MigrationsTableName, + ) + + rows, err := sourceSQL.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + migrations := []Migration{} + for rows.Next() { + var version string + var dump string + if err := rows.Scan(&version, &dump); err != nil { + return nil, err + } + + m := Migration{ + Applied: false, + FileName: fmt.Sprintf("%s.sql", version), // keep sorting consistent with file-based + Version: version, + + // DB-based payload + Dump: dump, + } + + // mark as applied if TARGET already has this version + if ok := appliedMigrations[m.Version]; ok { + m.Applied = true + } + + migrations = append(migrations, m) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + // ------------------------------------------------------------ + // 4) Sort (same behavior as FindMigrations: by FileName) + // ------------------------------------------------------------ + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].FileName < migrations[j].FileName + }) + + return migrations, nil +} - return nil +// UpdateEmptyDumps updates migrations table "dump" column: +// - if empty => stores full (up+down) +// - if only down => upgrades to full (up+down) +// - if already full => does nothing +func (db *DB) UpdateEmptyDumps() error { + drv, err := db.Driver() + if err != nil { + return err + } + + sqlDB, err := drv.Open() + if err != nil { + return err + } + defer dbutil.MustClose(sqlDB) + + migrationsTableExists, err := drv.MigrationsTableExists(sqlDB) + if err != nil { + return err + } + if !migrationsTableExists { + return nil + } + + // Get all migrations dump from database (version -> dump) + dumpMigrations, err := drv.SelectMigrationsFromVersion(sqlDB, "") + if err != nil { + return err + } + + // Get applied migrations from database (version -> true) + appliedMigrations, err := drv.SelectMigrations(sqlDB, -1) + if err != nil { + return err + } + + for _, dir := range db.MigrationsDir { + files, err := db.readMigrationsDir(dir) + if err != nil { + return fmt.Errorf("%w `%s`", ErrMigrationDirNotFound, dir) + } + + for _, file := range files { + if file.IsDir() { + continue + } + + matches := migrationFileRegexp.FindStringSubmatch(file.Name()) + if len(matches) < 2 { + continue + } + + migration := Migration{ + Applied: false, + FileName: matches[0], + FilePath: path.Join(dir, matches[0]), + FS: db.FS, + Version: matches[1], + } + + // only touch migrations that are applied on this DB + if !appliedMigrations[migration.Version] { + continue + } + + currentDump := dumpMigrations[migration.Version] + + // Case 3: already full -> do nothing + if dumpHasUpDown(currentDump) { + continue + } + + // Case 1: empty OR Case 2: only down OR any other "non-full" legacy format -> upgrade to full + // (dumpHasOnlyDown(currentDump) is informative, but the action is the same: upgrade) + fmt.Fprintf(db.Log, "Updating dump column of: %s\n", migration.Version) + start := time.Now() + + parsed, err := migration.Parse() + if err != nil { + return err + } + + // parsed.Full is the whole file (up+down) you added at step 3 + newDump := parsed.Full + + err = drv.UpdateMigrationDump(sqlDB, migration.Version, newDump) + if err != nil { + return err + } + + // keep local cache coherent + dumpMigrations[migration.Version] = newDump + + elapsed := time.Since(start) + fmt.Fprintf(db.Log, "Updated dump column of: %s in %s\n", migration.Version, elapsed) + } + } + + return nil } // Rollback rolls back the most recent migration @@ -692,17 +847,27 @@ func (db *DB) Synchronize() error { } defer dbutil.MustClose(sqlDB) - // find last applied migration - migrations, err := db.FindMigrations() - if err != nil { + // ensure dump column charset is correct + if err := db.EnsureDumpColumnEncoding(sqlDB); err != nil { return err } - err = db.UpdateEmptyDumps() + // find last applied migration + var migrations []Migration + if db.MigrationsUrl != "" { + migrations, err = db.FindMigrationsFromDB() // nuovo + } else { + migrations, err = db.FindMigrations() // esistente + } if err != nil { return err } + if db.MigrationsUrl == "" { + // file-mode: posso normalizzare dump dal filesystem + if err := db.UpdateEmptyDumps(); err != nil { return err } + } + highestAppliedMigrationVersion := "" pendingMigrations := []Migration{} for _, migration := range migrations { @@ -746,7 +911,7 @@ func (db *DB) Synchronize() error { } // record migration - return drv.InsertMigration(tx, migration.Version, parsed.Down) + return drv.InsertMigration(tx, migration.Version, parsed.Full) } if parsed.UpOptions.Transaction() { @@ -771,7 +936,11 @@ func (db *DB) Synchronize() error { return err } - fmt.Fprintf(db.Log, "Latest available migration file: %s.sql\n", highestAppliedMigrationVersion) + if highestAppliedMigrationVersion == "" { + fmt.Fprintf(db.Log, "No applied migrations found on target DB\n") + } else { + fmt.Fprintf(db.Log, "Latest available migration file: %s.sql\n", highestAppliedMigrationVersion) + } migrationsToBeRolledBack := map[string]string{} if migrationsTableExists { @@ -780,15 +949,24 @@ func (db *DB) Synchronize() error { return err } } - for version, dump := range migrationsToBeRolledBack { + + versions := make([]string, 0, len(migrationsToBeRolledBack)) + for v := range migrationsToBeRolledBack { + versions = append(versions, v) + } + sort.Sort(sort.Reverse(sort.StringSlice(versions))) + + for _, version := range versions { + dump := migrationsToBeRolledBack[version] fmt.Fprintf(db.Log, "Rolling back later db migration: %s\n", version) start := time.Now() // run actual migration dump rollback - result, err := sqlDB.Exec(dump) + downSQL := extractDown(dump) // nuova helper (fallback compatibile con down-only) + result, err := sqlDB.Exec(downSQL) if err != nil { - return drv.QueryError(dump, err) + return drv.QueryError(downSQL, err) } else if db.Verbose { db.printVerbose(result) } diff --git a/pkg/dbmate/migration.go b/pkg/dbmate/migration.go index ec67f7cf..af4aae8c 100644 --- a/pkg/dbmate/migration.go +++ b/pkg/dbmate/migration.go @@ -15,6 +15,7 @@ type Migration struct { FilePath string FS fs.FS Version string + Dump string } func (m *Migration) readFile() (string, error) { @@ -29,12 +30,16 @@ func (m *Migration) readFile() (string, error) { // Parse a migration func (m *Migration) Parse() (*ParsedMigration, error) { - contents, err := m.readFile() - if err != nil { - return nil, err - } + if m.Dump != "" { + return parseMigrationContents(m.Dump) + } + + contents, err := m.readFile() + if err != nil { + return nil, err + } - return parseMigrationContents(contents) + return parseMigrationContents(contents) } // ParsedMigration contains the migration contents and options @@ -43,6 +48,7 @@ type ParsedMigration struct { UpOptions ParsedMigrationOptions Down string DownOptions ParsedMigrationOptions + Full string } // ParsedMigrationOptions is an interface for accessing migration options @@ -76,38 +82,34 @@ var ( ErrParseUnexpectedStmt = errors.New("dbmate does not support statements preceding the '-- migrate:up' block") ) -// parseMigrationContents parses the string contents of a migration. -// It will return two Migration objects, the first representing the "up" -// block and the second representing the "down" block. This function -// requires that at least an up block was defined and will otherwise -// return an error. func parseMigrationContents(contents string) (*ParsedMigration, error) { - upDirectiveStart, hasDefinedUpBlock := getMatchPosition(contents, upRegExp) - downDirectiveStart, hasDefinedDownBlock := getMatchPosition(contents, downRegExp) - - if !hasDefinedUpBlock { - return nil, ErrParseMissingUp - } - if !hasDefinedDownBlock { - return nil, ErrParseMissingDown - } - if upDirectiveStart > downDirectiveStart { - return nil, ErrParseWrongOrder - } - if statementsPrecedeMigrateBlocks(contents, upDirectiveStart) { - return nil, ErrParseUnexpectedStmt - } - - upBlock := substring(contents, upDirectiveStart, downDirectiveStart) - downBlock := substring(contents, downDirectiveStart, len(contents)) - - parsed := ParsedMigration{ - Up: upBlock, - UpOptions: parseMigrationOptions(upBlock), - Down: downBlock, - DownOptions: parseMigrationOptions(downBlock), - } - return &parsed, nil + upDirectiveStart, hasDefinedUpBlock := getMatchPosition(contents, upRegExp) + downDirectiveStart, hasDefinedDownBlock := getMatchPosition(contents, downRegExp) + + if !hasDefinedUpBlock { + return nil, ErrParseMissingUp + } + if !hasDefinedDownBlock { + return nil, ErrParseMissingDown + } + if upDirectiveStart > downDirectiveStart { + return nil, ErrParseWrongOrder + } + if statementsPrecedeMigrateBlocks(contents, upDirectiveStart) { + return nil, ErrParseUnexpectedStmt + } + + upBlock := substring(contents, upDirectiveStart, downDirectiveStart) + downBlock := substring(contents, downDirectiveStart, len(contents)) + + parsed := ParsedMigration{ + Up: upBlock, + UpOptions: parseMigrationOptions(upBlock), + Down: downBlock, + DownOptions: parseMigrationOptions(downBlock), + Full: contents, + } + return &parsed, nil } // parseMigrationOptions parses the migration options out of a block @@ -150,6 +152,38 @@ func parseMigrationOptions(contents string) ParsedMigrationOptions { return options } +func dumpHasUpDown(dump string) bool { + upStart, hasUp := getMatchPosition(dump, upRegExp) + downStart, hasDown := getMatchPosition(dump, downRegExp) + return hasUp && hasDown && upStart < downStart +} + +func dumpHasOnlyDown(dump string) bool { + _, hasUp := getMatchPosition(dump, upRegExp) + _, hasDown := getMatchPosition(dump, downRegExp) + return !hasUp && hasDown +} + +func extractDown(dump string) string { + // se non c’è il marker down, assume che dump sia già "down-only" + downStart, hasDown := getMatchPosition(dump, downRegExp) + if !hasDown { + return dump + } + + // prendi la riga del direttivo "-- migrate:down ..." e poi tutto quello che segue + // trova fine riga dopo downStart + i := downStart + for i < len(dump) && dump[i] != '\n' && dump[i] != '\r' { + i++ + } + // salta \r\n o \n + for i < len(dump) && (dump[i] == '\n' || dump[i] == '\r') { + i++ + } + return dump[i:] +} + // statementsPrecedeMigrateBlocks inspects the contents between the first character // of a string and the index of the first block directive to see if there are any statements // defined outside of the block directive. It'll return true if it finds any such statements. From 023db220e1ab518b470081b675b15abb572be254 Mon Sep 17 00:00:00 2001 From: Michele Manotti Date: Thu, 28 May 2026 17:08:41 +0200 Subject: [PATCH 46/46] Edit version to v1.0.1. --- pkg/dbmate/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/version.go b/pkg/dbmate/version.go index 13c32e02..b6be519b 100644 --- a/pkg/dbmate/version.go +++ b/pkg/dbmate/version.go @@ -1,4 +1,4 @@ package dbmate // Version of dbmate -const Version = "1.0.0" +const Version = "1.0.1"