test(e2e): run schema-drift migrations in an isolated database

Follow-up fixes from the review of apache#9015:
- add e2ehelper.NewIsolatedMigrationDb so the schema-drift guards run the real
  migration scripts against a dedicated, empty database instead of the shared
  E2E_DB_URL one
- use it in the jira and cross-plugin schema-drift tests
- fix the teambition scope-config migration accordingly
- document the findings in AGENTS.md

Author: DoDiODev

AGENTS.md

@@ -180,14 +180,44 @@ Notes:
 - `backend/plugins/schema_e2e/` is **not** a plugin (no `main` package); it is
   excluded in `backend/scripts/build-plugins.sh`, otherwise `make build-plugin`
   fails with "-buildmode=plugin requires exactly one main package".
-- Both tests set a fallback `ENCRYPTION_SECRET` and call `dalgorm.Init(...)`;
+- Both tests run the migrations in a **dedicated, empty database**
+  (`<e2e-db>_<suffix>`) created by `e2ehelper.NewIsolatedMigrationDb` and dropped
+  again afterwards. The shared `E2E_DB_URL` database must not be used: the other
+  e2e tests `AutoMigrate` tables without writing to `_devlake_migration_history`,
+  so re-running the real scripts against it fails with e.g. `Table
+  'cicd_pipeline_commits' already exists`.
+- The helper sets a fallback `ENCRYPTION_SECRET` and calls `dalgorm.Init(...)`;
   without it migrations fail with `invalid encKey` / `invalid serializer type
   encdec` (`runner.CreateBasicRes` does not register the serializer, only
   `CreateAppBasicRes` does).
+- An **auto-increment primary key cannot be added to an existing table** via
+  `AutoMigrate` (MySQL: "there can be only one auto column and it must be
+  defined as a key"); use explicit dialect-specific DDL (`BIGINT UNSIGNED NOT
+  NULL AUTO_INCREMENT PRIMARY KEY` / `BIGSERIAL PRIMARY KEY`). Do **not** use
+  `migrationhelper.TransformTable` for this: it copies rows with a zero PK, so
+  all of them collapse into a single upserted row.
 - Migration scripts must not import `core/models/common` (enforced by
   `make migration-script-lint`) — use `core/models/migrationscripts/archived`.
 
 
+### CI im Fork (GitHub Actions)
+Die Workflows in `.github/workflows/` sind (bis auf Verbesserungen in `build.yml`)
+identisch mit `upstream/main`. Zwei Fork-spezifische Fallstricke:
+- Workflows mit `schedule:`-Trigger werden von GitHub in Forks automatisch
+  deaktiviert (`disabled_fork`) — betraf `unit-test` (`test.yml`). Mit
+  `gh workflow enable <id>` reaktivieren; `stale.yml` bleibt bewusst deaktiviert.
+- Die Upstream-Workflows triggern nur auf `push` nach `main` bzw.
+  `pull_request` mit Base `main`/`release-*`. Fork-Feature-Branches (`pr/**`)
+  und *stacked* PRs (Base ≠ `main`) bekommen dadurch keine CI.
+  Dafür existiert `.github/workflows/fork-ci.yml` (`branches-ignore: [main]`,
+  `workflow_dispatch`, `if: github.repository != 'apache/devlake'`), das
+  golangci-lint, migration-script-lint, unit-test, e2e (MySQL), config-ui,
+  ASF-Header- und Grafana-Check spiegelt.
+  **Wichtig:** `fork-ci.yml` gehört nur in den Fork-`main`. Upstream-PR-Branches
+  von `upstream/main` abzweigen — bei `pull_request`-Events nutzt GitHub die
+  Workflows des Merge-Refs, die Datei greift also trotzdem, ohne im Upstream-PR
+  als Diff aufzutauchen.
+
 ## Python Plugins
 Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/python/README.md](backend/python/README.md).
 Python is pinned to **3.11**. Note (verified 2026-07-28): Pydantic was migrated to

backend/helpers/e2ehelper/migration_db.go

@@ -0,0 +1,126 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2ehelper
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/apache/incubator-devlake/core/config"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/runner"
+	"github.com/apache/incubator-devlake/impls/dalgorm"
+	"github.com/apache/incubator-devlake/impls/logruslog"
+	"gorm.io/gorm"
+)
+
+// NewIsolatedMigrationDb creates a dedicated, empty database next to the one
+// referenced by E2E_DB_URL (`<e2e-db>_<suffix>`) and returns a connection to it.
+//
+// Tests that execute the REAL migration scripts must not share the regular e2e
+// database: the other plugin e2e tests seed/AutoMigrate tables (domain layer
+// included) without recording anything in `_devlake_migration_history`, so a
+// subsequent migration run fails with errors such as
+// "Table 'cicd_pipeline_commits' already exists" when it tries to create or
+// rename a table that is already there.
+//
+// The database is dropped again when the test finishes. If E2E_DB_URL is not
+// set the test is skipped.
+func NewIsolatedMigrationDb(t *testing.T, suffix string) *gorm.DB {
+	cfg := config.GetConfig()
+	e2eDbUrl := cfg.GetString("E2E_DB_URL")
+	if e2eDbUrl == "" {
+		t.Skip("E2E_DB_URL is not set; skipping migration schema check")
+	}
+	u, err := url.Parse(e2eDbUrl)
+	if err != nil {
+		t.Fatalf("unable to parse E2E_DB_URL: %v", err)
+	}
+	isolatedName := fmt.Sprintf("%s_%s", strings.TrimPrefix(u.Path, "/"), suffix)
+	quotedName := quoteDbName(u.Scheme, isolatedName)
+
+	gormConf := &gorm.Config{SkipDefaultTransaction: true}
+	adminDb, err := runner.MakeDbConnection(e2eDbUrl, gormConf)
+	if err != nil {
+		t.Fatalf("unable to connect to E2E_DB_URL: %v", err)
+	}
+	if err = adminDb.Exec("DROP DATABASE IF EXISTS " + quotedName).Error; err != nil {
+		t.Fatalf("unable to drop leftover database %s: %v", isolatedName, err)
+	}
+	if err = adminDb.Exec("CREATE DATABASE " + quotedName).Error; err != nil {
+		t.Fatalf("unable to create database %s: %v", isolatedName, err)
+	}
+	closeDb(adminDb)
+
+	isolatedUrl := *u
+	isolatedUrl.Path = "/" + isolatedName
+	db, err := runner.MakeDbConnection(isolatedUrl.String(), gormConf)
+	if err != nil {
+		t.Fatalf("unable to connect to %s: %v", isolatedName, err)
+	}
+
+	// migration scripts and models read DB_URL from the global config, keep it
+	// consistent with the connection we hand out and restore it afterwards.
+	previousDbUrl := cfg.GetString("DB_URL")
+	cfg.Set("DB_URL", isolatedUrl.String())
+
+	// Some migration scripts refuse to run without an encryption secret
+	// (e.g. jira 20220716: "jira v0.11 invalid encKey"). CI does not
+	// necessarily provide one, so fall back to a deterministic test value.
+	// dalgorm.Init registers the `encdec` GORM serializer used by connection
+	// models - without it migrations fail with "invalid serializer type encdec"
+	// (runner.CreateBasicRes does not register it, only CreateAppBasicRes does).
+	if cfg.GetString(plugin.EncodeKeyEnvStr) == "" {
+		cfg.Set(plugin.EncodeKeyEnvStr, "devlake-e2e-test-encryption-secret")
+	}
+	dalgorm.Init(cfg.GetString(plugin.EncodeKeyEnvStr))
+
+	t.Cleanup(func() {
+		cfg.Set("DB_URL", previousDbUrl)
+		closeDb(db)
+		cleanupDb, cleanupErr := runner.MakeDbConnection(e2eDbUrl, gormConf)
+		if cleanupErr != nil {
+			t.Logf("unable to connect for dropping %s: %v", isolatedName, cleanupErr)
+			return
+		}
+		defer closeDb(cleanupDb)
+		if dropErr := cleanupDb.Exec("DROP DATABASE IF EXISTS " + quotedName).Error; dropErr != nil {
+			t.Logf("unable to drop database %s: %v", isolatedName, dropErr)
+		}
+	})
+
+	logruslog.Global.Info("running migrations against isolated database %s", isolatedName)
+	return db
+}
+
+func quoteDbName(scheme string, name string) string {
+	// database names are derived from E2E_DB_URL + a constant suffix, but quote
+	// them anyway to stay safe with reserved words.
+	if strings.EqualFold(scheme, "mysql") {
+		return "`" + strings.ReplaceAll(name, "`", "") + "`"
+	}
+	return `"` + strings.ReplaceAll(name, `"`, "") + `"`
+}
+
+func closeDb(db *gorm.DB) {
+	if sqlDb, err := db.DB(); err == nil {
+		_ = sqlDb.Close()
+	}
+}

backend/plugins/jira/e2e/migration_schema_test.go

@@ -21,13 +21,14 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/apache/incubator-devlake/core/config"
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/migration"
 	coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts"
-	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/core/runner"
 	"github.com/apache/incubator-devlake/helpers/e2ehelper"
 	"github.com/apache/incubator-devlake/impls/dalgorm"
+	"github.com/apache/incubator-devlake/impls/logruslog"
 	"github.com/apache/incubator-devlake/plugins/jira/impl"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -52,23 +53,20 @@ import (
 // the migrated table. Any future migration that forgets to embed
 // common.NoPKModel (or otherwise omits a column) will fail this test.
 //
+// The migrations run against a dedicated, empty database (see
+// e2ehelper.NewIsolatedMigrationDb) because the shared e2e database is polluted
+// by the other e2e tests, which AutoMigrate tables without recording anything
+// in `_devlake_migration_history`.
+//
 // Requires E2E_DB_URL (runs under `make e2e-test` / `make e2e-test-go-plugins`).
 func TestMigrationSchema(t *testing.T) {
 	var pluginInstance impl.Jira
-	dataflowTester := e2ehelper.NewDataFlowTester(t, "jira", pluginInstance)
 
-	// Some jira migration scripts refuse to run without an encryption secret
-	// (20220716: "jira v0.11 invalid encKey"); CI does not necessarily provide
-	// one, so fall back to a deterministic test value. dalgorm.Init registers
-	// the `encdec` GORM serializer used by connection models — without it the
-	// migrations fail with "invalid serializer type encdec".
-	if dataflowTester.Cfg.GetString(plugin.EncodeKeyEnvStr) == "" {
-		dataflowTester.Cfg.Set(plugin.EncodeKeyEnvStr, "jira-e2e-test-encryption-secret")
-	}
-	dalgorm.Init(dataflowTester.Cfg.GetString(plugin.EncodeKeyEnvStr))
+	db := e2ehelper.NewIsolatedMigrationDb(t, "jira_migration_schema")
+	dalInstance := dalgorm.NewDalgorm(db)
 
 	// Apply the real migration scripts so the schema matches a production install.
-	basicRes := runner.CreateBasicRes(dataflowTester.Cfg, dataflowTester.Log, dataflowTester.Db)
+	basicRes := runner.CreateBasicRes(config.GetConfig(), logruslog.Global, db)
 	migrator, err := migration.NewMigrator(basicRes)
 	require.NoError(t, err)
 	migrator.Register(coreMigration.All(), "Framework")
@@ -85,7 +83,7 @@ func TestMigrationSchema(t *testing.T) {
 			require.NoErrorf(t, err, "unable to parse schema for %T", table)
 
 			// Columns that actually exist in the migrated table.
-			actualColumns, err := dal.GetColumnNames(dataflowTester.Dal, table, keepAll)
+			actualColumns, err := dal.GetColumnNames(dalInstance, table, keepAll)
 			if err != nil || len(actualColumns) == 0 {
 				// No migration creates this table (e.g. API response models
 				// that are listed in GetTablesInfo but never persisted) —

backend/plugins/schema_e2e/migration_schema_test.go

@@ -38,6 +38,7 @@ import (
 	coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/core/runner"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
 	"github.com/apache/incubator-devlake/impls/dalgorm"
 	"github.com/apache/incubator-devlake/impls/logruslog"
 	"github.com/stretchr/testify/assert"
@@ -173,28 +174,15 @@ func TestAllGoPluginsListed(t *testing.T) {
 // Tables that no migration creates (e.g. models materialized lazily at runtime)
 // are skipped, so the check specifically targets *drift* between an existing
 // table and its model — which is exactly the failure mode above.
+//
+// The migrations run against a dedicated, empty database (see
+// e2ehelper.NewIsolatedMigrationDb) because the shared e2e database is polluted
+// by the other e2e tests, which AutoMigrate tables without recording anything
+// in `_devlake_migration_history`.
 func TestMigrationSchemaMatchesModels(t *testing.T) {
-	cfg := config.GetConfig()
-	e2eDbURL := cfg.GetString("E2E_DB_URL")
-	if e2eDbURL == "" {
-		t.Skip("E2E_DB_URL is not set; skipping cross-plugin migration schema check")
-	}
-	cfg.Set("DB_URL", e2eDbURL)
-	// Some migration scripts refuse to run without an encryption secret
-	// (e.g. jira 20220716: "jira v0.11 invalid encKey"). CI does not
-	// necessarily provide one, so fall back to a deterministic test value.
-	if cfg.GetString(plugin.EncodeKeyEnvStr) == "" {
-		cfg.Set(plugin.EncodeKeyEnvStr, "schema-e2e-test-encryption-secret")
-	}
-
-	db, err := runner.NewGormDb(cfg, logruslog.Global)
-	require.NoError(t, err)
-	// register the `encdec` GORM serializer, otherwise migration scripts that
-	// touch connection models fail with "invalid serializer type encdec"
-	// (runner.CreateBasicRes does not do this, only CreateAppBasicRes does).
-	dalgorm.Init(cfg.GetString(plugin.EncodeKeyEnvStr))
+	db := e2ehelper.NewIsolatedMigrationDb(t, "schema_drift")
 	dalInstance := dalgorm.NewDalgorm(db)
-	basicRes := runner.CreateBasicRes(cfg, logruslog.Global, db)
+	basicRes := runner.CreateBasicRes(config.GetConfig(), logruslog.Global, db)
 
 	// Apply the migrations exactly the way the server does on startup.
 	migrator, migErr := migration.NewMigrator(basicRes)

backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go

@@ -18,23 +18,20 @@ limitations under the License.
 package migrationscripts
 
 import (
+	"fmt"
+
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
 	"github.com/apache/incubator-devlake/helpers/migrationhelper"
 )
 
+const teambitionScopeConfigTable20260727 = "_tool_teambition_scope_configs"
+
 // teambitionScopeConfig20260727 mirrors models.TeambitionScopeConfig. The
 // migration that created `_tool_teambition_scope_configs` did not include the
 // columns of the embedded common.Model (`id`, `created_at`, `updated_at`),
 // which the runtime model expects.
-//
-// Adding the AUTO_INCREMENT primary key `id` to the already existing (and
-// primary-key-less) table works: GORM issues a plain ADD COLUMN and MySQL
-// backfills consecutive ids for existing rows — verified against MySQL 8.4.
-// The `uniqueIndex` on `name` is likewise safe, because newly added columns are
-// nullable and both MySQL and PostgreSQL permit duplicate NULLs in a unique
-// index.
 type teambitionScopeConfig20260727 struct {
 	archived.Model
 	Entities          []string          `gorm:"type:json;serializer:json" json:"entities"`
@@ -48,12 +45,39 @@ type teambitionScopeConfig20260727 struct {
 }
 
 func (teambitionScopeConfig20260727) TableName() string {
-	return "_tool_teambition_scope_configs"
+	return teambitionScopeConfigTable20260727
 }
 
 type addMissingScopeConfigColumns struct{}
 
+// Up adds the columns of the embedded common.Model that the runtime model
+// expects.
+//
+// `id` is an auto-increment primary key, which GORM's AutoMigrate cannot append
+// to an existing table: it emits a plain `ADD COLUMN ... AUTO_INCREMENT`, which
+// MySQL rejects with "Incorrect table definition; there can be only one auto
+// column and it must be defined as a key". The column is therefore added with
+// explicit DDL (the table has no primary key so far), letting the database
+// backfill ids for existing rows and keep the sequence/counter in sync. The
+// remaining columns (`created_at`, `updated_at`) and the indexes are then
+// created by AutoMigrate as usual.
 func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if !db.HasColumn(teambitionScopeConfigTable20260727, "id") {
+		ddl := fmt.Sprintf(
+			"ALTER TABLE %s ADD COLUMN id BIGSERIAL PRIMARY KEY",
+			teambitionScopeConfigTable20260727,
+		)
+		if db.Dialect() == "mysql" {
+			ddl = fmt.Sprintf(
+				"ALTER TABLE %s ADD COLUMN id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY",
+				teambitionScopeConfigTable20260727,
+			)
+		}
+		if err := db.Exec(ddl); err != nil {
+			return err
+		}
+	}
 	return migrationhelper.AutoMigrateTables(basicRes, &teambitionScopeConfig20260727{})
 }