From f6221c183070f6bb85afd3eb7f04f631117c2c6d Mon Sep 17 00:00:00 2001 From: ULookup Date: Thu, 28 May 2026 00:30:36 -0700 Subject: [PATCH 01/31] feat(frontend): support schema evolution in data branch diff/merge Replace strict schema equivalence check with named-column compatibility check that allows compatible schema evolution (extra columns). Project base-side batches to target layout before hashmap encoding. Limit row comparison to common columns only. Fixes #24549 --- pkg/frontend/data_branch.go | 89 ++++++--- pkg/frontend/data_branch_hashdiff.go | 48 ++++- pkg/frontend/data_branch_hashdiff_test.go | 156 +++++++++++++++ pkg/frontend/data_branch_test.go | 185 ++++++++++++++++++ pkg/frontend/data_branch_types.go | 4 + .../branch/diff/diff_schema_evolve.result | 19 ++ .../branch/diff/diff_schema_evolve.sql | 23 +++ .../branch/merge/merge_schema_evolve.result | 15 ++ .../branch/merge/merge_schema_evolve.sql | 21 ++ 9 files changed, 533 insertions(+), 27 deletions(-) create mode 100644 test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result create mode 100644 test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql create mode 100644 test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result create mode 100644 test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 5ff61988255e2..9fad86d707837 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -930,10 +930,23 @@ func getTableStuff( tarTblDef = tblStuff.tarRel.GetTableDef(ctx) baseTblDef = tblStuff.baseRel.GetTableDef(ctx) - if !isSchemaEquivalent(tarTblDef, baseTblDef) { - err = moerr.NewInternalErrorNoCtx("the target table schema is not equivalent to the base table.") + var commonIdxes, tarOnlyIdxes []int + if commonIdxes, tarOnlyIdxes, err = checkSchemaCompatibility(tarTblDef, baseTblDef); err != nil { return } + tblStuff.def.commonIdxes = commonIdxes + tblStuff.def.tarOnlyIdxes = tarOnlyIdxes + + // Build baseColToTarIdx: for each physical base column, find the matching target column index. + tblStuff.def.baseColToTarIdx = make([]int, len(baseTblDef.Cols)) + for i := range tblStuff.def.baseColToTarIdx { + tblStuff.def.baseColToTarIdx[i] = -1 + } + for i, baseCol := range baseTblDef.Cols { + if tarColIdx, ok := tarTblDef.Name2ColIndex[strings.ToLower(baseCol.Name)]; ok { + tblStuff.def.baseColToTarIdx[i] = int(tarColIdx) + } + } if baseTblDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { tblStuff.def.pkKind = fakeKind @@ -1071,38 +1084,68 @@ func diffOnBase( return } -func isSchemaEquivalent(leftDef, rightDef *plan.TableDef) bool { - if len(leftDef.Cols) != len(rightDef.Cols) { - return false +func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, tarOnlyIdxes []int, err error) { + // Build a map from lowercase base column name to *ColDef. + baseColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) + for _, col := range baseDef.Cols { + baseColMap[strings.ToLower(col.Name)] = col } - for i := range leftDef.Cols { - if leftDef.Cols[i].ColId != rightDef.Cols[i].ColId { - return false - } - - if leftDef.Cols[i].Typ.Id != rightDef.Cols[i].Typ.Id { - return false + // Iterate over target columns to find common and target-only columns. + for i, tarCol := range tarDef.Cols { + if tarCol.Name == catalog.Row_ID || + tarCol.Name == catalog.FakePrimaryKeyColName || + tarCol.Name == catalog.CPrimaryKeyColName { + continue } - if leftDef.Cols[i].ClusterBy != rightDef.Cols[i].ClusterBy { - return false + baseCol, found := baseColMap[strings.ToLower(tarCol.Name)] + if found { + if baseCol.Typ.Id == tarCol.Typ.Id { + commonIdxes = append(commonIdxes, i) + } else { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: column '%s' exists in both schemas but has different types (target: %d, base: %d)", + tarCol.Name, tarCol.Typ.Id, baseCol.Typ.Id, + ) + return + } + } else { + tarOnlyIdxes = append(tarOnlyIdxes, i) } + } - if leftDef.Cols[i].Primary != rightDef.Cols[i].Primary { - return false - } + // Verify PK columns exist in common columns. + commonColNameSet := make(map[string]bool, len(commonIdxes)) + for _, idx := range commonIdxes { + commonColNameSet[strings.ToLower(tarDef.Cols[idx].Name)] = true + } - if leftDef.Cols[i].Seqnum != rightDef.Cols[i].Seqnum { - return false + if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { + // For fake PK tables, the PK column is generated from all columns. + // No specific PK name to verify. + } else if baseDef.Pkey.CompPkeyCol != nil { + for _, pkName := range baseDef.Pkey.Names { + if !commonColNameSet[strings.ToLower(pkName)] { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: composite primary key column '%s' is not present in common columns", + pkName, + ) + return + } } - - if leftDef.Cols[i].NotNull != rightDef.Cols[i].NotNull { - return false + } else { + pkName := baseDef.Pkey.PkeyColName + if !commonColNameSet[strings.ToLower(pkName)] { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: primary key column '%s' is not present in common columns", + pkName, + ) + return } } - return true + return } func getRelationById( diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 67285f03aaa98..066835c3871aa 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -1150,7 +1150,11 @@ func compareRowInWrappedBatches( return 0, nil } - for i, colIdx := range tblStuff.def.visibleIdxes { + compareIdxes := tblStuff.def.commonIdxes + if len(compareIdxes) == 0 { + compareIdxes = tblStuff.def.visibleIdxes // backward compat when commonIdxes not set + } + for _, colIdx := range compareIdxes { if skipPKCols { if slices.Index(tblStuff.def.pkColIdxes, colIdx) != -1 { continue @@ -1158,8 +1162,8 @@ func compareRowInWrappedBatches( } var ( - vec1 = wrapped1.batch.Vecs[i] - vec2 = wrapped2.batch.Vecs[i] + vec1 = wrapped1.batch.Vecs[colIdx] + vec2 = wrapped2.batch.Vecs[colIdx] ) if cmp, err := compareSingleValInVector( @@ -1738,7 +1742,11 @@ func diffDataHelper( } notSame := false - for _, idx := range tblStuff.def.visibleIdxes { + compareIdxes := tblStuff.def.commonIdxes + if len(compareIdxes) == 0 { + compareIdxes = tblStuff.def.visibleIdxes // backward compat when commonIdxes not set + } + for _, idx := range compareIdxes { if slices.Index(tblStuff.def.pkColIdxes, idx) != -1 { // pk columns already compared continue @@ -2073,6 +2081,11 @@ func buildHashmapForTable( return } + if dataBat != nil && dataBat.RowCount() > 0 && side == "base" && len(tblStuff.def.tarOnlyIdxes) > 0 { + projected := projectBaseBatchToTarget(dataBat, tblStuff, mp) + dataBat = projected // projected will be cleaned in putVectors + } + if dataBat != nil && dataBat.RowCount() > 0 { totalRows += int64(dataBat.RowCount()) totalBytes += int64(dataBat.Size()) @@ -2287,3 +2300,30 @@ func buildHashmapForTable( return } + +// projectBaseBatchToTarget projects a base-side data batch to match the +// target column layout. It creates a new batch with the same number of +// columns as target data batches (1 RowID + len(colNames)), copies the +// RowID vector as-is, places each base physical column at its target +// position using baseColToTarIdx, and fills target-only columns with +// constant-NULL vectors. +func projectBaseBatchToTarget( + baseBat *batch.Batch, + tblStuff *tableStuff, + mp *mpool.MPool, +) *batch.Batch { + out := batch.NewWithSize(len(tblStuff.def.colNames) + 1) + out.Vecs[0] = baseBat.Vecs[0] // RowID + for i := range tblStuff.def.colNames { + out.Vecs[i+1] = vector.NewConstNull(tblStuff.def.colTypes[i], baseBat.RowCount(), mp) + } + baseColCount := baseBat.VectorCount() - 1 // subtract RowID + for baseColIdx := 0; baseColIdx < baseColCount; baseColIdx++ { + tarColIdx := tblStuff.def.baseColToTarIdx[baseColIdx] + if tarColIdx >= 0 { + out.Vecs[tarColIdx+1] = baseBat.Vecs[baseColIdx+1] + } + } + out.SetRowCount(baseBat.RowCount()) + return out +} diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index 01565e5a44499..5e5f89ccbf310 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -1107,3 +1107,159 @@ func TestValidateLeadingRowID(t *testing.T) { require.NoError(t, validateLeadingRowID("base", "t", false, bat)) }) } + +func TestProjectBaseBatchToTarget(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + + // Simulate target having 3 columns [id, name, extra], base has 2 [id, name] + tblStuff.def.colNames = []string{"a", "b", "c"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_varchar.ToType(), + types.T_int64.ToType(), + } + // base has [a, b] only. baseColToTarIdx says: + // base col 0 (a) -> target idx 0 + // base col 1 (b) -> target idx 1 + // c is target-only + tblStuff.def.baseColToTarIdx = []int{0, 1} + tblStuff.def.tarOnlyIdxes = []int{2} + + mp := ses.proc.Mp() + + // Build a base-side data batch with 1 RowID + 2 columns [a, b] + baseBat := batch.NewWithSize(3) + baseBat.SetAttributes([]string{catalog.Row_ID, "a", "b"}) + baseBat.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + baseBat.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + baseBat.Vecs[2] = vector.NewVec(types.T_varchar.ToType()) + + uid, err := types.BuildUuid() + require.NoError(t, err) + blkID := objectio.NewBlockid(&uid, 0, 1) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[0], types.NewRowid(blkID, 0), false, mp)) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[1], int64(42), false, mp)) + require.NoError(t, vector.AppendBytes(baseBat.Vecs[2], []byte("hello"), false, mp)) + baseBat.SetRowCount(1) + + projected := projectBaseBatchToTarget(baseBat, &tblStuff, mp) + + // Should have 4 vectors: RowID + [a, b, c] + require.Equal(t, 4, projected.VectorCount()) + require.Equal(t, 1, projected.RowCount()) + + // RowID should be preserved + require.Equal(t, types.T_Rowid, projected.Vecs[0].GetType().Oid) + + // Col a (index 1): should be int64(42) + require.Equal(t, types.T_int64, projected.Vecs[1].GetType().Oid) + require.Equal(t, int64(42), vector.MustFixedColWithTypeCheck[int64](projected.Vecs[1])[0]) + + // Col b (index 2): should be "hello" + require.Equal(t, types.T_varchar, projected.Vecs[2].GetType().Oid) + require.Equal(t, "hello", string(projected.Vecs[2].GetBytesAt(0))) + + // Col c (index 3): should be constant NULL (target-only) + require.Equal(t, types.T_int64, projected.Vecs[3].GetType().Oid) + require.True(t, projected.Vecs[3].IsConst()) + require.True(t, projected.Vecs[3].IsConstNull(), "target-only column should be const NULL") + + projected.Clean(mp) + baseBat.Clean(mp) +} + +func TestCompareRowInWrappedBatches_WithCommonIdxes(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + + // Set up: target has [id(0), name(1), extra(2)] + // commonIdxes = [0, 1] (only id and name are common, extra is target-only) + tblStuff.def.colNames = []string{"id", "name", "extra"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_varchar.ToType(), + types.T_int64.ToType(), + } + tblStuff.def.pkKind = normalKind + tblStuff.def.visibleIdxes = []int{0, 1, 2} + tblStuff.def.commonIdxes = []int{0, 1} // only compare id and name + tblStuff.def.pkColIdx = 0 + tblStuff.def.pkColIdxes = []int{0} + + mp := ses.proc.Mp() + + // Batch1: [id=1, name="same", extra=999] + bat1 := batch.NewWithSize(3) + bat1.SetAttributes([]string{"id", "name", "extra"}) + bat1.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + bat1.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) + bat1.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixed(bat1.Vecs[0], int64(1), false, mp)) + require.NoError(t, vector.AppendBytes(bat1.Vecs[1], []byte("same"), false, mp)) + require.NoError(t, vector.AppendFixed(bat1.Vecs[2], int64(999), false, mp)) + bat1.SetRowCount(1) + + // Batch2: [id=1, name="same", extra=0] (extra differs but is target-only) + bat2 := batch.NewWithSize(3) + bat2.SetAttributes([]string{"id", "name", "extra"}) + bat2.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + bat2.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) + bat2.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixed(bat2.Vecs[0], int64(1), false, mp)) + require.NoError(t, vector.AppendBytes(bat2.Vecs[1], []byte("same"), false, mp)) + require.NoError(t, vector.AppendFixed(bat2.Vecs[2], int64(0), false, mp)) + bat2.SetRowCount(1) + + wrapped1 := batchWithKind{kind: diffInsert, batch: bat1} + wrapped2 := batchWithKind{kind: diffInsert, batch: bat2} + + // Rows should be considered equal because extra (index 2) is NOT in commonIdxes + cmp, err := compareRowInWrappedBatches( + context.Background(), + ses, + tblStuff, + 0, 0, + true, // skipPKCols + wrapped1, + wrapped2, + ) + require.NoError(t, err) + require.Zero(t, cmp, "rows should be considered equal when only non-common columns differ") + + // Now test with a difference in a common column (name) + bat3 := batch.NewWithSize(3) + bat3.SetAttributes([]string{"id", "name", "extra"}) + bat3.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + bat3.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) + bat3.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixed(bat3.Vecs[0], int64(1), false, mp)) + require.NoError(t, vector.AppendBytes(bat3.Vecs[1], []byte("different"), false, mp)) + require.NoError(t, vector.AppendFixed(bat3.Vecs[2], int64(999), false, mp)) + bat3.SetRowCount(1) + + wrapped3 := batchWithKind{kind: diffInsert, batch: bat3} + + // Rows should differ because name differs and name IS in commonIdxes + cmp, err = compareRowInWrappedBatches( + context.Background(), + ses, + tblStuff, + 0, 0, + false, // do NOT skip PK + wrapped1, + wrapped3, + ) + require.NoError(t, err) + require.NotZero(t, cmp, "rows should differ when a common column differs") + + bat1.Clean(mp) + bat2.Clean(mp) + bat3.Clean(mp) +} diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index 992281dc7da27..0e3af52005d03 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -23,6 +23,7 @@ import ( "path/filepath" "testing" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/config" @@ -30,6 +31,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/stage" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/stretchr/testify/require" @@ -715,3 +717,186 @@ func TestValidateOutputDirPath(t *testing.T) { require.Error(t, err) }) } + +func TestCheckSchemaCompatibility_Identical(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1}, commonIdxes) + require.Empty(t, tarOnlyIdxes) +} + +func TestCheckSchemaCompatibility_ExtraColumnOnTarget(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{2}, tarOnlyIdxes) +} + +func TestCheckSchemaCompatibility_PKChanged(t *testing.T) { + // base has PK on column "a" which is also in target, so it passes. + // We need a case where the BASE's PK column does NOT exist in target. + // Target has different columns entirely, so base PK "a" is missing from common. + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"x"}, + PkeyColName: "x", + }, + Cols: []*plan.ColDef{ + {Name: "x", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "y", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.Error(t, err) + require.Contains(t, err.Error(), "primary key column") +} + +func TestCheckSchemaCompatibility_TypeMismatch(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_varchar)}}, // varchar in target + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, // int in base + }, + } + + _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.Error(t, err) + require.Contains(t, err.Error(), "has different types") +} + +func TestCheckSchemaCompatibility_CompositePK(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a", "b"}, + PkeyColName: "__cpkey__", + CompPkeyCol: &plan.ColDef{Name: "__cpkey__"}, + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a", "b"}, + PkeyColName: "__cpkey__", + CompPkeyCol: &plan.ColDef{Name: "__cpkey__"}, + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{2}, tarOnlyIdxes) +} + +func TestCheckSchemaCompatibility_FakePK(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + PkeyColName: catalog.FakePrimaryKeyColName, + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + PkeyColName: catalog.FakePrimaryKeyColName, + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{2}, tarOnlyIdxes) +} diff --git a/pkg/frontend/data_branch_types.go b/pkg/frontend/data_branch_types.go index ed725bfb4d3f2..795180bd4ff81 100644 --- a/pkg/frontend/data_branch_types.go +++ b/pkg/frontend/data_branch_types.go @@ -195,6 +195,10 @@ type tableStuff struct { pkSeqnum int // physical column seqnum for PK (for ZoneMap lookup) pkColIdxes []int // expanded pk columns pkKind int + + commonIdxes []int // indices of common columns (target ordering) + tarOnlyIdxes []int // indices of target-only columns (target ordering) + baseColToTarIdx []int // for base batch Vec[i+1], the target column index, or -1 } worker *ants.Pool diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result new file mode 100644 index 0000000000000..cd3a36b01e1ec --- /dev/null +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -0,0 +1,19 @@ +drop database if exists test; +create database test; +use test; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set c=10 where a=1; +insert into t1 values(4,4,40); +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a b +t1 INSERT 4 4 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +drop database test; diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql new file mode 100644 index 0000000000000..87b3d5857a33a --- /dev/null +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -0,0 +1,23 @@ +drop database if exists test; +create database test; +use test; + +-- Schema evolution: target table has an extra column compared to base +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set c=10 where a=1; +insert into t1 values(4,4,40); +create snapshot sp1 for table test t1; + +-- DIFF should succeed: t1 has [a,b,c], t0 has [a,b] +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result new file mode 100644 index 0000000000000..ecb73c234770d --- /dev/null +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -0,0 +1,15 @@ +drop database if exists test; +create database test; +use test; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set c=10 where a=1; +insert into t1 values(4,4,40); +create snapshot sp1 for table test t1; +data branch merge t1 into t0; +drop table t0; +drop table t1; +drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql new file mode 100644 index 0000000000000..9cf529afce104 --- /dev/null +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -0,0 +1,21 @@ +drop database if exists test; +create database test; +use test; + +-- Schema evolution merge: target has extra column, base has original schema +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set c=10 where a=1; +insert into t1 values(4,4,40); +create snapshot sp1 for table test t1; + +-- MERGE should succeed: t1 has [a,b,c], t0 has [a,b] +data branch merge t1 into t0; + +drop table t0; +drop table t1; +drop database test; From d82470c5fc9b545e8471583b20b8148b57719cf1 Mon Sep 17 00:00:00 2001 From: ULookup Date: Thu, 28 May 2026 02:16:24 -0700 Subject: [PATCH 02/31] fix(frontend): add bounds check in projectBaseBatchToTarget Fixes index out of range panic when base data batch has more columns than base table definition (e.g., composite PK columns). Clamp baseColCount to len(baseColToTarIdx) and validate tarColIdx against len(colNames) before accessing out.Vecs. --- pkg/frontend/data_branch_hashdiff.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 066835c3871aa..25742cc506a09 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -2318,9 +2318,12 @@ func projectBaseBatchToTarget( out.Vecs[i+1] = vector.NewConstNull(tblStuff.def.colTypes[i], baseBat.RowCount(), mp) } baseColCount := baseBat.VectorCount() - 1 // subtract RowID + if baseColCount > len(tblStuff.def.baseColToTarIdx) { + baseColCount = len(tblStuff.def.baseColToTarIdx) + } for baseColIdx := 0; baseColIdx < baseColCount; baseColIdx++ { tarColIdx := tblStuff.def.baseColToTarIdx[baseColIdx] - if tarColIdx >= 0 { + if tarColIdx >= 0 && tarColIdx < len(tblStuff.def.colNames) { out.Vecs[tarColIdx+1] = baseBat.Vecs[baseColIdx+1] } } From f0aa5375e0181c2de67a4179fc9e67b2e71db12b Mon Sep 17 00:00:00 2001 From: ULookup Date: Thu, 28 May 2026 03:41:32 -0700 Subject: [PATCH 03/31] fix(frontend): skip target-only columns in MERGE INSERT SQL generation When target table has extra columns not present in base (schema evolution), INSERT SQL for MERGE must only reference common columns. Filter visibleNames in newPKBatchInfo and use commonIdxes in appendBatchRowsAsSQLValues/writeInsertRowValues when tarOnlyIdxes is non-empty. Also update diff_schema_evolve.result to expect 5 columns including the extra column c in DIFF output. --- pkg/frontend/data_branch_output.go | 21 ++++++++++++++----- pkg/frontend/data_branch_output_test.go | 2 +- .../branch/diff/diff_schema_evolve.result | 4 ++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/frontend/data_branch_output.go b/pkg/frontend/data_branch_output.go index e6a77f6582ec8..5ca01991bd7f5 100644 --- a/pkg/frontend/data_branch_output.go +++ b/pkg/frontend/data_branch_output.go @@ -995,6 +995,12 @@ func appendBatchRowsAsSQLValues( appender sqlValuesAppender, ) (err error) { + // For INSERT, skip target-only columns that don't exist in the base table. + insertIdxes := tblStuff.def.visibleIdxes + if len(tblStuff.def.tarOnlyIdxes) > 0 { + insertIdxes = tblStuff.def.commonIdxes + } + //seenCols := make(map[int]struct{}, len(tblStuff.def.visibleIdxes)) row := make([]any, len(tblStuff.def.colNames)) @@ -1070,7 +1076,7 @@ func appendBatchRowsAsSQLValues( } } } else { - if err = writeInsertRowValues(ses, tblStuff, row, tmpValsBuffer); err != nil { + if err = writeInsertRowValues(ses, tblStuff, row, tmpValsBuffer, insertIdxes); err != nil { return } } @@ -1393,8 +1399,12 @@ func newPKBatchInfo(ctx context.Context, ses *Session, tblStuff tableStuff) *pkB pkNames[i] = tblStuff.def.colNames[idx] } - visibleNames := make([]string, len(tblStuff.def.visibleIdxes)) - for i, idx := range tblStuff.def.visibleIdxes { + idxes := tblStuff.def.visibleIdxes + if len(tblStuff.def.tarOnlyIdxes) > 0 { + idxes = tblStuff.def.commonIdxes + } + visibleNames := make([]string, len(idxes)) + for i, idx := range idxes { visibleNames[i] = tblStuff.def.colNames[idx] } @@ -1436,13 +1446,14 @@ func writeInsertRowValues( tblStuff tableStuff, row []any, buf *bytes.Buffer, + idxes []int, ) error { buf.WriteString("(") - for i, idx := range tblStuff.def.visibleIdxes { + for i, idx := range idxes { if err := formatValIntoString(ses, row[idx], tblStuff.def.colTypes[idx], buf); err != nil { return err } - if i != len(tblStuff.def.visibleIdxes)-1 { + if i != len(idxes)-1 { buf.WriteString(",") } } diff --git a/pkg/frontend/data_branch_output_test.go b/pkg/frontend/data_branch_output_test.go index f30de977fd015..204f423a440a9 100644 --- a/pkg/frontend/data_branch_output_test.go +++ b/pkg/frontend/data_branch_output_test.go @@ -412,7 +412,7 @@ func TestDataBranchOutputWriteRowValues(t *testing.T) { row := []any{int64(7), "alice"} insertBuf := &bytes.Buffer{} - require.NoError(t, writeInsertRowValues(nil, tblStuff, row, insertBuf)) + require.NoError(t, writeInsertRowValues(nil, tblStuff, row, insertBuf, tblStuff.def.visibleIdxes)) require.Equal(t, "(7,'alice')", insertBuf.String()) deleteBuf := &bytes.Buffer{} diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index cd3a36b01e1ec..b141ecd87f04a 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -10,8 +10,8 @@ update t1 set c=10 where a=1; insert into t1 values(4,4,40); create snapshot sp1 for table test t1; data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; -diff t1 against t0 flag a b -t1 INSERT 4 4 +diff t1 against t0 flag a b c +t1 INSERT 4 4 40 drop snapshot sp1; drop snapshot sp0; drop table t0; From 03492a1ba29cd1f6d2a07e42fb35adf9041d16b0 Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 3 Jun 2026 02:46:39 -0700 Subject: [PATCH 04/31] fix(frontend): correct Cols-index vs batch-index mapping in schema evolution - compareRowInWrappedBatches: use batchIdx for Vecs access, colIdx for pk check - baseColToTarIdx: build after colNames populated, filter hidden cols, map to colNames positions - data_branch_pick: add missing idxes parameter for writeInsertRowValues --- pkg/frontend/data_branch.go | 40 ++++++++++++++++++++-------- pkg/frontend/data_branch_hashdiff.go | 6 ++--- pkg/frontend/data_branch_pick.go | 6 ++++- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 9fad86d707837..cf9f1ad6300bf 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -937,17 +937,6 @@ func getTableStuff( tblStuff.def.commonIdxes = commonIdxes tblStuff.def.tarOnlyIdxes = tarOnlyIdxes - // Build baseColToTarIdx: for each physical base column, find the matching target column index. - tblStuff.def.baseColToTarIdx = make([]int, len(baseTblDef.Cols)) - for i := range tblStuff.def.baseColToTarIdx { - tblStuff.def.baseColToTarIdx[i] = -1 - } - for i, baseCol := range baseTblDef.Cols { - if tarColIdx, ok := tarTblDef.Name2ColIndex[strings.ToLower(baseCol.Name)]; ok { - tblStuff.def.baseColToTarIdx[i] = int(tarColIdx) - } - } - if baseTblDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { tblStuff.def.pkKind = fakeKind for i, col := range baseTblDef.Cols { @@ -992,6 +981,35 @@ func getTableStuff( tblStuff.def.visibleIdxes = append(tblStuff.def.visibleIdxes, i) } + // Build baseColToTarIdx: map each base data column (in batch column order, + // excluding Row_ID / FakePK / CPK) to its position in target colNames. + // colNames is already populated above from tarTblDef.Cols (excluding Row_ID, + // including FakePK/CPK). This mapping is used by projectBaseBatchToTarget to + // place base batch vectors at the correct target positions. + { + baseDataNames := make([]string, 0, len(baseTblDef.Cols)) + for _, col := range baseTblDef.Cols { + if col.Name == catalog.Row_ID || + col.Name == catalog.FakePrimaryKeyColName || + col.Name == catalog.CPrimaryKeyColName { + continue + } + baseDataNames = append(baseDataNames, col.Name) + } + tblStuff.def.baseColToTarIdx = make([]int, len(baseDataNames)) + for i := range tblStuff.def.baseColToTarIdx { + tblStuff.def.baseColToTarIdx[i] = -1 + } + for i, name := range baseDataNames { + for j, tarName := range tblStuff.def.colNames { + if strings.EqualFold(name, tarName) { + tblStuff.def.baseColToTarIdx[i] = j + break + } + } + } + } + tblStuff.retPool = &retBatchList{} tblStuff.bufPool = &sync.Pool{ New: func() any { diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 25742cc506a09..b7599d1f31243 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -1154,7 +1154,7 @@ func compareRowInWrappedBatches( if len(compareIdxes) == 0 { compareIdxes = tblStuff.def.visibleIdxes // backward compat when commonIdxes not set } - for _, colIdx := range compareIdxes { + for batchIdx, colIdx := range compareIdxes { if skipPKCols { if slices.Index(tblStuff.def.pkColIdxes, colIdx) != -1 { continue @@ -1162,8 +1162,8 @@ func compareRowInWrappedBatches( } var ( - vec1 = wrapped1.batch.Vecs[colIdx] - vec2 = wrapped2.batch.Vecs[colIdx] + vec1 = wrapped1.batch.Vecs[batchIdx] + vec2 = wrapped2.batch.Vecs[batchIdx] ) if cmp, err := compareSingleValInVector( diff --git a/pkg/frontend/data_branch_pick.go b/pkg/frontend/data_branch_pick.go index b366f2af8f12e..50186f51f8677 100644 --- a/pkg/frontend/data_branch_pick.go +++ b/pkg/frontend/data_branch_pick.go @@ -408,7 +408,11 @@ func appendPickedBatchRows( } } } else { - if err = writeInsertRowValues(ses, tblStuff, row, tmpValsBuffer); err != nil { + pickInsertIdxes := tblStuff.def.visibleIdxes + if len(tblStuff.def.tarOnlyIdxes) > 0 { + pickInsertIdxes = tblStuff.def.commonIdxes + } + if err = writeInsertRowValues(ses, tblStuff, row, tmpValsBuffer, pickInsertIdxes); err != nil { return } } From cda05f8378591500fa69deb0f1fefec4af04e2c3 Mon Sep 17 00:00:00 2001 From: ULookup Date: Tue, 9 Jun 2026 21:03:31 -0700 Subject: [PATCH 05/31] fix(test): add missing snapshot cleanup in merge_schema_evolve BVT The merge_schema_evolve.sql test creates snapshots sp0 and sp1 but never drops them. Since snapshot names are globally unique (checked by sname without level filtering), these leftover snapshots conflict with clone_reference.sql which creates account-level sp0, causing 6 BVT failures and dropping the pass rate below 100%. Add explicit drop snapshot sp1/sp0 cleanup matching the pattern in diff_schema_evolve.sql. --- .../cases/git4data/branch/merge/merge_schema_evolve.result | 2 ++ .../cases/git4data/branch/merge/merge_schema_evolve.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index ecb73c234770d..8dae787cffdda 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -10,6 +10,8 @@ update t1 set c=10 where a=1; insert into t1 values(4,4,40); create snapshot sp1 for table test t1; data branch merge t1 into t0; +drop snapshot sp1; +drop snapshot sp0; drop table t0; drop table t1; drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index 9cf529afce104..a827292e8541c 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -16,6 +16,8 @@ create snapshot sp1 for table test t1; -- MERGE should succeed: t1 has [a,b,c], t0 has [a,b] data branch merge t1 into t0; +drop snapshot sp1; +drop snapshot sp0; drop table t0; drop table t1; drop database test; From 3bd0acd3097e77f57ca8c4374375178aeb633dff Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 8 Jul 2026 14:51:06 +0800 Subject: [PATCH 06/31] test: expand BVT coverage for schema evolution diff/merge Expand diff_schema_evolve (8 cases) and merge_schema_evolve (5 cases) to cover: basic evolution, multiple extra columns, UPDATE on common col, UPDATE on target-only col (no diff), DELETE, composite PK, idempotent merge, type mismatch rejection, and PK removal rejection. Update branch_unhappy_path and branch_edge_cases to reflect the new schema evolution behavior: ALTER ADD COLUMN no longer errors, and type mismatch on common columns is the new rejection path. --- .../branch/diff/diff_schema_evolve.result | 96 +++++++++++ .../branch/diff/diff_schema_evolve.sql | 159 +++++++++++++++++- .../branch/edge/branch_edge_cases.result | 18 +- .../branch/edge/branch_edge_cases.sql | 16 +- .../branch/edge/branch_unhappy_path.result | 12 +- .../branch/edge/branch_unhappy_path.sql | 11 +- .../branch/merge/merge_schema_evolve.result | 82 +++++++++ .../branch/merge/merge_schema_evolve.sql | 108 +++++++++++- 8 files changed, 477 insertions(+), 25 deletions(-) diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index b141ecd87f04a..b20eb6e8420cd 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -16,4 +16,100 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +alter table t1 add column d varchar(20) default 'x'; +insert into t1 values(3,3,30,'new'); +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a b c d +t1 INSERT 3 3 30 new +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set b=99 where a=1; +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a b c +t1 UPDATE 1 99 0 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set c=99 where a=1; +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a b c +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +delete from t1 where a=2; +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a b c +t1 DELETE 2 2 null +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, c int, primary key(a,b)); +insert into t0 values(1,1,10),(2,2,20); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column d int default 0; +insert into t1 values(3,3,30,300); +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a b c d +t1 INSERT 3 3 30 300 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; +create table t1(a int, b varchar(20), primary key(a)); +insert into t1 values(1,'1'),(2,'2'); +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +-- @regex("schema compatibility check: column 'b' exists in both schemas but has different types", true) +internal error: schema compatibility check: column 'b' exists in both schemas but has different types (target: 61, base: 22) +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 drop column a; +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +-- @regex("schema compatibility check: primary key column 'a' is not present in common columns", true) +internal error: schema compatibility check: primary key column 'a' is not present in common columns +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; drop database test; diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 87b3d5857a33a..b5ee035abb093 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -2,7 +2,11 @@ drop database if exists test; create database test; use test; --- Schema evolution: target table has an extra column compared to base +-- ===================================================== +-- Case 1: basic schema evolution - target has one extra column +-- base: [a, b] +-- target: [a, b, c] +-- ===================================================== create table t0(a int, b int, primary key(a)); insert into t0 values(1,1),(2,2),(3,3); create snapshot sp0 for table test t0; @@ -13,11 +17,162 @@ update t1 set c=10 where a=1; insert into t1 values(4,4,40); create snapshot sp1 for table test t1; --- DIFF should succeed: t1 has [a,b,c], t0 has [a,b] +-- DIFF should succeed: only common columns (a, b) are compared. +-- Row a=4 is new in t1; rows a=1,2,3 have matching common columns. data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; + +-- ===================================================== +-- Case 2: multiple extra columns on target +-- base: [a, b] +-- target: [a, b, c, d] +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +alter table t1 add column d varchar(20) default 'x'; +insert into t1 values(3,3,30,'new'); +create snapshot sp1 for table test t1; + +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 3: extra column + UPDATE on a common column +-- t1 updates b (common col) -> diff should show UPDATE +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set b=99 where a=1; +create snapshot sp1 for table test t1; + +-- a=1: common col b differs (1 vs 99) -> UPDATE +-- a=2: no change +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 4: extra column + UPDATE only on target-only column +-- t1 updates c (target-only) -> diff should show NO change +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set c=99 where a=1; +create snapshot sp1 for table test t1; + +-- Common columns (a, b) match for all rows -> empty diff +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 5: extra column + DELETE on target +-- t1 deletes a row -> diff should show DELETE +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +delete from t1 where a=2; +create snapshot sp1 for table test t1; + +-- a=2: t1 deleted it, t0 still has it -> DELETE (c is null, projected from base) +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 6: composite primary key + extra column +-- base: [a, b, c] PK(a, b) +-- target: [a, b, c, d] +-- ===================================================== +create table t0(a int, b int, c int, primary key(a,b)); +insert into t0 values(1,1,10),(2,2,20); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column d int default 0; +insert into t1 values(3,3,30,300); +create snapshot sp1 for table test t1; + +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 7: type mismatch on a common column -> error +-- base: [a, b int] +-- target: [a, b varchar] +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; + +create table t1(a int, b varchar(20), primary key(a)); +insert into t1 values(1,'1'),(2,'2'); +create snapshot sp1 for table test t1; + +-- @regex("schema compatibility check: column 'b' exists in both schemas but has different types", true) +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 8: PK column removed from target -> error +-- base: [a, b] PK(a) +-- target: [b] (a was dropped, no PK) +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 drop column a; +create snapshot sp1 for table test t1; + +-- @regex("schema compatibility check: primary key column 'a' is not present in common columns", true) +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + drop database test; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result index bf283b32d8631..27f759172eecb 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result @@ -56,10 +56,10 @@ insert into base values (1, 10), (2, 20), (3, 30); data branch create table left_add from base; data branch create table right_add from base; alter table left_add add column c varchar(20) default 'left-only'; -update left_add set b = 11, c = 'changed' where a = 1; insert into right_add values (4, 40); -data branch diff left_add against right_add columns (a, b) output summary; -internal error: the target table schema is not equivalent to the base table. +data branch diff left_add against right_add; +diff left_add against right_add flag a b c +right_add INSERT 4 40 null data branch create table both_add_left from base; data branch create table both_add_right from base; alter table both_add_left add column c varchar(20) default 'same'; @@ -72,11 +72,13 @@ both_add_left INSERT 2 left both_add_right INSERT 2 same both_add_left INSERT 3 same both_add_right INSERT 3 right -data branch create table drop_left from base; -data branch create table drop_right from base; -alter table drop_left drop column b; -data branch diff drop_left against drop_right; -internal error: the target table schema is not equivalent to the base table. +data branch create table type_left from base; +data branch create table type_right from base; +alter table type_left add column c int default 0; +alter table type_right add column c varchar(20) default 'x'; +data branch diff type_left against type_right; +-- @regex("schema compatibility check: column 'c' exists in both schemas but has different types", true) +internal error: schema compatibility check: column 'c' exists in both schemas but has different types (target: 22, base: 61) data branch create table rename_left from base; data branch create table rename_right from base; alter table rename_left rename column b to bb; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql index 95b1443b1952f..96a8791367ae0 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql @@ -60,9 +60,9 @@ insert into base values (1, 10), (2, 20), (3, 30); data branch create table left_add from base; data branch create table right_add from base; alter table left_add add column c varchar(20) default 'left-only'; -update left_add set b = 11, c = 'changed' where a = 1; insert into right_add values (4, 40); -data branch diff left_add against right_add columns (a, b) output summary; +-- Target-only column c must not produce false diffs; only a=4 (base-only) appears. +data branch diff left_add against right_add; data branch create table both_add_left from base; data branch create table both_add_right from base; @@ -72,11 +72,13 @@ update both_add_left set c = 'left' where a = 2; update both_add_right set c = 'right' where a = 3; data branch diff both_add_left against both_add_right columns (a, c); -data branch create table drop_left from base; -data branch create table drop_right from base; -alter table drop_left drop column b; --- Divergent schemas are rejected with a schema-equivalence error. -data branch diff drop_left against drop_right; +data branch create table type_left from base; +data branch create table type_right from base; +alter table type_left add column c int default 0; +alter table type_right add column c varchar(20) default 'x'; +-- Type mismatch on a common column is rejected. +-- @regex("schema compatibility check: column 'c' exists in both schemas but has different types",true) +data branch diff type_left against type_right; data branch create table rename_left from base; data branch create table rename_right from base; diff --git a/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.result b/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.result index 214f3f3f274a8..25f3cb6da3e93 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.result +++ b/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.result @@ -22,11 +22,15 @@ data branch diff br against missing_base; SQL parser error: table "missing_base" does not exist alter table br add column c int default 0; data branch diff br against base; --- @regex("schema is not equivalent", true) -internal error: the target table schema is not equivalent to the base table. +diff br against base flag a b c data branch merge br into base; --- @regex("schema is not equivalent", true) -internal error: the target table schema is not equivalent to the base table. +create table type_base(a int primary key, b int); +insert into type_base values (1, 10); +create table type_branch(a int primary key, b varchar(20)); +insert into type_branch values (1, '10'); +data branch diff type_branch against type_base; +-- @regex("schema compatibility check: column 'b' exists in both schemas but has different types", true) +internal error: schema compatibility check: column 'b' exists in both schemas but has different types (target: 61, base: 22) create table no_pk(a int, b int); insert into no_pk values (1, 10), (2, 20); data branch create table no_pk_branch from no_pk; diff --git a/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.sql b/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.sql index dfc20a36488f4..120ebf8c662e0 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_unhappy_path.sql @@ -25,11 +25,18 @@ data branch diff br against missing_base; -- @bvt:issue alter table br add column c int default 0; --- @regex("schema is not equivalent",true) +-- Schema evolution (added column) now supports diff/merge. data branch diff br against base; --- @regex("schema is not equivalent",true) data branch merge br into base; +-- Type mismatch on a common column is still rejected. +create table type_base(a int primary key, b int); +insert into type_base values (1, 10); +create table type_branch(a int primary key, b varchar(20)); +insert into type_branch values (1, '10'); +-- @regex("schema compatibility check: column 'b' exists in both schemas but has different types",true) +data branch diff type_branch against type_base; + create table no_pk(a int, b int); insert into no_pk values (1, 10), (2, 20); data branch create table no_pk_branch from no_pk; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index 8dae787cffdda..f089103151925 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -10,6 +10,88 @@ update t1 set c=10 where a=1; insert into t1 values(4,4,40); create snapshot sp1 for table test t1; data branch merge t1 into t0; +select * from t0 order by a; +a b +1 1 +2 2 +3 3 +4 4 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set b=99, c=10 where a=1; +insert into t1 values(4,4,40); +create snapshot sp1 for table test t1; +data branch merge t1 into t0; +select * from t0 order by a; +a b +1 99 +2 2 +3 3 +4 4 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +delete from t1 where a=2; +create snapshot sp1 for table test t1; +data branch merge t1 into t0; +select * from t0 order by a; +a b +1 1 +3 3 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, c int, primary key(a,b)); +insert into t0 values(1,1,10),(2,2,20); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column d int default 0; +update t1 set c=99 where a=1 and b=1; +insert into t1 values(3,3,30,300); +create snapshot sp1 for table test t1; +data branch merge t1 into t0; +select * from t0 order by a; +a b c +1 1 99 +2 2 20 +3 3 30 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +insert into t1 values(3,3,30); +create snapshot sp1 for table test t1; +data branch merge t1 into t0; +select * from t0 order by a; +a b +1 1 +2 2 +3 3 +data branch merge t1 into t0; +select * from t0 order by a; +a b +1 1 +2 2 +3 3 drop snapshot sp1; drop snapshot sp0; drop table t0; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index a827292e8541c..4e349fe620669 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -2,7 +2,11 @@ drop database if exists test; create database test; use test; --- Schema evolution merge: target has extra column, base has original schema +-- ===================================================== +-- Case 1: basic schema evolution merge - target has extra column +-- base: [a, b] +-- target: [a, b, c] +-- ===================================================== create table t0(a int, b int, primary key(a)); insert into t0 values(1,1),(2,2),(3,3); create snapshot sp0 for table test t0; @@ -13,11 +17,111 @@ update t1 set c=10 where a=1; insert into t1 values(4,4,40); create snapshot sp1 for table test t1; --- MERGE should succeed: t1 has [a,b,c], t0 has [a,b] +-- MERGE should succeed: only common columns (a, b) are written to base. data branch merge t1 into t0; +-- Verify: t0 gets a=4 inserted, c is not written (t0 has no column c). +select * from t0 order by a; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 2: merge with INSERT + UPDATE on common column +-- t1 updates b (common) and adds a new row -> both applied to t0 +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +update t1 set b=99, c=10 where a=1; +insert into t1 values(4,4,40); +create snapshot sp1 for table test t1; + +data branch merge t1 into t0; + +-- t0: a=1 b updated to 99, a=4 inserted +select * from t0 order by a; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 3: merge with DELETE +-- t1 deletes a row -> merge deletes it from t0 +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +delete from t1 where a=2; +create snapshot sp1 for table test t1; + +data branch merge t1 into t0; + +-- t0: a=2 deleted +select * from t0 order by a; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 4: merge with composite PK + extra column +-- base: [a, b, c] PK(a, b) +-- target: [a, b, c, d] +-- ===================================================== +create table t0(a int, b int, c int, primary key(a,b)); +insert into t0 values(1,1,10),(2,2,20); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column d int default 0; +update t1 set c=99 where a=1 and b=1; +insert into t1 values(3,3,30,300); +create snapshot sp1 for table test t1; + +data branch merge t1 into t0; + +-- t0: (1,1) c updated to 99, (3,3) inserted, d is not written +select * from t0 order by a; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 5: merge is idempotent - merging twice produces no new changes +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0; +insert into t1 values(3,3,30); +create snapshot sp1 for table test t1; + +data branch merge t1 into t0; +select * from t0 order by a; + +-- Second merge: no diff, no changes +data branch merge t1 into t0; +select * from t0 order by a; + drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; + drop database test; From ab3099ccd3876d30fa408685d044120390e0c88e Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 8 Jul 2026 16:38:13 +0800 Subject: [PATCH 07/31] fix(frontend): correct tuple indexing for schema evolution diff/merge The getTupleColumnValue function had an off-by-one bug for tuples with totalColCnt+1 elements. When projectBaseBatchToTarget creates a batch with [RowID, data...] (no commit_ts), the value tuple has totalColCnt+1 elements with RowID at index 0. The old code used tuple[colIdx] which read RowID bytes as data column values, causing 'unexpected byte slice for fixed-width column type INT' errors. Fix: use tuple[colIdx+1] for totalColCnt+1 tuples, matching the existing totalColCnt+2 case which already skips the leading RowID. Also add WHEN CONFLICT ACCEPT to merge_schema_evolve.sql Cases 2 and 4 which have UPDATE conflicts (same PK, different values on common columns). Without this clause, merge defaults to CONFLICT_FAIL. --- pkg/frontend/data_branch_hashdiff.go | 4 ++-- .../cases/git4data/branch/merge/merge_schema_evolve.result | 4 ++-- .../cases/git4data/branch/merge/merge_schema_evolve.sql | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 760015571ded3..0e4373cff1be6 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -1524,9 +1524,9 @@ func getTupleColumnValue(tuple types.Tuple, tblStuff tableStuff, colIdx int) (an ) } switch len(tuple) { - case totalColCnt, totalColCnt + 1: + case totalColCnt: return normalizeTupleColumnValue(tuple[colIdx], tblStuff.def.colTypes[colIdx]) - case totalColCnt + 2: + case totalColCnt + 1, totalColCnt + 2: return normalizeTupleColumnValue(tuple[colIdx+1], tblStuff.def.colTypes[colIdx]) default: return nil, moerr.NewInternalErrorNoCtxf( diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index f089103151925..4f56fbd2ba5ff 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -28,7 +28,7 @@ alter table t1 add column c int default 0; update t1 set b=99, c=10 where a=1; insert into t1 values(4,4,40); create snapshot sp1 for table test t1; -data branch merge t1 into t0; +data branch merge t1 into t0 when conflict accept; select * from t0 order by a; a b 1 99 @@ -63,7 +63,7 @@ alter table t1 add column d int default 0; update t1 set c=99 where a=1 and b=1; insert into t1 values(3,3,30,300); create snapshot sp1 for table test t1; -data branch merge t1 into t0; +data branch merge t1 into t0 when conflict accept; select * from t0 order by a; a b c 1 1 99 diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index 4e349fe620669..ca39b47041a59 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -42,7 +42,7 @@ update t1 set b=99, c=10 where a=1; insert into t1 values(4,4,40); create snapshot sp1 for table test t1; -data branch merge t1 into t0; +data branch merge t1 into t0 when conflict accept; -- t0: a=1 b updated to 99, a=4 inserted select * from t0 order by a; @@ -90,7 +90,7 @@ update t1 set c=99 where a=1 and b=1; insert into t1 values(3,3,30,300); create snapshot sp1 for table test t1; -data branch merge t1 into t0; +data branch merge t1 into t0 when conflict accept; -- t0: (1,1) c updated to 99, (3,3) inserted, d is not written select * from t0 order by a; From 568d5f717b01196f41f84c43137603b08b71d7a1 Mon Sep 17 00:00:00 2001 From: ULookup Date: Thu, 9 Jul 2026 15:17:51 +0800 Subject: [PATCH 08/31] fix(frontend): align data branch column indexes --- pkg/frontend/data_branch.go | 89 +++++++++++++++++++++----------- pkg/frontend/data_branch_test.go | 35 ++++++++++++- 2 files changed, 92 insertions(+), 32 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 76d17d4906936..40d383aecdb7c 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -69,10 +69,15 @@ func isDataBranchUserVisibleColumn(col *plan.ColDef) bool { func dataBranchFakePKColIdxes(tblDef *plan.TableDef) []int { idxes := make([]int, 0, len(tblDef.Cols)) - for i, col := range tblDef.Cols { + dataIdx := 0 + for _, col := range tblDef.Cols { + if col.Name == catalog.Row_ID { + continue + } if isDataBranchUserVisibleColumn(col) { - idxes = append(idxes, i) + idxes = append(idxes, dataIdx) } + dataIdx++ } return idxes } @@ -936,6 +941,21 @@ func getTableStuff( tarTblDef = tblStuff.tarRel.GetTableDef(ctx) baseTblDef = tblStuff.baseRel.GetTableDef(ctx) + for _, col := range tarTblDef.Cols { + if col.Name == catalog.Row_ID { + continue + } + + t := types.New(types.T(col.Typ.Id), col.Typ.Width, col.Typ.Scale) + + tblStuff.def.colNames = append(tblStuff.def.colNames, col.Name) + tblStuff.def.colTypes = append(tblStuff.def.colTypes, t) + + if isDataBranchUserVisibleColumn(col) { + tblStuff.def.visibleIdxes = append(tblStuff.def.visibleIdxes, len(tblStuff.def.colNames)-1) + } + } + var commonIdxes, tarOnlyIdxes []int if commonIdxes, tarOnlyIdxes, err = checkSchemaCompatibility(tarTblDef, baseTblDef); err != nil { return @@ -950,34 +970,31 @@ func getTableStuff( tblStuff.def.pkKind = compositeKind pkNames := baseTblDef.Pkey.Names for _, name := range pkNames { - idx := int(baseTblDef.Name2ColIndex[name]) + idx := dataBranchColumnIndexByName(tblStuff.def.colNames, name) + if idx < 0 { + err = moerr.NewInternalErrorNoCtxf("primary key column %q is not present in target data columns", name) + return + } tblStuff.def.pkColIdxes = append(tblStuff.def.pkColIdxes, idx) } } else { // normal pk tblStuff.def.pkKind = normalKind pkName := baseTblDef.Pkey.PkeyColName - idx := int(baseTblDef.Name2ColIndex[pkName]) + idx := dataBranchColumnIndexByName(tblStuff.def.colNames, pkName) + if idx < 0 { + err = moerr.NewInternalErrorNoCtxf("primary key column %q is not present in target data columns", pkName) + return + } tblStuff.def.pkColIdxes = append(tblStuff.def.pkColIdxes, idx) } - tblStuff.def.pkColIdx = int(baseTblDef.Name2ColIndex[baseTblDef.Pkey.PkeyColName]) - tblStuff.def.pkSeqnum = int(baseTblDef.Cols[tblStuff.def.pkColIdx].Seqnum) - - for i, col := range tarTblDef.Cols { - if col.Name == catalog.Row_ID { - continue - } - - t := types.New(types.T(col.Typ.Id), col.Typ.Width, col.Typ.Scale) - - tblStuff.def.colNames = append(tblStuff.def.colNames, col.Name) - tblStuff.def.colTypes = append(tblStuff.def.colTypes, t) - - if isDataBranchUserVisibleColumn(col) { - tblStuff.def.visibleIdxes = append(tblStuff.def.visibleIdxes, i) - } + tblStuff.def.pkColIdx = dataBranchColumnIndexByName(tblStuff.def.colNames, baseTblDef.Pkey.PkeyColName) + if tblStuff.def.pkColIdx < 0 { + err = moerr.NewInternalErrorNoCtxf("primary key column %q is not present in target data columns", baseTblDef.Pkey.PkeyColName) + return } + tblStuff.def.pkSeqnum = int(baseTblDef.Cols[baseTblDef.Name2ColIndex[baseTblDef.Pkey.PkeyColName]].Seqnum) if tblStuff.def.pkKind == fakeKind { tblStuff.def.pkColIdxes = dataBranchFakePKColIdxes(baseTblDef) } @@ -1024,6 +1041,15 @@ func getTableStuff( } +func dataBranchColumnIndexByName(colNames []string, name string) int { + for i, colName := range colNames { + if strings.EqualFold(colName, name) { + return i + } + } + return -1 +} + func diffOnBase( ctx context.Context, ses *Session, @@ -1111,17 +1137,23 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, tarO } // Iterate over target columns to find common and target-only columns. - for i, tarCol := range tarDef.Cols { - if tarCol.Name == catalog.Row_ID || - tarCol.Name == catalog.FakePrimaryKeyColName || + commonColNameSet := make(map[string]bool) + dataIdx := 0 + for _, tarCol := range tarDef.Cols { + if tarCol.Name == catalog.Row_ID { + continue + } + if tarCol.Name == catalog.FakePrimaryKeyColName || tarCol.Name == catalog.CPrimaryKeyColName { + dataIdx++ continue } baseCol, found := baseColMap[strings.ToLower(tarCol.Name)] if found { if baseCol.Typ.Id == tarCol.Typ.Id { - commonIdxes = append(commonIdxes, i) + commonIdxes = append(commonIdxes, dataIdx) + commonColNameSet[strings.ToLower(tarCol.Name)] = true } else { err = moerr.NewInternalErrorNoCtxf( "schema compatibility check: column '%s' exists in both schemas but has different types (target: %d, base: %d)", @@ -1130,14 +1162,9 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, tarO return } } else { - tarOnlyIdxes = append(tarOnlyIdxes, i) + tarOnlyIdxes = append(tarOnlyIdxes, dataIdx) } - } - - // Verify PK columns exist in common columns. - commonColNameSet := make(map[string]bool, len(commonIdxes)) - for _, idx := range commonIdxes { - commonColNameSet[strings.ToLower(tarDef.Cols[idx].Name)] = true + dataIdx++ } if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index de73b29217e94..32f01f07b7331 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -49,11 +49,11 @@ func TestDataBranchUserVisibleColumn(t *testing.T) { func TestDataBranchFakePKColIdxesUseOnlyVisibleColumns(t *testing.T) { tblDef := &plan.TableDef{ Cols: []*plan.ColDef{ + {Name: catalog.Row_ID, Hidden: true}, {Name: "tenant"}, {Name: "__mo_cbkey_006tenant003seq", Hidden: true}, {Name: "payload"}, {Name: catalog.FakePrimaryKeyColName, Hidden: true}, - {Name: catalog.Row_ID, Hidden: true}, }, } require.Equal(t, []int{0, 2}, dataBranchFakePKColIdxes(tblDef)) @@ -904,6 +904,39 @@ func TestCheckSchemaCompatibility_ExtraColumnOnTarget(t *testing.T) { require.Equal(t, []int{2}, tarOnlyIdxes) } +func TestCheckSchemaCompatibility_ReturnsDataBatchIndexes(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: catalog.Row_ID, Hidden: true}, + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: catalog.Row_ID, Hidden: true}, + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{2}, tarOnlyIdxes) +} + func TestCheckSchemaCompatibility_PKChanged(t *testing.T) { // base has PK on column "a" which is also in target, so it passes. // We need a case where the BASE's PK column does NOT exist in target. From 3cf182ea3568ef7bdce0d587bb870a1fcd012835 Mon Sep 17 00:00:00 2001 From: ULookup Date: Thu, 9 Jul 2026 15:17:51 +0800 Subject: [PATCH 09/31] fix(frontend): align data branch column indexes --- pkg/frontend/data_branch.go | 35 +++-- pkg/frontend/data_branch_hashdiff.go | 32 +++-- pkg/frontend/data_branch_hashdiff_test.go | 131 ++++++++++++------ pkg/frontend/data_branch_output.go | 10 +- pkg/frontend/data_branch_output_test.go | 25 ++++ pkg/frontend/data_branch_test.go | 83 ++++++++++- pkg/frontend/data_branch_types.go | 7 +- .../branch/diff/diff_schema_evolve.result | 41 ++++++ .../branch/diff/diff_schema_evolve.sql | 60 ++++++++ .../branch/edge/branch_edge_cases.result | 3 +- .../branch/edge/branch_edge_cases.sql | 3 +- .../branch/merge/merge_schema_evolve.result | 32 +++++ .../branch/merge/merge_schema_evolve.sql | 42 ++++++ 13 files changed, 427 insertions(+), 77 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 40d383aecdb7c..0c6ab237256c0 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -956,11 +956,12 @@ func getTableStuff( } } - var commonIdxes, tarOnlyIdxes []int - if commonIdxes, tarOnlyIdxes, err = checkSchemaCompatibility(tarTblDef, baseTblDef); err != nil { + var commonIdxes, commonVisibleIdxes, tarOnlyIdxes []int + if commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err = checkSchemaCompatibility(tarTblDef, baseTblDef); err != nil { return } tblStuff.def.commonIdxes = commonIdxes + tblStuff.def.commonVisibleIdxes = commonVisibleIdxes tblStuff.def.tarOnlyIdxes = tarOnlyIdxes if baseTblDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { @@ -1000,16 +1001,14 @@ func getTableStuff( } // Build baseColToTarIdx: map each base data column (in batch column order, - // excluding Row_ID / FakePK / CPK) to its position in target colNames. + // excluding Row_ID) to its position in target colNames. // colNames is already populated above from tarTblDef.Cols (excluding Row_ID, - // including FakePK/CPK). This mapping is used by projectBaseBatchToTarget to - // place base batch vectors at the correct target positions. + // including FakePK/CPK). Hidden key columns must stay in the mapping because + // BranchHashmap keys may be built from FakePK/CPK vectors. { baseDataNames := make([]string, 0, len(baseTblDef.Cols)) for _, col := range baseTblDef.Cols { - if col.Name == catalog.Row_ID || - col.Name == catalog.FakePrimaryKeyColName || - col.Name == catalog.CPrimaryKeyColName { + if col.Name == catalog.Row_ID { continue } baseDataNames = append(baseDataNames, col.Name) @@ -1129,7 +1128,7 @@ func diffOnBase( return } -func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, tarOnlyIdxes []int, err error) { +func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, commonVisibleIdxes, tarOnlyIdxes []int, err error) { // Build a map from lowercase base column name to *ColDef. baseColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) for _, col := range baseDef.Cols { @@ -1153,7 +1152,10 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, tarO if found { if baseCol.Typ.Id == tarCol.Typ.Id { commonIdxes = append(commonIdxes, dataIdx) - commonColNameSet[strings.ToLower(tarCol.Name)] = true + if isDataBranchUserVisibleColumn(tarCol) { + commonVisibleIdxes = append(commonVisibleIdxes, dataIdx) + commonColNameSet[strings.ToLower(tarCol.Name)] = true + } } else { err = moerr.NewInternalErrorNoCtxf( "schema compatibility check: column '%s' exists in both schemas but has different types (target: %d, base: %d)", @@ -1191,6 +1193,19 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, tarO } } + for _, baseCol := range baseDef.Cols { + if !isDataBranchUserVisibleColumn(baseCol) { + continue + } + if !commonColNameSet[strings.ToLower(baseCol.Name)] { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: base column '%s' is not present in target schema", + baseCol.Name, + ) + return + } + } + return } diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 0e4373cff1be6..c904dcafa3578 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -1150,7 +1150,7 @@ func compareRowInWrappedBatches( if len(compareIdxes) == 0 { compareIdxes = tblStuff.def.visibleIdxes // backward compat when commonIdxes not set } - for batchIdx, colIdx := range compareIdxes { + for _, colIdx := range compareIdxes { if skipPKCols { if slices.Index(tblStuff.def.pkColIdxes, colIdx) != -1 { continue @@ -1158,8 +1158,8 @@ func compareRowInWrappedBatches( } var ( - vec1 = wrapped1.batch.Vecs[batchIdx] - vec2 = wrapped2.batch.Vecs[batchIdx] + vec1 = wrapped1.batch.Vecs[colIdx] + vec2 = wrapped2.batch.Vecs[colIdx] ) if cmp, err := compareSingleValInVector( @@ -1182,7 +1182,11 @@ func compareTupleWithBatchRow( rowIdx int, skipPKCols bool, ) (int, error) { - for _, colIdx := range tblStuff.def.visibleIdxes { + compareIdxes := tblStuff.def.commonIdxes + if len(compareIdxes) == 0 { + compareIdxes = tblStuff.def.visibleIdxes // backward compat when commonIdxes not set + } + for _, colIdx := range compareIdxes { if skipPKCols && slices.Index(tblStuff.def.pkColIdxes, colIdx) != -1 { continue } @@ -2392,12 +2396,9 @@ func buildHashmapForTable( return } -// projectBaseBatchToTarget projects a base-side data batch to match the -// target column layout. It creates a new batch with the same number of -// columns as target data batches (1 RowID + len(colNames)), copies the -// RowID vector as-is, places each base physical column at its target -// position using baseColToTarIdx, and fills target-only columns with -// constant-NULL vectors. +// projectBaseBatchToTarget projects a base-side data batch to match the target +// column layout. Ownership of moved vectors is transferred to the returned +// batch; any vectors left behind on baseBat are cleaned before returning. func projectBaseBatchToTarget( baseBat *batch.Batch, tblStuff *tableStuff, @@ -2405,9 +2406,7 @@ func projectBaseBatchToTarget( ) *batch.Batch { out := batch.NewWithSize(len(tblStuff.def.colNames) + 1) out.Vecs[0] = baseBat.Vecs[0] // RowID - for i := range tblStuff.def.colNames { - out.Vecs[i+1] = vector.NewConstNull(tblStuff.def.colTypes[i], baseBat.RowCount(), mp) - } + baseBat.Vecs[0] = nil baseColCount := baseBat.VectorCount() - 1 // subtract RowID if baseColCount > len(tblStuff.def.baseColToTarIdx) { baseColCount = len(tblStuff.def.baseColToTarIdx) @@ -2416,8 +2415,15 @@ func projectBaseBatchToTarget( tarColIdx := tblStuff.def.baseColToTarIdx[baseColIdx] if tarColIdx >= 0 && tarColIdx < len(tblStuff.def.colNames) { out.Vecs[tarColIdx+1] = baseBat.Vecs[baseColIdx+1] + baseBat.Vecs[baseColIdx+1] = nil + } + } + for i := range tblStuff.def.colNames { + if out.Vecs[i+1] == nil { + out.Vecs[i+1] = vector.NewConstNull(tblStuff.def.colTypes[i], baseBat.RowCount(), mp) } } out.SetRowCount(baseBat.RowCount()) + baseBat.Clean(mp) return out } diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index 769e536f4a341..c51bb9fe056ac 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -1822,19 +1822,19 @@ func TestProjectBaseBatchToTarget(t *testing.T) { tblStuff := newTestBranchTableStuff(ctrl) - // Simulate target having 3 columns [id, name, extra], base has 2 [id, name] - tblStuff.def.colNames = []string{"a", "b", "c"} + // Simulate target having [a, c, b], base has [a, b]. + tblStuff.def.colNames = []string{"a", "c", "b"} tblStuff.def.colTypes = []types.Type{ types.T_int64.ToType(), - types.T_varchar.ToType(), types.T_int64.ToType(), + types.T_varchar.ToType(), } // base has [a, b] only. baseColToTarIdx says: // base col 0 (a) -> target idx 0 - // base col 1 (b) -> target idx 1 - // c is target-only - tblStuff.def.baseColToTarIdx = []int{0, 1} - tblStuff.def.tarOnlyIdxes = []int{2} + // base col 1 (b) -> target idx 2 + // c is target-only and sits between the common columns + tblStuff.def.baseColToTarIdx = []int{0, 2} + tblStuff.def.tarOnlyIdxes = []int{1} mp := ses.proc.Mp() @@ -1853,12 +1853,20 @@ func TestProjectBaseBatchToTarget(t *testing.T) { require.NoError(t, vector.AppendBytes(baseBat.Vecs[2], []byte("hello"), false, mp)) baseBat.SetRowCount(1) + rowIDVec := baseBat.Vecs[0] + aVec := baseBat.Vecs[1] + bVec := baseBat.Vecs[2] projected := projectBaseBatchToTarget(baseBat, &tblStuff, mp) - // Should have 4 vectors: RowID + [a, b, c] + // Should have 4 vectors: RowID + [a, c, b] require.Equal(t, 4, projected.VectorCount()) require.Equal(t, 1, projected.RowCount()) + // Moved vectors transfer ownership to projected. + require.Same(t, rowIDVec, projected.Vecs[0]) + require.Same(t, aVec, projected.Vecs[1]) + require.Same(t, bVec, projected.Vecs[3]) + // RowID should be preserved require.Equal(t, types.T_Rowid, projected.Vecs[0].GetType().Oid) @@ -1866,17 +1874,62 @@ func TestProjectBaseBatchToTarget(t *testing.T) { require.Equal(t, types.T_int64, projected.Vecs[1].GetType().Oid) require.Equal(t, int64(42), vector.MustFixedColWithTypeCheck[int64](projected.Vecs[1])[0]) - // Col b (index 2): should be "hello" - require.Equal(t, types.T_varchar, projected.Vecs[2].GetType().Oid) - require.Equal(t, "hello", string(projected.Vecs[2].GetBytesAt(0))) + // Col c (index 2): should be constant NULL (target-only) + require.Equal(t, types.T_int64, projected.Vecs[2].GetType().Oid) + require.True(t, projected.Vecs[2].IsConst()) + require.True(t, projected.Vecs[2].IsConstNull(), "target-only column should be const NULL") - // Col c (index 3): should be constant NULL (target-only) - require.Equal(t, types.T_int64, projected.Vecs[3].GetType().Oid) - require.True(t, projected.Vecs[3].IsConst()) - require.True(t, projected.Vecs[3].IsConstNull(), "target-only column should be const NULL") + // Col b (index 3): should be "hello" + require.Equal(t, types.T_varchar, projected.Vecs[3].GetType().Oid) + require.Equal(t, "hello", string(projected.Vecs[3].GetBytesAt(0))) projected.Clean(mp) - baseBat.Clean(mp) +} + +func TestProjectBaseBatchToTargetMovesHiddenKeyColumns(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.def.colNames = []string{"a", "b", "d", catalog.CPrimaryKeyColName} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_int64.ToType(), + types.T_int64.ToType(), + types.T_varchar.ToType(), + } + tblStuff.def.baseColToTarIdx = []int{0, 1, 3} + tblStuff.def.tarOnlyIdxes = []int{2} + tblStuff.def.pkColIdx = 3 + tblStuff.def.pkColIdxes = []int{0, 1} + + mp := ses.proc.Mp() + baseBat := batch.NewWithSize(4) + baseBat.SetAttributes([]string{catalog.Row_ID, "a", "b", catalog.CPrimaryKeyColName}) + baseBat.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + baseBat.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + baseBat.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + baseBat.Vecs[3] = vector.NewVec(types.T_varchar.ToType()) + + uid, err := types.BuildUuid() + require.NoError(t, err) + blkID := objectio.NewBlockid(&uid, 0, 1) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[0], types.NewRowid(blkID, 0), false, mp)) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[1], int64(1), false, mp)) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[2], int64(2), false, mp)) + require.NoError(t, vector.AppendBytes(baseBat.Vecs[3], []byte("cpk-1-2"), false, mp)) + baseBat.SetRowCount(1) + + cpkVec := baseBat.Vecs[3] + projected := projectBaseBatchToTarget(baseBat, &tblStuff, mp) + defer projected.Clean(mp) + + require.Equal(t, 5, projected.VectorCount()) + require.Same(t, cpkVec, projected.Vecs[4]) + require.False(t, projected.Vecs[4].IsConstNull()) + require.Equal(t, "cpk-1-2", string(projected.Vecs[4].GetBytesAt(0))) + require.True(t, projected.Vecs[3].IsConstNull()) } func TestCompareRowInWrappedBatches_WithCommonIdxes(t *testing.T) { @@ -1886,48 +1939,48 @@ func TestCompareRowInWrappedBatches_WithCommonIdxes(t *testing.T) { tblStuff := newTestBranchTableStuff(ctrl) - // Set up: target has [id(0), name(1), extra(2)] - // commonIdxes = [0, 1] (only id and name are common, extra is target-only) - tblStuff.def.colNames = []string{"id", "name", "extra"} + // Set up: target has [id(0), extra(1), name(2)] + // commonIdxes = [0, 2] (only id and name are common, extra is target-only) + tblStuff.def.colNames = []string{"id", "extra", "name"} tblStuff.def.colTypes = []types.Type{ types.T_int64.ToType(), - types.T_varchar.ToType(), types.T_int64.ToType(), + types.T_varchar.ToType(), } tblStuff.def.pkKind = normalKind tblStuff.def.visibleIdxes = []int{0, 1, 2} - tblStuff.def.commonIdxes = []int{0, 1} // only compare id and name + tblStuff.def.commonIdxes = []int{0, 2} // only compare id and name tblStuff.def.pkColIdx = 0 tblStuff.def.pkColIdxes = []int{0} mp := ses.proc.Mp() - // Batch1: [id=1, name="same", extra=999] + // Batch1: [id=1, extra=999, name="same"] bat1 := batch.NewWithSize(3) - bat1.SetAttributes([]string{"id", "name", "extra"}) + bat1.SetAttributes([]string{"id", "extra", "name"}) bat1.Vecs[0] = vector.NewVec(types.T_int64.ToType()) - bat1.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) - bat1.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + bat1.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + bat1.Vecs[2] = vector.NewVec(types.T_varchar.ToType()) require.NoError(t, vector.AppendFixed(bat1.Vecs[0], int64(1), false, mp)) - require.NoError(t, vector.AppendBytes(bat1.Vecs[1], []byte("same"), false, mp)) - require.NoError(t, vector.AppendFixed(bat1.Vecs[2], int64(999), false, mp)) + require.NoError(t, vector.AppendFixed(bat1.Vecs[1], int64(999), false, mp)) + require.NoError(t, vector.AppendBytes(bat1.Vecs[2], []byte("same"), false, mp)) bat1.SetRowCount(1) - // Batch2: [id=1, name="same", extra=0] (extra differs but is target-only) + // Batch2: [id=1, extra=0, name="same"] (extra differs but is target-only) bat2 := batch.NewWithSize(3) - bat2.SetAttributes([]string{"id", "name", "extra"}) + bat2.SetAttributes([]string{"id", "extra", "name"}) bat2.Vecs[0] = vector.NewVec(types.T_int64.ToType()) - bat2.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) - bat2.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + bat2.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + bat2.Vecs[2] = vector.NewVec(types.T_varchar.ToType()) require.NoError(t, vector.AppendFixed(bat2.Vecs[0], int64(1), false, mp)) - require.NoError(t, vector.AppendBytes(bat2.Vecs[1], []byte("same"), false, mp)) - require.NoError(t, vector.AppendFixed(bat2.Vecs[2], int64(0), false, mp)) + require.NoError(t, vector.AppendFixed(bat2.Vecs[1], int64(0), false, mp)) + require.NoError(t, vector.AppendBytes(bat2.Vecs[2], []byte("same"), false, mp)) bat2.SetRowCount(1) wrapped1 := batchWithKind{kind: diffInsert, batch: bat1} wrapped2 := batchWithKind{kind: diffInsert, batch: bat2} - // Rows should be considered equal because extra (index 2) is NOT in commonIdxes + // Rows should be considered equal because extra (index 1) is NOT in commonIdxes cmp, err := compareRowInWrappedBatches( context.Background(), ses, @@ -1942,13 +1995,13 @@ func TestCompareRowInWrappedBatches_WithCommonIdxes(t *testing.T) { // Now test with a difference in a common column (name) bat3 := batch.NewWithSize(3) - bat3.SetAttributes([]string{"id", "name", "extra"}) + bat3.SetAttributes([]string{"id", "extra", "name"}) bat3.Vecs[0] = vector.NewVec(types.T_int64.ToType()) - bat3.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) - bat3.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + bat3.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + bat3.Vecs[2] = vector.NewVec(types.T_varchar.ToType()) require.NoError(t, vector.AppendFixed(bat3.Vecs[0], int64(1), false, mp)) - require.NoError(t, vector.AppendBytes(bat3.Vecs[1], []byte("different"), false, mp)) - require.NoError(t, vector.AppendFixed(bat3.Vecs[2], int64(999), false, mp)) + require.NoError(t, vector.AppendFixed(bat3.Vecs[1], int64(999), false, mp)) + require.NoError(t, vector.AppendBytes(bat3.Vecs[2], []byte("different"), false, mp)) bat3.SetRowCount(1) wrapped3 := batchWithKind{kind: diffInsert, batch: bat3} diff --git a/pkg/frontend/data_branch_output.go b/pkg/frontend/data_branch_output.go index cdddc5c19e2df..016a067cfbaa7 100644 --- a/pkg/frontend/data_branch_output.go +++ b/pkg/frontend/data_branch_output.go @@ -156,7 +156,7 @@ func newApplyBatchInfo( visibleIdxes := tblStuff.def.visibleIdxes if len(tblStuff.def.tarOnlyIdxes) > 0 { - visibleIdxes = tblStuff.def.commonIdxes + visibleIdxes = tblStuff.def.commonVisibleIdxes } visibleNames := make([]string, len(visibleIdxes)) for i, idx := range visibleIdxes { @@ -1183,7 +1183,7 @@ func appendDataBranchApplyRowAsSQLValues( } else { insertIdxes := tblStuff.def.visibleIdxes if len(tblStuff.def.tarOnlyIdxes) > 0 { - insertIdxes = tblStuff.def.commonIdxes + insertIdxes = tblStuff.def.commonVisibleIdxes } if err = writeInsertRowValues(ses, tblStuff, row, tmpValsBuffer, insertIdxes); err != nil { return err @@ -1846,13 +1846,17 @@ func flushSqlValues( defer releaseBuffer(tblStuff.bufPool, sqlBuffer) initInsertIntoBuf := func() { + insertIdxes := tblStuff.def.visibleIdxes + if len(tblStuff.def.tarOnlyIdxes) > 0 { + insertIdxes = tblStuff.def.commonVisibleIdxes + } sqlBuffer.WriteString(fmt.Sprintf( "insert into %s (%s) values ", qualifiedTableName( tblStuff.baseRel.GetTableDef(ctx).DbName, tblStuff.baseRel.GetTableDef(ctx).Name, ), - strings.Join(quotedColumnNamesByIdxes(tblStuff, tblStuff.def.visibleIdxes), ","), + strings.Join(quotedColumnNamesByIdxes(tblStuff, insertIdxes), ","), )) } diff --git a/pkg/frontend/data_branch_output_test.go b/pkg/frontend/data_branch_output_test.go index d93bd190ad67a..3ebb402416943 100644 --- a/pkg/frontend/data_branch_output_test.go +++ b/pkg/frontend/data_branch_output_test.go @@ -982,6 +982,31 @@ func TestDataBranchOutputNewSingleWriteAppenderSubmitFail(t *testing.T) { require.Error(t, err) } +func TestNewApplyBatchInfoUsesCommonVisibleColumnsForEvolvedSchema(t *testing.T) { + ctx := context.Background() + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.def.colNames = []string{"a", "__mo_cbkey_001a", "c", "b"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_varchar.ToType(), + types.T_int64.ToType(), + types.T_int64.ToType(), + } + tblStuff.def.visibleIdxes = []int{0, 2, 3} + tblStuff.def.commonIdxes = []int{0, 1, 3} + tblStuff.def.commonVisibleIdxes = []int{0, 3} + tblStuff.def.tarOnlyIdxes = []int{2} + + info := newApplyBatchInfo(ctx, ses, tblStuff, []int{0}, false) + require.NotNil(t, info) + require.Equal(t, []string{"a"}, info.deleteKeyNames) + require.Equal(t, []string{"a", "b"}, info.visibleNames) +} + func TestDataBranchOutputRemoveFileIgnoreError(t *testing.T) { ctx := context.Background() filePath := filepath.Join(t.TempDir(), "diff.sql") diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index 32f01f07b7331..69e4d8d16fd99 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -867,9 +867,10 @@ func TestCheckSchemaCompatibility_Identical(t *testing.T) { }, } - commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) require.NoError(t, err) require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{0, 1}, commonVisibleIdxes) require.Empty(t, tarOnlyIdxes) } @@ -898,9 +899,10 @@ func TestCheckSchemaCompatibility_ExtraColumnOnTarget(t *testing.T) { }, } - commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) require.NoError(t, err) require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{0, 1}, commonVisibleIdxes) require.Equal(t, []int{2}, tarOnlyIdxes) } @@ -931,9 +933,46 @@ func TestCheckSchemaCompatibility_ReturnsDataBatchIndexes(t *testing.T) { }, } - commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) require.NoError(t, err) require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{0, 1}, commonVisibleIdxes) + require.Equal(t, []int{2}, tarOnlyIdxes) +} + +func TestCheckSchemaCompatibility_SeparatesPhysicalAndVisibleCommonIndexes(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: catalog.Row_ID, Hidden: true}, + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "__mo_cbkey_001a", Typ: plan.Type{Id: int32(types.T_varchar)}, Hidden: true}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: catalog.Row_ID, Hidden: true}, + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "__mo_cbkey_001a", Typ: plan.Type{Id: int32(types.T_varchar)}, Hidden: true}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1, 3}, commonIdxes) + require.Equal(t, []int{0, 3}, commonVisibleIdxes) require.Equal(t, []int{2}, tarOnlyIdxes) } @@ -963,7 +1002,7 @@ func TestCheckSchemaCompatibility_PKChanged(t *testing.T) { }, } - _, _, err := checkSchemaCompatibility(tarDef, baseDef) + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) require.Error(t, err) require.Contains(t, err.Error(), "primary key column") } @@ -992,11 +1031,39 @@ func TestCheckSchemaCompatibility_TypeMismatch(t *testing.T) { }, } - _, _, err := checkSchemaCompatibility(tarDef, baseDef) + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) require.Error(t, err) require.Contains(t, err.Error(), "has different types") } +func TestCheckSchemaCompatibility_BaseOnlyVisibleColumnRejected(t *testing.T) { + tarDef := &plan.TableDef{ + Name: "target", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Name: "base", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"a"}, + PkeyColName: "a", + }, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.Error(t, err) + require.Contains(t, err.Error(), "base column 'b' is not present in target schema") +} + func TestCheckSchemaCompatibility_CompositePK(t *testing.T) { tarDef := &plan.TableDef{ Name: "target", @@ -1024,9 +1091,10 @@ func TestCheckSchemaCompatibility_CompositePK(t *testing.T) { }, } - commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) require.NoError(t, err) require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{0, 1}, commonVisibleIdxes) require.Equal(t, []int{2}, tarOnlyIdxes) } @@ -1053,8 +1121,9 @@ func TestCheckSchemaCompatibility_FakePK(t *testing.T) { }, } - commonIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) require.NoError(t, err) require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{0, 1}, commonVisibleIdxes) require.Equal(t, []int{2}, tarOnlyIdxes) } diff --git a/pkg/frontend/data_branch_types.go b/pkg/frontend/data_branch_types.go index 442b391d2912c..cf7d2f6dd866b 100644 --- a/pkg/frontend/data_branch_types.go +++ b/pkg/frontend/data_branch_types.go @@ -204,9 +204,10 @@ type tableStuff struct { pkColIdxes []int // expanded pk columns pkKind int - commonIdxes []int // indices of common columns (target ordering) - tarOnlyIdxes []int // indices of target-only columns (target ordering) - baseColToTarIdx []int // for base batch Vec[i+1], the target column index, or -1 + commonIdxes []int // indices of common columns (target data-batch ordering) + commonVisibleIdxes []int // visible subset of commonIdxes for SQL/output/apply + tarOnlyIdxes []int // indices of target-only columns (target data-batch ordering) + baseColToTarIdx []int // for base batch Vec[i+1], the target column index, or -1 } worker *ants.Pool diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index b20eb6e8420cd..85c6d7ca8ddfd 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -112,4 +112,45 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +update t1 set b=99 where a=1; +update t1 set c=88 where a=2; +insert into t1(a,c,b) values(4,40,4); +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a c b +t1 UPDATE 1 0 99 +t1 INSERT 4 40 4 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int) cluster by(a,b); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +diff t1 against t0 flag a c b +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 drop column b; +create snapshot sp1 for table test t1; +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +-- @regex("schema compatibility check: base column 'b' is not present in target schema", true) +internal error: schema compatibility check: base column 'b' is not present in target schema +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; drop database test; diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index b5ee035abb093..5b9eb90b6efee 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -175,4 +175,64 @@ drop snapshot sp0; drop table t0; drop table t1; +-- ===================================================== +-- Case 9: target-only column sits between common columns +-- base: [a, b] +-- target: [a, c, b] +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +update t1 set b=99 where a=1; +update t1 set c=88 where a=2; +insert into t1(a,c,b) values(4,40,4); +create snapshot sp1 for table test t1; + +-- a=1 common column b changed; a=2 only target-only c changed; a=4 inserted. +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 10: cluster-by hidden helper columns stay out of SQL/output columns +-- ===================================================== +create table t0(a int, b int) cluster by(a,b); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +create snapshot sp1 for table test t1; + +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 11: base-only non-PK column is rejected +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 drop column b; +create snapshot sp1 for table test t1; + +-- @regex("schema compatibility check: base column 'b' is not present in target schema", true) +data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + drop database test; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result index 27f759172eecb..bae92b6d051b3 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result @@ -83,7 +83,8 @@ data branch create table rename_left from base; data branch create table rename_right from base; alter table rename_left rename column b to bb; data branch diff rename_left against rename_right; -diff rename_left against rename_right flag a bb +-- @regex("schema compatibility check: base column 'b' is not present in target schema",true) +internal error: schema compatibility check: base column 'b' is not present in target schema drop database br_schema_drift; drop database if exists br_txn_src; drop database if exists br_txn_dst; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql index 96a8791367ae0..70915d3709644 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql @@ -83,7 +83,8 @@ data branch diff type_left against type_right; data branch create table rename_left from base; data branch create table rename_right from base; alter table rename_left rename column b to bb; --- A rename on one branch keeps the renamed table queryable for diff. +-- A one-sided rename removes base-visible column b from the target schema. +-- @regex("schema compatibility check: base column 'b' is not present in target schema",true) data branch diff rename_left against rename_right; drop database br_schema_drift; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index 4f56fbd2ba5ff..3929b91aeb3d9 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -96,4 +96,36 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +update t1 set b=99 where a=1; +update t1 set c=88 where a=2; +insert into t1(a,c,b) values(4,40,4); +create snapshot sp1 for table test t1; +data branch merge t1 into t0 when conflict accept; +select * from t0 order by a; +a b +1 99 +2 2 +3 3 +4 4 +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; +create table t0(a int, b int) cluster by(a,b); +create snapshot sp0 for table test t0; +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +create snapshot sp1 for table test t1; +data branch merge t1 into t0 when conflict accept; +select * from t0 order by a; +a b +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index ca39b47041a59..0690723048070 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -124,4 +124,46 @@ drop snapshot sp0; drop table t0; drop table t1; +-- ===================================================== +-- Case 6: target-only column sits between common columns +-- base: [a, b] +-- target: [a, c, b] +-- ===================================================== +create table t0(a int, b int, primary key(a)); +insert into t0 values(1,1),(2,2),(3,3); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +update t1 set b=99 where a=1; +update t1 set c=88 where a=2; +insert into t1(a,c,b) values(4,40,4); +create snapshot sp1 for table test t1; + +data branch merge t1 into t0 when conflict accept; +select * from t0 order by a; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 7: cluster-by hidden helper columns stay out of SQL/apply columns +-- ===================================================== +create table t0(a int, b int) cluster by(a,b); +create snapshot sp0 for table test t0; + +data branch create table t1 from t0{snapshot="sp0"}; +alter table t1 add column c int default 0 after a; +create snapshot sp1 for table test t1; + +data branch merge t1 into t0 when conflict accept; +select * from t0 order by a; + +drop snapshot sp1; +drop snapshot sp0; +drop table t0; +drop table t1; + drop database test; From ee0b8aad212fcd878ab4c5b38828ffef777c238e Mon Sep 17 00:00:00 2001 From: ULookup Date: Fri, 10 Jul 2026 12:01:29 +0800 Subject: [PATCH 10/31] fix(frontend): harden schema compatibility checks --- pkg/frontend/data_branch.go | 56 +++++++++++++++++++--------- pkg/frontend/data_branch_test.go | 64 +++++++++++++++++++++++++++++--- 2 files changed, 97 insertions(+), 23 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 0c6ab237256c0..6b4a8c7b616f2 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -1129,13 +1129,19 @@ func diffOnBase( } func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, commonVisibleIdxes, tarOnlyIdxes []int, err error) { - // Build a map from lowercase base column name to *ColDef. baseColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) + baseVisibleColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) for _, col := range baseDef.Cols { - baseColMap[strings.ToLower(col.Name)] = col + if col.Name == catalog.Row_ID { + continue + } + name := strings.ToLower(col.Name) + baseColMap[name] = col + if isDataBranchUserVisibleColumn(col) { + baseVisibleColMap[name] = col + } } - // Iterate over target columns to find common and target-only columns. commonColNameSet := make(map[string]bool) dataIdx := 0 for _, tarCol := range tarDef.Cols { @@ -1148,13 +1154,26 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm continue } - baseCol, found := baseColMap[strings.ToLower(tarCol.Name)] + name := strings.ToLower(tarCol.Name) + baseCol, found := baseColMap[name] if found { if baseCol.Typ.Id == tarCol.Typ.Id { + if baseCol.ColId != tarCol.ColId || + baseCol.ClusterBy != tarCol.ClusterBy || + baseCol.Primary != tarCol.Primary || + baseCol.Seqnum != tarCol.Seqnum || + baseCol.NotNull != tarCol.NotNull { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: column '%s' exists in both schemas but has different metadata", + tarCol.Name, + ) + return + } commonIdxes = append(commonIdxes, dataIdx) if isDataBranchUserVisibleColumn(tarCol) { commonVisibleIdxes = append(commonVisibleIdxes, dataIdx) - commonColNameSet[strings.ToLower(tarCol.Name)] = true + commonColNameSet[name] = true + delete(baseVisibleColMap, name) } } else { err = moerr.NewInternalErrorNoCtxf( @@ -1164,10 +1183,18 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm return } } else { - tarOnlyIdxes = append(tarOnlyIdxes, dataIdx) + if isDataBranchUserVisibleColumn(tarCol) { + tarOnlyIdxes = append(tarOnlyIdxes, dataIdx) + } } dataIdx++ } + if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName && len(tarOnlyIdxes) > 0 { + err = moerr.NewInternalErrorNoCtx( + "schema compatibility check: fake primary key tables do not support target-only user-visible columns", + ) + return + } if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { // For fake PK tables, the PK column is generated from all columns. @@ -1193,17 +1220,12 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm } } - for _, baseCol := range baseDef.Cols { - if !isDataBranchUserVisibleColumn(baseCol) { - continue - } - if !commonColNameSet[strings.ToLower(baseCol.Name)] { - err = moerr.NewInternalErrorNoCtxf( - "schema compatibility check: base column '%s' is not present in target schema", - baseCol.Name, - ) - return - } + for _, baseCol := range baseVisibleColMap { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: base-only user-visible column '%s' is not supported", + baseCol.Name, + ) + return } return diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index 69e4d8d16fd99..a6c74d1e4f69c 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -976,6 +976,61 @@ func TestCheckSchemaCompatibility_SeparatesPhysicalAndVisibleCommonIndexes(t *te require.Equal(t, []int{2}, tarOnlyIdxes) } +func TestCheckSchemaCompatibility_HiddenColumnsAreNotOutputColumns(t *testing.T) { + tarDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "hidden", Hidden: true, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }} + baseDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "hidden", Hidden: true, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }} + + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1, 2}, commonIdxes) + require.Equal(t, []int{0, 2}, commonVisibleIdxes) + require.Empty(t, tarOnlyIdxes) +} + +func TestCheckSchemaCompatibility_RejectsBaseOnlyVisibleColumn(t *testing.T) { + tarDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{{Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}}} + baseDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "removed", Typ: plan.Type{Id: int32(types.T_int64)}}, + }} + + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.ErrorContains(t, err, "base-only user-visible column 'removed'") +} + +func TestCheckSchemaCompatibility_RejectsChangedSharedColumnMetadata(t *testing.T) { + metadataChanges := []struct { + name string + mutate func(*plan.ColDef) + }{ + {"column id", func(col *plan.ColDef) { col.ColId++ }}, + {"cluster by", func(col *plan.ColDef) { col.ClusterBy = !col.ClusterBy }}, + {"primary", func(col *plan.ColDef) { col.Primary = !col.Primary }}, + {"sequence number", func(col *plan.ColDef) { col.Seqnum++ }}, + {"not null", func(col *plan.ColDef) { col.NotNull = !col.NotNull }}, + } + for _, tc := range metadataChanges { + t.Run(tc.name, func(t *testing.T) { + tarCol := &plan.ColDef{Name: "a", ColId: 1, ClusterBy: true, Primary: true, Seqnum: 1, NotNull: true, Typ: plan.Type{Id: int32(types.T_int64)}} + baseCol := &plan.ColDef{Name: "a", ColId: 1, ClusterBy: true, Primary: true, Seqnum: 1, NotNull: true, Typ: plan.Type{Id: int32(types.T_int64)}} + tc.mutate(tarCol) + tarDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{tarCol}} + baseDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{baseCol}} + + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.ErrorContains(t, err, "has different metadata") + }) + } +} + func TestCheckSchemaCompatibility_PKChanged(t *testing.T) { // base has PK on column "a" which is also in target, so it passes. // We need a case where the BASE's PK column does NOT exist in target. @@ -1061,7 +1116,7 @@ func TestCheckSchemaCompatibility_BaseOnlyVisibleColumnRejected(t *testing.T) { _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) require.Error(t, err) - require.Contains(t, err.Error(), "base column 'b' is not present in target schema") + require.Contains(t, err.Error(), "base-only user-visible column 'b' is not supported") } func TestCheckSchemaCompatibility_CompositePK(t *testing.T) { @@ -1121,9 +1176,6 @@ func TestCheckSchemaCompatibility_FakePK(t *testing.T) { }, } - commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) - require.NoError(t, err) - require.Equal(t, []int{0, 1}, commonIdxes) - require.Equal(t, []int{0, 1}, commonVisibleIdxes) - require.Equal(t, []int{2}, tarOnlyIdxes) + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.ErrorContains(t, err, "fake primary key") } From d5a5c6a27d746c4f48cba94c7a323f87654c0e14 Mon Sep 17 00:00:00 2001 From: ULookup Date: Fri, 10 Jul 2026 13:24:35 +0800 Subject: [PATCH 11/31] fix(frontend): relax schema evolution metadata checks --- pkg/frontend/data_branch.go | 13 +------------ pkg/frontend/data_branch_test.go | 29 ++--------------------------- 2 files changed, 3 insertions(+), 39 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 6b4a8c7b616f2..25f00afd19cc2 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -1158,17 +1158,6 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm baseCol, found := baseColMap[name] if found { if baseCol.Typ.Id == tarCol.Typ.Id { - if baseCol.ColId != tarCol.ColId || - baseCol.ClusterBy != tarCol.ClusterBy || - baseCol.Primary != tarCol.Primary || - baseCol.Seqnum != tarCol.Seqnum || - baseCol.NotNull != tarCol.NotNull { - err = moerr.NewInternalErrorNoCtxf( - "schema compatibility check: column '%s' exists in both schemas but has different metadata", - tarCol.Name, - ) - return - } commonIdxes = append(commonIdxes, dataIdx) if isDataBranchUserVisibleColumn(tarCol) { commonVisibleIdxes = append(commonVisibleIdxes, dataIdx) @@ -1222,7 +1211,7 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm for _, baseCol := range baseVisibleColMap { err = moerr.NewInternalErrorNoCtxf( - "schema compatibility check: base-only user-visible column '%s' is not supported", + "schema compatibility check: base column '%s' is not present in target schema", baseCol.Name, ) return diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index a6c74d1e4f69c..0611344000526 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -1003,32 +1003,7 @@ func TestCheckSchemaCompatibility_RejectsBaseOnlyVisibleColumn(t *testing.T) { }} _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) - require.ErrorContains(t, err, "base-only user-visible column 'removed'") -} - -func TestCheckSchemaCompatibility_RejectsChangedSharedColumnMetadata(t *testing.T) { - metadataChanges := []struct { - name string - mutate func(*plan.ColDef) - }{ - {"column id", func(col *plan.ColDef) { col.ColId++ }}, - {"cluster by", func(col *plan.ColDef) { col.ClusterBy = !col.ClusterBy }}, - {"primary", func(col *plan.ColDef) { col.Primary = !col.Primary }}, - {"sequence number", func(col *plan.ColDef) { col.Seqnum++ }}, - {"not null", func(col *plan.ColDef) { col.NotNull = !col.NotNull }}, - } - for _, tc := range metadataChanges { - t.Run(tc.name, func(t *testing.T) { - tarCol := &plan.ColDef{Name: "a", ColId: 1, ClusterBy: true, Primary: true, Seqnum: 1, NotNull: true, Typ: plan.Type{Id: int32(types.T_int64)}} - baseCol := &plan.ColDef{Name: "a", ColId: 1, ClusterBy: true, Primary: true, Seqnum: 1, NotNull: true, Typ: plan.Type{Id: int32(types.T_int64)}} - tc.mutate(tarCol) - tarDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{tarCol}} - baseDef := &plan.TableDef{Pkey: &plan.PrimaryKeyDef{PkeyColName: "a"}, Cols: []*plan.ColDef{baseCol}} - - _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) - require.ErrorContains(t, err, "has different metadata") - }) - } + require.ErrorContains(t, err, "base column 'removed' is not present in target schema") } func TestCheckSchemaCompatibility_PKChanged(t *testing.T) { @@ -1116,7 +1091,7 @@ func TestCheckSchemaCompatibility_BaseOnlyVisibleColumnRejected(t *testing.T) { _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) require.Error(t, err) - require.Contains(t, err.Error(), "base-only user-visible column 'b' is not supported") + require.Contains(t, err.Error(), "base column 'b' is not present in target schema") } func TestCheckSchemaCompatibility_CompositePK(t *testing.T) { From 04467eef6bf1fe75d4b16014e67160182349e693 Mon Sep 17 00:00:00 2001 From: ULookup Date: Fri, 10 Jul 2026 14:05:54 +0800 Subject: [PATCH 12/31] fix(frontend): preserve branch lineage after schema evolution --- pkg/frontend/data_branch.go | 37 ++- pkg/frontend/data_branch_hashdiff.go | 15 +- pkg/frontend/data_branch_hashdiff_test.go | 292 ++++++++++++++++++++++ pkg/frontend/data_branch_test.go | 9 +- 4 files changed, 337 insertions(+), 16 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 25f00afd19cc2..a5ded978c5468 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -1178,13 +1178,6 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm } dataIdx++ } - if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName && len(tarOnlyIdxes) > 0 { - err = moerr.NewInternalErrorNoCtx( - "schema compatibility check: fake primary key tables do not support target-only user-visible columns", - ) - return - } - if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { // For fake PK tables, the PK column is generated from all columns. // No specific PK name to verify. @@ -2454,21 +2447,30 @@ func decideLCABranchTSFromBranchDAG( return } - if lcaTableID, _, _, hasLca := dag.FindLCA(tarTableID, baseTableID); hasLca { - tarIDs, tarTSs, ok := dag.PathFromAncestor(tarTableID, lcaTableID) + tarLineageID := dataBranchLineageTableID(ctx, dag, tblStuff.tarRel) + baseLineageID := dataBranchLineageTableID(ctx, dag, tblStuff.baseRel) + + if lcaTableID, _, _, hasLca := dag.FindLCA(tarLineageID, baseLineageID); hasLca { + tarIDs, tarTSs, ok := dag.PathFromAncestor(tarLineageID, lcaTableID) if !ok { err = moerr.NewInternalErrorNoCtxf( "data branch: DAG path broken from LCA %d to tar %d", - lcaTableID, tarTableID) + lcaTableID, tarLineageID) return } - baseIDs, baseTSs, ok := dag.PathFromAncestor(baseTableID, lcaTableID) + baseIDs, baseTSs, ok := dag.PathFromAncestor(baseLineageID, lcaTableID) if !ok { err = moerr.NewInternalErrorNoCtxf( "data branch: DAG path broken from LCA %d to base %d", - lcaTableID, baseTableID) + lcaTableID, baseLineageID) return } + if tarLineageID != tarTableID { + tarIDs[len(tarIDs)-1] = tarTableID + } + if baseLineageID != baseTableID { + baseIDs[len(baseIDs)-1] = baseTableID + } branchInfo.lcaTableId = lcaTableID branchInfo.pathFromLCAToTar = tarIDs branchInfo.pathFromLCAToTarTS = buildTSs(tarTSs) @@ -2491,6 +2493,17 @@ func decideLCABranchTSFromBranchDAG( return } +func dataBranchLineageTableID(ctx context.Context, dag *databranchutils.DataBranchDAG, rel engine.Relation) uint64 { + tableID := rel.GetTableID(ctx) + if dag.Exists(tableID) { + return tableID + } + if tblDef := rel.GetTableDef(ctx); tblDef != nil && tblDef.LogicalId != 0 && dag.Exists(tblDef.LogicalId) { + return tblDef.LogicalId + } + return tableID +} + // buildTSs lifts a slice of int64 physical timestamps into the // types.TS form expected by branchMetaInfo. func buildTSs(in []int64) []types.TS { diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index c904dcafa3578..a58045ac62649 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -1548,6 +1548,19 @@ func visibleTupleKeyIdxes(tblStuff tableStuff) []int { return idxes } +func fakePKTupleKeyIdxes(tblStuff tableStuff) []int { + keyColIdxes := tblStuff.def.commonVisibleIdxes + if len(keyColIdxes) == 0 { + keyColIdxes = tblStuff.def.visibleIdxes + } + + idxes := make([]int, len(keyColIdxes)) + for i, colIdx := range keyColIdxes { + idxes[i] = colIdx + 1 + } + return idxes +} + func batchSampleRowsForLog(bat *batch.Batch, limit int) []string { if bat == nil || bat.RowCount() == 0 || limit <= 0 { return nil @@ -1747,7 +1760,7 @@ func diffDataHelper( if tblStuff.def.pkKind == fakeKind { var ( - keyIdxes = visibleTupleKeyIdxes(tblStuff) + keyIdxes = fakePKTupleKeyIdxes(tblStuff) newHashmap databranchutils.BranchHashmap ) diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index c51bb9fe056ac..f1a531804de16 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -524,6 +524,174 @@ func TestDiffDataHelperClosesMigratedHashmaps(t *testing.T) { require.Equal(t, 1, tarMigratedHashmap.closeCalls) } +func TestDiffDataHelperFakePKUsesCommonVisibleColumnsAsKey(t *testing.T) { + ses := newValidateSession(t) + mp := ses.proc.Mp() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.def.pkKind = fakeKind + tblStuff.def.colNames = []string{"a", "c", "b", catalog.FakePrimaryKeyColName} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_int64.ToType(), + types.T_int64.ToType(), + types.T_uint64.ToType(), + } + tblStuff.def.visibleIdxes = []int{0, 1, 2} + tblStuff.def.commonIdxes = []int{0, 2} + tblStuff.def.commonVisibleIdxes = []int{0, 2} + tblStuff.def.tarOnlyIdxes = []int{1} + tblStuff.def.pkColIdxes = []int{0, 2} + + // Rows differ only in target-only column c. Fake-PK matching must use the + // common visible columns [a,b], otherwise schema evolution turns a no-op + // target-only update into one target INSERT plus one base INSERT. + rowColTypes := append([]types.Type{types.T_Rowid.ToType()}, tblStuff.def.colTypes...) + tarHashmap := buildTestBranchHashmap( + t, mp, rowColTypes, + [][]any{{buildHashDiffRowID(t, 1), int64(1), int64(99), int64(1), uint64(11)}}, + ) + defer func() { + require.NoError(t, tarHashmap.Close()) + }() + baseHashmap := buildTestBranchHashmap( + t, mp, rowColTypes, + [][]any{{buildHashDiffRowID(t, 2), int64(1), nil, int64(1), uint64(22)}}, + ) + defer func() { + require.NoError(t, baseHashmap.Close()) + }() + + var got []capturedBatch + var mu sync.Mutex + err := diffDataHelper( + context.Background(), + ses, + compositeOption{}, + tblStuff, + func(w batchWithKind) (bool, error) { + rows := decodeCapturedRows(t, w.batch, tblStuff.def.colTypes) + mu.Lock() + if len(rows) != 0 { + got = append(got, capturedBatch{ + kind: w.kind, + side: w.side, + rows: rows, + }) + } + mu.Unlock() + tblStuff.retPool.releaseRetBatch(w.batch, false) + return false, nil + }, + tarHashmap, + baseHashmap, + ) + require.NoError(t, err) + require.Empty(t, got) +} + +func TestBuildHashmapForTableProjectsBaseRowsBeforeKeying(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.def.colNames = []string{"a", "b", "c"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_int64.ToType(), + types.T_int64.ToType(), + } + tblStuff.def.visibleIdxes = []int{0, 1, 2} + tblStuff.def.commonIdxes = []int{0, 1} + tblStuff.def.commonVisibleIdxes = []int{0, 1} + tblStuff.def.tarOnlyIdxes = []int{2} + tblStuff.def.baseColToTarIdx = []int{0, 1} + tblStuff.def.pkColIdx = 0 + tblStuff.def.pkColIdxes = []int{0} + tblStuff.hashmapAllocator = newBranchHashmapAllocator(dataBranchHashmapLimitRate) + worker, err := ants.NewPool(1) + require.NoError(t, err) + defer worker.Release() + tblStuff.worker = worker + + baseRowID := buildHashDiffRowID(t, 1) + baseBat := buildIntDataBatch( + t, ses.proc.Mp(), []string{catalog.Row_ID, "a", "b"}, + []types.Type{types.T_Rowid.ToType(), types.T_int64.ToType(), types.T_int64.ToType()}, + [][]any{{baseRowID, int64(1), int64(1)}}, + ) + baseDataHashmap, baseTombstoneHashmap, err := buildHashmapForTable( + context.Background(), + ses.proc.Mp(), + &tblStuff, + []engine.ChangesHandle{&stubEngineChangesHandle{ + responses: []stubEngineChangesHandleResponse{{data: baseBat}}, + }}, + "base", + nil, + ) + require.NoError(t, err) + defer func() { + require.NoError(t, baseDataHashmap.Close()) + require.NoError(t, baseTombstoneHashmap.Close()) + }() + + targetRowID := buildHashDiffRowID(t, 2) + targetBat := buildIntDataBatch( + t, ses.proc.Mp(), []string{catalog.Row_ID, "a", "b", "c"}, + []types.Type{types.T_Rowid.ToType(), types.T_int64.ToType(), types.T_int64.ToType(), types.T_int64.ToType()}, + [][]any{{targetRowID, int64(1), int64(99), int64(0)}}, + ) + targetDataHashmap, targetTombstoneHashmap, err := buildHashmapForTable( + context.Background(), + ses.proc.Mp(), + &tblStuff, + []engine.ChangesHandle{&stubEngineChangesHandle{ + responses: []stubEngineChangesHandleResponse{{data: targetBat}}, + }}, + "target", + nil, + ) + require.NoError(t, err) + defer func() { + require.NoError(t, targetDataHashmap.Close()) + require.NoError(t, targetTombstoneHashmap.Close()) + }() + + require.NoError(t, targetDataHashmap.ForEachShardParallel(func(cursor databranchutils.ShardCursor) error { + return cursor.ForEach(func(key []byte, _ []byte) error { + ret, err := baseDataHashmap.PopByEncodedKey(key, false) + require.NoError(t, err) + require.True(t, ret.Exists) + require.Len(t, ret.Rows, 1) + baseTuple, _, err := baseDataHashmap.DecodeRow(ret.Rows[0]) + require.NoError(t, err) + require.Len(t, baseTuple, 4) + return nil + }) + }, -1)) +} + +func TestDataBranchLineageTableIDFallsBackToLogicalID(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + dag := databranchutils.NewDAG([]databranchutils.DataBranchMetadata{ + {TableID: 10, PTableID: 1, CloneTS: 100}, + }) + rel := mock_frontend.NewMockRelation(ctrl) + rel.EXPECT().GetTableID(gomock.Any()).Return(uint64(20)).AnyTimes() + rel.EXPECT().GetTableDef(gomock.Any()).Return(&plan.TableDef{ + TblId: 20, + LogicalId: 10, + }).AnyTimes() + + require.Equal(t, uint64(10), dataBranchLineageTableID(context.Background(), dag, rel)) +} + func TestDiffDataHelper_ConflictAcceptExpandUpdate(t *testing.T) { ses := newValidateSession(t) mp := ses.proc.Mp() @@ -893,6 +1061,10 @@ func appendTestVectorValue(vec *vector.Vector, val any, mp *mpool.MPool) error { switch x := val.(type) { case int64: return vector.AppendFixed(vec, x, false, mp) + case uint64: + return vector.AppendFixed(vec, x, false, mp) + case types.Rowid: + return vector.AppendFixed(vec, x, false, mp) case string: return vector.AppendBytes(vec, []byte(x), false, mp) case []byte: @@ -1389,6 +1561,107 @@ func TestHashDiff_HasLCANoopUpdateFiltering(t *testing.T) { } } +func TestHashDiff_HasLCAUpdateIgnoresTargetOnlyColumns(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().SnapshotTS().Return(types.BuildTS(20, 0).ToTimestamp()).AnyTimes() + ses.txnHandler = &TxnHandler{ + txnOp: txnOp, + } + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.def.commonIdxes = []int{0, 1} + tblStuff.def.commonVisibleIdxes = []int{0, 1} + tblStuff.def.tarOnlyIdxes = []int{2} + tblStuff.def.baseColToTarIdx = []int{0, 1} + worker, err := ants.NewPool(1) + require.NoError(t, err) + defer worker.Release() + tblStuff.worker = worker + tblStuff.maxTombstoneBatchCnt = 1 + tblStuff.hashmapAllocator = newBranchHashmapAllocator(dataBranchHashmapLimitRate) + + lcaRel := mock_frontend.NewMockRelation(ctrl) + lcaDef := &plan.TableDef{ + DbName: "db1", + Name: "lca_tbl", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"id"}, + PkeyColName: "id", + }, + } + lcaRel.EXPECT().GetTableDef(gomock.Any()).Return(lcaDef).AnyTimes() + lcaRel.EXPECT().GetTableID(gomock.Any()).Return(uint64(77)).AnyTimes() + tblStuff.lcaRel = lcaRel + + bh := mock_frontend.NewMockBackgroundExec(ctrl) + bh.EXPECT().Exec(gomock.Any(), gomock.Any()).Return(nil).Times(1) + bh.EXPECT().GetExecResultSet().Return([]interface{}{ + buildLCAProbeResultSetWithRows([]interface{}{int64(0), int64(1), "before", nil}), + }).Times(1) + bh.EXPECT().ClearExecResultSet().Times(1) + + oldRowID := buildHashDiffRowID(t, 0) + newRowID := buildHashDiffRowID(t, 1) + tarData := buildHashDiffDataBatchWithRowIDs( + t, ses.proc.Mp(), []types.Rowid{newRowID}, + [][]any{{int64(1), "after", "target-only", commitTSBytes(types.BuildTS(15, 0))}}, + ) + tarTombstone := buildHashDiffTombstoneBatchWithRowIDs( + t, ses.proc.Mp(), []types.Rowid{oldRowID}, + [][]any{{int64(1), commitTSBytes(types.BuildTS(12, 0))}}, + ) + + dagInfo := branchMetaInfo{ + lcaTableId: 77, + pathFromLCAToTar: []uint64{77, 88}, + pathFromLCAToTarTS: []types.TS{{}, types.BuildTS(10, 0)}, + pathFromLCAToBase: []uint64{77}, + pathFromLCAToBaseTS: []types.TS{{}}, + } + + var got []capturedBatch + var mu sync.Mutex + err = hashDiff( + context.Background(), + ses, + bh, + tblStuff, + dagInfo, + compositeOption{}, + func(w batchWithKind) (bool, error) { + rows := decodeCapturedRows(t, w.batch, tblStuff.def.colTypes) + mu.Lock() + if len(rows) != 0 { + got = append(got, capturedBatch{ + kind: w.kind, + side: w.side, + rows: rows, + }) + } + mu.Unlock() + tblStuff.retPool.releaseRetBatch(w.batch, false) + return false, nil + }, + []engine.ChangesHandle{&stubEngineChangesHandle{ + responses: []stubEngineChangesHandleResponse{{ + data: tarData, + tombstone: tarTombstone, + }}, + }}, + nil, + nil, + ) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, diffUpdate, got[0].kind) + require.Equal(t, diffSideTarget, got[0].side) + require.Equal(t, [][]any{{int64(1), "after", "target-only"}}, got[0].rows) +} + func TestHashDiff_HasLCAFakePKUpdateIsNotNoop(t *testing.T) { ses := newValidateSession(t) ctrl := gomock.NewController(t) @@ -1549,6 +1822,25 @@ func buildVisibleComparisonBatch(t *testing.T, mp *mpool.MPool, rows [][]any) *b return bat } +func buildIntDataBatch(t *testing.T, mp *mpool.MPool, attrs []string, colTypes []types.Type, rows [][]any) *batch.Batch { + t.Helper() + + require.Len(t, attrs, len(colTypes)) + bat := batch.NewWithSize(len(colTypes)) + bat.SetAttributes(attrs) + for i, typ := range colTypes { + bat.Vecs[i] = vector.NewVec(typ) + } + for _, row := range rows { + require.Len(t, row, len(colTypes)) + for i, val := range row { + require.NoError(t, appendTestVectorValue(bat.Vecs[i], val, mp)) + } + } + bat.SetRowCount(len(rows)) + return bat +} + func buildLCAProbeResultSet() *MysqlResultSet { return buildLCAProbeResultSetWithRows( []interface{}{int64(0), int64(1), "alice", "h1"}, diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index 0611344000526..993cbe509107a 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -1128,7 +1128,7 @@ func TestCheckSchemaCompatibility_CompositePK(t *testing.T) { require.Equal(t, []int{2}, tarOnlyIdxes) } -func TestCheckSchemaCompatibility_FakePK(t *testing.T) { +func TestCheckSchemaCompatibility_FakePKAllowsTargetOnlyColumns(t *testing.T) { tarDef := &plan.TableDef{ Name: "target", Pkey: &plan.PrimaryKeyDef{ @@ -1151,6 +1151,9 @@ func TestCheckSchemaCompatibility_FakePK(t *testing.T) { }, } - _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) - require.ErrorContains(t, err, "fake primary key") + commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) + require.NoError(t, err) + require.Equal(t, []int{0, 1}, commonIdxes) + require.Equal(t, []int{0, 1}, commonVisibleIdxes) + require.Equal(t, []int{2}, tarOnlyIdxes) } From e90a008066755e01cd4a3f11204eb4ed073c8c4f Mon Sep 17 00:00:00 2001 From: ULookup Date: Fri, 10 Jul 2026 15:39:41 +0800 Subject: [PATCH 13/31] fix(frontend): probe LCA with common columns --- pkg/frontend/data_branch_hashdiff.go | 131 ++++++++++++++++++---- pkg/frontend/data_branch_hashdiff_test.go | 18 +++ 2 files changed, 126 insertions(+), 23 deletions(-) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index a58045ac62649..5e9e94d05d5e9 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -43,6 +43,47 @@ import ( "go.uber.org/zap" ) +type lcaProbeLayout struct { + attrs []string + types []types.Type + targetIdxes []int +} + +// lcaProbeColumnLayout selects only columns that exist in both the LCA and +// target layouts. The caller reconstructs omitted target-only columns as NULL, +// because they cannot exist in an ancestor snapshot. +func lcaProbeColumnLayout(lcaDef *plan2.TableDef, targetColNames []string, targetColTypes []types.Type) lcaProbeLayout { + if len(lcaDef.Cols) == 0 { + layout := lcaProbeLayout{ + attrs: append([]string(nil), targetColNames...), + types: append([]types.Type(nil), targetColTypes...), + } + for i := range targetColNames { + layout.targetIdxes = append(layout.targetIdxes, i) + } + return layout + } + targetByName := make(map[string]int, len(targetColNames)) + for idx, name := range targetColNames { + targetByName[strings.ToLower(name)] = idx + } + + layout := lcaProbeLayout{} + for _, col := range lcaDef.Cols { + if col.Name == catalog.Row_ID { + continue + } + targetIdx, ok := targetByName[strings.ToLower(col.Name)] + if !ok { + continue + } + layout.attrs = append(layout.attrs, col.Name) + layout.types = append(layout.types, types.New(types.T(col.Typ.Id), col.Typ.Width, col.Typ.Scale)) + layout.targetIdxes = append(layout.targetIdxes, targetIdx) + } + return layout +} + // should read the LCA table to get all column values. func handleDelsOnLCA( ctx context.Context, @@ -67,6 +108,7 @@ func handleDelsOnLCA( expandedPKColIdxes = tblStuff.def.pkColIdxes snapshotTS = types.TimestampToTS(snapshot) ) + lcaLayout := lcaProbeColumnLayout(lcaTblDef, tblStuff.def.colNames, tblStuff.def.colTypes) forceReaderProbe := tblStuff.lcaReaderProbeMode != nil && tblStuff.lcaReaderProbeMode.Load() if forceReaderProbe { @@ -144,9 +186,9 @@ func handleDelsOnLCA( } } - selectCols := make([]string, len(tblStuff.def.colNames)+1) + selectCols := make([]string, len(lcaLayout.attrs)+1) selectCols[0] = "pks.__idx_" - for i, colName := range tblStuff.def.colNames { + for i, colName := range lcaLayout.attrs { selectCols[i+1] = fmt.Sprintf("lca.%s", quoteIdentifierForSQL(colName)) } @@ -277,16 +319,32 @@ func handleDelsOnLCA( lcaHitRows++ for j := 1; j < len(cols); j++ { - if err = dBat.Vecs[j-1].UnionOne(cols[j], int64(i), ses.proc.Mp()); err != nil { + targetIdx := lcaLayout.targetIdxes[j-1] + if err = dBat.Vecs[targetIdx].UnionOne(cols[j], int64(i), ses.proc.Mp()); err != nil { return false } } } - dBat.SetRowCount(dBat.Vecs[0].Length()) return true }) + selectedTargetIdxes := make(map[int]struct{}, len(lcaLayout.targetIdxes)) + for _, targetIdx := range lcaLayout.targetIdxes { + selectedTargetIdxes[targetIdx] = struct{}{} + } + for targetIdx, typ := range tblStuff.def.colTypes { + if _, selected := selectedTargetIdxes[targetIdx]; selected { + continue + } + nullVec := vector.NewConstNull(typ, dBat.Vecs[0].Length(), ses.proc.Mp()) + if err = dBat.Vecs[targetIdx].UnionBatch(nullVec, 0, dBat.Vecs[0].Length(), nil, ses.proc.Mp()); err != nil { + nullVec.Free(ses.proc.Mp()) + return nil, err + } + nullVec.Free(ses.proc.Mp()) + } + dBat.SetRowCount(dBat.Vecs[0].Length()) sqlRet.Close() logutil.Debug( @@ -394,6 +452,16 @@ func runLCAProbeWithReaderFallback( inputKeys[string(pkVec.GetRawBytesAt(i))] = struct{}{} } lcaTblDef := tblStuff.lcaRel.GetTableDef(ctx) + lcaLayout := lcaProbeColumnLayout(lcaTblDef, tblStuff.def.colNames, tblStuff.def.colTypes) + lcaPKIdx := slices.IndexFunc(lcaLayout.attrs, func(name string) bool { + return strings.EqualFold(name, lcaTblDef.Pkey.PkeyColName) + }) + if lcaPKIdx < 0 { + return executor.Result{}, moerr.NewInternalErrorNoCtxf( + "data branch: LCA primary key column %q is absent from probe layout", + lcaTblDef.Pkey.PkeyColName, + ) + } // Build a sorted IN vector for reader-side PK filtering. // The sorted-search path uses binary search over the IN value array and // assumes the array is ordered; an unsorted IN vector can drop valid hits. @@ -406,8 +474,8 @@ func runLCAProbeWithReaderFallback( pkFilterExpr := readutil.ConstructInExpr(ctx, lcaTblDef.Pkey.PkeyColName, filterVec) prepareCost = time.Since(start) - tmp := batch.NewWithSize(len(tblStuff.def.colNames)) - for i, typ := range tblStuff.def.colTypes { + tmp := batch.NewWithSize(len(lcaLayout.attrs)) + for i, typ := range lcaLayout.types { tmp.Vecs[i] = vector.NewVec(typ) } defer tmp.Clean(mp) @@ -462,8 +530,8 @@ func runLCAProbeWithReaderFallback( ses, tblStuff.lcaRel.GetTableID(ctx), snapshotTS, - tblStuff.def.colNames, - tblStuff.def.colTypes, + lcaLayout.attrs, + lcaLayout.types, pkFilterExpr, 0, func(readBatch *batch.Batch) error { @@ -475,7 +543,7 @@ func runLCAProbeWithReaderFallback( }() scannedBatchCnt++ scannedRowCnt += readBatch.RowCount() - readPK := readBatch.Vecs[tblStuff.def.pkColIdx] + readPK := readBatch.Vecs[lcaPKIdx] sels := make([]int64, 0, readBatch.RowCount()) keys := make([]string, 0, readBatch.RowCount()) filterStart := time.Now() @@ -497,7 +565,7 @@ func runLCAProbeWithReaderFallback( } base := tmp.Vecs[0].Length() unionStart := time.Now() - for colIdx := range tblStuff.def.colNames { + for colIdx := range lcaLayout.attrs { if err = tmp.Vecs[colIdx].Union(readBatch.Vecs[colIdx], sels, mp); err != nil { return err } @@ -555,6 +623,13 @@ func runLCAProbeWithReaderFallback( return executor.Result{}, err } + sourceIdxByTarget := make([]int, len(tblStuff.def.colNames)) + for i := range sourceIdxByTarget { + sourceIdxByTarget[i] = -1 + } + for sourceIdx, targetIdx := range lcaLayout.targetIdxes { + sourceIdxByTarget[targetIdx] = sourceIdx + } if !hasAnyHit { for colIdx := range tblStuff.def.colNames { nullConst := vector.NewConstNull(tblStuff.def.colTypes[colIdx], rowCount, mp) @@ -565,18 +640,36 @@ func runLCAProbeWithReaderFallback( } } } else if len(missedRows) == 0 { - for colIdx := range tblStuff.def.colNames { - if err = out.Vecs[colIdx+1].UnionBatch(tmp.Vecs[colIdx], 0, rowCount, nil, mp); err != nil { + for targetIdx, sourceIdx := range sourceIdxByTarget { + if sourceIdx < 0 { + nullConst := vector.NewConstNull(tblStuff.def.colTypes[targetIdx], rowCount, mp) + err = out.Vecs[targetIdx+1].UnionBatch(nullConst, 0, rowCount, nil, mp) + nullConst.Free(mp) + if err != nil { + return executor.Result{}, err + } + continue + } + if err = out.Vecs[targetIdx+1].UnionBatch(tmp.Vecs[sourceIdx], 0, rowCount, nil, mp); err != nil { return executor.Result{}, err } } } else { - for colIdx := range tblStuff.def.colNames { - if err = out.Vecs[colIdx+1].Union(tmp.Vecs[colIdx], sels, mp); err != nil { + for targetIdx, sourceIdx := range sourceIdxByTarget { + if sourceIdx < 0 { + nullConst := vector.NewConstNull(tblStuff.def.colTypes[targetIdx], rowCount, mp) + err = out.Vecs[targetIdx+1].UnionBatch(nullConst, 0, rowCount, nil, mp) + nullConst.Free(mp) + if err != nil { + return executor.Result{}, err + } + continue + } + if err = out.Vecs[targetIdx+1].Union(tmp.Vecs[sourceIdx], sels, mp); err != nil { return executor.Result{}, err } for _, rowIdx := range missedRows { - nulls.Add(out.Vecs[colIdx+1].GetNulls(), rowIdx) + nulls.Add(out.Vecs[targetIdx+1].GetNulls(), rowIdx) } } } @@ -1540,14 +1633,6 @@ func getTupleColumnValue(tuple types.Tuple, tblStuff tableStuff, colIdx int) (an } } -func visibleTupleKeyIdxes(tblStuff tableStuff) []int { - idxes := make([]int, len(tblStuff.def.visibleIdxes)) - for i, colIdx := range tblStuff.def.visibleIdxes { - idxes[i] = colIdx + 1 - } - return idxes -} - func fakePKTupleKeyIdxes(tblStuff tableStuff) []int { keyColIdxes := tblStuff.def.commonVisibleIdxes if len(keyColIdxes) == 0 { diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index f1a531804de16..ee5764a9b0b40 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -1022,6 +1022,24 @@ func newTestBranchTableStuff(ctrl *gomock.Controller) tableStuff { return tblStuff } +func TestLCAProbeColumnLayoutExcludesTargetOnlyColumns(t *testing.T) { + lcaDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "id", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "name", Typ: plan.Type{Id: int32(types.T_varchar)}}, + {Name: catalog.Row_ID, Typ: plan.Type{Id: int32(types.T_Rowid)}}, + }} + + layout := lcaProbeColumnLayout( + lcaDef, + []string{"id", "name", "added"}, + []types.Type{types.T_int64.ToType(), types.T_varchar.ToType(), types.T_int64.ToType()}, + ) + + require.Equal(t, []string{"id", "name"}, layout.attrs) + require.Equal(t, []int{0, 1}, layout.targetIdxes) + require.Equal(t, []types.T{types.T_int64, types.T_varchar}, []types.T{layout.types[0].Oid, layout.types[1].Oid}) +} + func makeTestBranchTableStuffFakePK(tblStuff *tableStuff) { tblStuff.def.pkKind = fakeKind tblStuff.def.pkColIdxes = []int{0, 1} From 8a998b7d397255f4a997e026c8d1b55546239bcd Mon Sep 17 00:00:00 2001 From: ULookup Date: Fri, 10 Jul 2026 18:42:39 +0800 Subject: [PATCH 14/31] fix(frontend): preserve data branch changes across alter generations --- pkg/frontend/clone.go | 150 ++++++- pkg/frontend/clone_test.go | 14 + pkg/frontend/data_branch.go | 276 ++++++++++--- pkg/frontend/data_branch_hashdiff.go | 371 ++++++++++++++++-- pkg/frontend/data_branch_hashdiff_test.go | 248 +++++++++++- pkg/frontend/data_branch_privilege.go | 4 +- pkg/frontend/data_branch_snapshot_test.go | 14 + pkg/frontend/data_branch_test.go | 103 ++++- pkg/frontend/data_branch_types.go | 47 ++- pkg/frontend/databranchutils/branch_dag.go | 40 ++ .../databranchutils/branch_dag_test.go | 20 + .../branch_protect_snapshot.go | 48 +++ pkg/frontend/feature_limit.go | 15 +- pkg/sql/compile/alter.go | 351 ++++++++++++++++- pkg/sql/compile/alter_test.go | 82 ++++ .../branch/diff/diff_schema_evolve.result | 145 ++++++- .../branch/diff/diff_schema_evolve.sql | 168 +++++++- .../branch/edge/branch_edge_cases.result | 6 +- .../branch/edge/branch_edge_cases.sql | 2 + .../branch/merge/merge_schema_evolve.result | 22 +- .../branch/merge/merge_schema_evolve.sql | 22 +- .../cases/git4data/branch/pick/pick_9.result | 68 ++++ .../cases/git4data/branch/pick/pick_9.sql | 96 +++++ 23 files changed, 2198 insertions(+), 114 deletions(-) diff --git a/pkg/frontend/clone.go b/pkg/frontend/clone.go index 55b6271c89d3a..14d18f26b7764 100644 --- a/pkg/frontend/clone.go +++ b/pkg/frontend/clone.go @@ -17,21 +17,29 @@ package frontend import ( "context" "fmt" + "sort" "strconv" "strings" "time" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/lock" plan2 "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/pb/timestamp" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/lockop" "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/plan" + "github.com/matrixorigin/matrixone/pkg/sql/plan/function" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vm/engine" + "github.com/matrixorigin/matrixone/pkg/vm/process" ) const ( @@ -70,6 +78,129 @@ type cloneReceipt struct { srcAccountName string } +type dataBranchCloneLockCtxKey struct{} + +func lockDataBranchCloneSource( + ctx context.Context, + ses *Session, + fromAccountID uint32, + databaseName, tableName string, +) error { + if locked, _ := ctx.Value(dataBranchCloneLockCtxKey{}).(bool); !locked { + return nil + } + txnOp := ses.proc.GetTxnOperator() + if !txnOp.Txn().IsPessimistic() { + // LockRows is intentionally a no-op for optimistic transactions. + // Keep DATA BRANCH available in that mode; ALTER records the lineage + // from its statement snapshot, and a missing edge caused by a true + // concurrent copy-and-swap is detected by the legacy-lineage guard + // before DIFF/MERGE can return an incorrect result. + return nil + } + sourceCtx := defines.AttachAccountId(ctx, fromAccountID) + eng := ses.proc.GetSessionInfo().StorageEngine + db, err := eng.Database(sourceCtx, catalog.MO_CATALOG, txnOp) + if err != nil { + return err + } + rel, err := db.Relation(sourceCtx, catalog.MO_TABLES, nil) + if err != nil { + return err + } + lockBat, err := dataBranchCloneCatalogLockBatch( + ses.proc, fromAccountID, databaseName, tableName, + ) + if err != nil { + return err + } + defer lockBat.Vecs[0].Free(ses.proc.Mp()) + // ALTER locks this exact mo_tables composite key exclusively. A shared + // catalog-row lock serializes source-ID/snapshot selection with ALTER while + // allowing source-table DML and sibling branch clones to continue. + return lockop.LockRows( + eng, + ses.proc, + rel, + rel.GetTableID(sourceCtx), + lockBat, + 0, + *lockBat.Vecs[0].GetType(), + lock.LockMode_Shared, + lock.Sharding_None, + fromAccountID, + ) +} + +func dataBranchCloneCatalogLockBatch( + proc *process.Process, + accountID uint32, + databaseName, tableName string, +) (*batch.Batch, error) { + inputs := make([]*vector.Vector, 3) + defer func() { + for _, input := range inputs { + if input != nil { + input.Free(proc.GetMPool()) + } + } + }() + inputs[0] = vector.NewVec(types.T_uint32.ToType()) + if err := vector.AppendFixed(inputs[0], accountID, false, proc.GetMPool()); err != nil { + return nil, err + } + for i, name := range []string{databaseName, tableName} { + inputs[i+1] = vector.NewVec(types.T_varchar.ToType()) + if err := vector.AppendBytes(inputs[i+1], []byte(name), false, proc.GetMPool()); err != nil { + return nil, err + } + } + encoded, err := function.RunFunctionDirectly( + proc, function.SerialFunctionEncodeID, inputs, 1, + ) + if err != nil { + return nil, err + } + bat := batch.NewWithSize(1) + bat.SetVector(0, encoded) + return bat, nil +} + +func lockDataBranchCloneDatabaseSources( + ctx context.Context, + ses *Session, + source cloneDatabaseSource, +) error { + if locked, _ := ctx.Value(dataBranchCloneLockCtxKey{}).(bool); !locked { + return nil + } + if source.snapshot != nil && source.snapshot.TS != nil { + return nil + } + fromAccountID := source.opAccountId + if source.snapshot != nil && source.snapshot.Tenant != nil { + fromAccountID = source.snapshot.Tenant.TenantID + } + tables := append([]*tableInfo(nil), source.srcTblInfos...) + sort.Slice(tables, func(i, j int) bool { + if tables[i].dbName != tables[j].dbName { + return tables[i].dbName < tables[j].dbName + } + return tables[i].tblName < tables[j].tblName + }) + for _, table := range tables { + if table.typ == view { + continue + } + if err := lockDataBranchCloneSource( + ctx, ses, fromAccountID, table.dbName, table.tblName, + ); err != nil { + return err + } + } + return nil +} + func getBackExecutor( ctx context.Context, ses *Session, @@ -317,6 +448,17 @@ func handleCloneTable( err = moerr.NewInternalErrorNoCtxf("only sys can clone table to another account") return } + if snapshot == nil || snapshot.TS == nil { + if err = lockDataBranchCloneSource( + reqCtx, + ses, + fromAccountId, + stmt.SrcTable.SchemaName.String(), + stmt.SrcTable.ObjectName.String(), + ); err != nil { + return + } + } ctx = defines.AttachAccountId(reqCtx, toAccountId) @@ -441,6 +583,9 @@ func handleCloneDatabaseWithSource( return } } + if err = lockDataBranchCloneDatabaseSources(reqCtx, ses, source); err != nil { + return + } ctx1 = defines.AttachAccountId(reqCtx, source.toAccountId) if err = bh.Exec(ctx1, @@ -675,7 +820,10 @@ func updateBranchMetaTable( tcc.SetContext(srcCtx) defer tcc.SetContext(origCtx) - if _, srcTblDef, err = tcc.Resolve(receipt.srcDb, receipt.srcTbl, nil); err != nil { + // The metadata parent must be the physical generation that supplied the + // clone data. For snapshot clones that can differ from the table currently + // reachable by name after one or more copy-and-swap ALTERs. + if _, srcTblDef, err = tcc.Resolve(receipt.srcDb, receipt.srcTbl, receipt.snapshot); err != nil { return err } if srcTblDef == nil { diff --git a/pkg/frontend/clone_test.go b/pkg/frontend/clone_test.go index 72ecd874b033a..541251c97c2f4 100644 --- a/pkg/frontend/clone_test.go +++ b/pkg/frontend/clone_test.go @@ -23,6 +23,20 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) +func TestDataBranchCloneCatalogLockBatch(t *testing.T) { + ses := newValidateSession(t) + mp := ses.proc.Mp() + baseline := mp.CurrNB() + + bat, err := dataBranchCloneCatalogLockBatch(ses.proc, 7, "db", "tbl") + require.NoError(t, err) + require.Len(t, bat.Vecs, 1) + require.Equal(t, 1, bat.Vecs[0].Length()) + require.NotEmpty(t, bat.Vecs[0].GetBytesAt(0)) + bat.Vecs[0].Free(mp) + require.Equal(t, baseline, mp.CurrNB()) +} + func Test_prepareCloneViewSnapshot(t *testing.T) { original := &plan.Snapshot{ Tenant: &plan.SnapshotTenant{TenantID: 1001}, diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index a5ded978c5468..ad874dc13c66a 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -363,6 +363,7 @@ func dataBranchCreateTable( }() execCtx.reqCtx = context.WithValue(execCtx.reqCtx, tree.CloneLevelCtxKey{}, tree.NormalCloneLevelTable) + execCtx.reqCtx = context.WithValue(execCtx.reqCtx, dataBranchCloneLockCtxKey{}, true) if receipt, err = handleCloneTable(execCtx, ses, cloneStmt, bh); err != nil { return @@ -406,6 +407,7 @@ func dataBranchCreateDatabase( execCtx.reqCtx = context.WithValue( execCtx.reqCtx, tree.CloneLevelCtxKey{}, tree.NormalCloneLevelDatabase, ) + execCtx.reqCtx = context.WithValue(execCtx.reqCtx, dataBranchCloneLockCtxKey{}, true) if !skipDataBranchPrivilegeCheck(ses) { if authStats, err = authenticateDataBranchCreateDatabase(execCtx.reqCtx, ses, stmt); err != nil { @@ -1128,7 +1130,53 @@ func diffOnBase( return } +func dataBranchPrimaryKeyColumns(tblDef *plan.TableDef) (kind int, names []string) { + if tblDef == nil || tblDef.Pkey == nil { + return -1, nil + } + if tblDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { + return fakeKind, []string{catalog.FakePrimaryKeyColName} + } + if tblDef.Pkey.CompPkeyCol != nil { + return compositeKind, tblDef.Pkey.Names + } + return normalKind, []string{tblDef.Pkey.PkeyColName} +} + +func checkDataBranchPrimaryKeyCompatibility(tarDef, baseDef *plan.TableDef) error { + tarKind, tarNames := dataBranchPrimaryKeyColumns(tarDef) + baseKind, baseNames := dataBranchPrimaryKeyColumns(baseDef) + compatible := tarKind == baseKind && len(tarNames) == len(baseNames) + if compatible { + for i := range tarNames { + if !strings.EqualFold(tarNames[i], baseNames[i]) { + compatible = false + break + } + } + } + if compatible { + return nil + } + return moerr.NewInternalErrorNoCtxf( + "schema compatibility check: target primary key columns (%s) do not match base primary key columns (%s)", + strings.Join(tarNames, ","), strings.Join(baseNames, ","), + ) +} + +func dataBranchColumnTypeAttributesEqual(left, right plan.Type) bool { + return left.Width == right.Width && + left.Scale == right.Scale && + left.Enumvalues == right.Enumvalues && + left.AutoIncr == right.AutoIncr && + left.NotNullable == right.NotNullable +} + func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, commonVisibleIdxes, tarOnlyIdxes []int, err error) { + if err = checkDataBranchPrimaryKeyCompatibility(tarDef, baseDef); err != nil { + return + } + baseColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) baseVisibleColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) for _, col := range baseDef.Cols { @@ -1158,6 +1206,20 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm baseCol, found := baseColMap[name] if found { if baseCol.Typ.Id == tarCol.Typ.Id { + if !dataBranchColumnTypeAttributesEqual(baseCol.Typ, tarCol.Typ) { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: column '%s' has different type attributes", + tarCol.Name, + ) + return + } + if baseCol.NotNull != tarCol.NotNull { + err = moerr.NewInternalErrorNoCtxf( + "schema compatibility check: column '%s' has different nullability", + tarCol.Name, + ) + return + } commonIdxes = append(commonIdxes, dataIdx) if isDataBranchUserVisibleColumn(tarCol) { commonVisibleIdxes = append(commonVisibleIdxes, dataIdx) @@ -1178,6 +1240,12 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm } dataIdx++ } + if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName && len(tarOnlyIdxes) > 0 { + err = moerr.NewInternalErrorNoCtx( + "schema compatibility check: target-only columns require an explicit primary key", + ) + return + } if baseDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { // For fake PK tables, the PK column is generated from all columns. // No specific PK name to verify. @@ -1432,6 +1500,7 @@ func constructChangeHandle( ); err != nil { return } + tarSnapshot, baseSnapshot := tables.resolvedSnapshots(ses) // BETWEEN SNAPSHOT: intersect the target collect range with [betweenFrom.Next(), betweenTo]. // This narrows the time window on the source side so only changes within the @@ -1461,6 +1530,16 @@ func constructChangeHandle( tarRange.end = tarRange.end[:j] tarRange.rel = tarRange.rel[:j] } + tarHydrationRel := tables.tarRel + tarHydrationSnapshot := tarSnapshot + if betweenTo != nil && betweenTo.LT(&tarHydrationSnapshot) && len(tarRange.rel) > 0 { + // PICK BETWEEN observes the target as of betweenTo, not as of the + // statement's later endpoint snapshot. The last surviving range is the + // physical generation that was active at that boundary. + tarHydrationRel = tarRange.rel[len(tarRange.rel)-1] + tarHydrationSnapshot = *betweenTo + } + targetDef := tables.tarRel.GetTableDef(ctx) // collectFn dispatches to the PK-filtered or plain CollectChanges variant. collectFn := func( @@ -1479,6 +1558,15 @@ func constructChangeHandle( for i := range tarRange.rel { collectStart := time.Now() + var sourceMapping []int + isHistorical := tarRange.rel[i].GetTableID(ctx) != tables.tarRel.GetTableID(ctx) + if isHistorical { + if sourceMapping, err = dataBranchSourceColToTargetIdx( + tarRange.rel[i].GetTableDef(ctx), targetDef, tables.def.colNames, + ); err != nil { + return + } + } if handle, err = collectFn( ctx, tarRange.rel[i], @@ -1498,12 +1586,31 @@ func constructChangeHandle( ) if handle != nil { + if isHistorical { + handle = &historicalDataBranchChangesHandle{ + inner: handle, + sourceMapping: sourceMapping, + tblStuff: tables, + ses: ses, + endpointRel: tarHydrationRel, + endpointSnapshot: tarHydrationSnapshot, + } + } tarHandle = append(tarHandle, handle) } } for i := range baseRange.rel { collectStart := time.Now() + var sourceMapping []int + isHistorical := baseRange.rel[i].GetTableID(ctx) != tables.baseRel.GetTableID(ctx) + if isHistorical { + if sourceMapping, err = dataBranchSourceColToTargetIdx( + baseRange.rel[i].GetTableDef(ctx), targetDef, tables.def.colNames, + ); err != nil { + return + } + } if handle, err = collectFn( ctx, baseRange.rel[i], @@ -1523,6 +1630,16 @@ func constructChangeHandle( ) if handle != nil { + if isHistorical { + handle = &historicalDataBranchChangesHandle{ + inner: handle, + sourceMapping: sourceMapping, + tblStuff: tables, + ses: ses, + endpointRel: tables.baseRel, + endpointSnapshot: baseSnapshot, + } + } baseHandle = append(baseHandle, handle) } } @@ -1635,15 +1752,19 @@ func decideCollectRange( // ancestor to anchor the comparison, so the DAG model does // not apply either. if dagInfo.lcaTableId == 0 { - tarCollectRange = collectRange{ - from: []types.TS{types.MinTs()}, - end: []types.TS{tarSp}, - rel: []engine.Relation{tables.tarRel}, + if tarCollectRange, err = buildSideCollectRange( + ctx, ses, bh, tables, tarSp, tarCTS, + dagInfo.pathFromLCAToTar, dagInfo.pathFromLCAToTarTS, + nil, nil, types.TS{}, true, + ); err != nil { + return } - baseCollectRange = collectRange{ - from: []types.TS{types.MinTs()}, - end: []types.TS{baseSp}, - rel: []engine.Relation{tables.baseRel}, + if baseCollectRange, err = buildSideCollectRange( + ctx, ses, bh, tables, baseSp, baseCTS, + dagInfo.pathFromLCAToBase, dagInfo.pathFromLCAToBaseTS, + nil, nil, types.TS{}, true, + ); err != nil { + return } return } @@ -1737,13 +1858,13 @@ func decideCollectRange( if tarCollectRange, err = buildSideCollectRange( ctx, ses, bh, tables, tarSp, tarCTS, - tarPath, tarPathTS, basePathTS, baseSp, + tarPath, tarPathTS, basePathTS, dagInfo.pathFromLCAToBaseLineageOnly, baseSp, false, ); err != nil { return } if baseCollectRange, err = buildSideCollectRange( ctx, ses, bh, tables, baseSp, baseCTS, - basePath, basePathTS, tarPathTS, tarSp, + basePath, basePathTS, tarPathTS, dagInfo.pathFromLCAToTarLineageOnly, tarSp, false, ); err != nil { return } @@ -1807,7 +1928,9 @@ func buildSideCollectRange( selfPath []uint64, selfPathTS []types.TS, otherPathTS []types.TS, + otherPathLineageOnly []bool, otherSP types.TS, + wholeHistory bool, ) (cr collectRange, err error) { endpointID := selfPath[len(selfPath)-1] @@ -1819,19 +1942,33 @@ func buildSideCollectRange( nodeCTS types.TS ) - // Resolve the relation handle for this node. - switch { - case nodeID == tables.tarRel.GetTableID(ctx): + // Resolve the segment boundary before opening the relation. ALTER + // generations are dropped after their replacement is materialized, so + // an intermediate physical table must be opened at the transition/fork + // snapshot where it was still visible, not at the endpoint snapshot. + if i == len(selfPath)-1 { + windowEnd = endpointSP + } else { + windowEnd = selfPathTS[i+1] + } + relationSnapshot := dataBranchCollectRelationSnapshot( + endpointSP, windowEnd, i == len(selfPath)-1, + ) + + // Reuse an endpoint handle only for the endpoint segment. A physical + // generation that is the opposite side's endpoint can still be an + // intermediate segment on this side, whose window extends to a later + // ALTER transition and therefore requires a relation opened there. + if i == len(selfPath)-1 && nodeID == tables.tarRel.GetTableID(ctx) { rel = tables.tarRel - case nodeID == tables.baseRel.GetTableID(ctx): + } else if i == len(selfPath)-1 && nodeID == tables.baseRel.GetTableID(ctx) { rel = tables.baseRel - case tables.lcaRel != nil && nodeID == tables.lcaRel.GetTableID(ctx): - rel = tables.lcaRel - default: + } else { + snapshot := relationSnapshot.ToTimestamp() if rel, err = getRelationById( ctx, ses, bh, nodeID, &plan2.Snapshot{ Tenant: &plan.SnapshotTenant{TenantID: ses.GetAccountId()}, - TS: ×tamp.Timestamp{PhysicalTime: endpointSP.Physical()}, + TS: &snapshot, }, ); err != nil { return @@ -1846,7 +1983,7 @@ func buildSideCollectRange( } else { ctsList, err2 := getTablesCreationCommitTS( ctx, ses, rel, rel, - []types.TS{endpointSP, endpointSP}, + []types.TS{relationSnapshot, relationSnapshot}, ) if err2 != nil { err = err2 @@ -1860,23 +1997,18 @@ func buildSideCollectRange( nodeCTS = ctsList[0] } - // Window upper bound. - if i == len(selfPath)-1 { - windowEnd = endpointSP + // Window lower bound. + if wholeHistory && i == 0 { + windowFrom = types.MinTs() } else { - windowEnd = selfPathTS[i+1] + windowFrom = nodeCTS.Next() } - // Window lower bound. - windowFrom = nodeCTS.Next() - // LCA prune (i == 0): clamp to skip the prefix that the // other side inherits identically via clone. - if i == 0 { - var otherFork types.TS - if len(otherPathTS) > 1 { - otherFork = otherPathTS[1] - } else { + if i == 0 && !wholeHistory { + otherFork, hasFork := firstDataBranchForkTS(otherPathTS, otherPathLineageOnly) + if !hasFork { // Other endpoint IS the LCA; observation goes up to // otherSP. otherFork = otherSP @@ -1897,6 +2029,13 @@ func buildSideCollectRange( return } +func dataBranchCollectRelationSnapshot(endpointSP, windowEnd types.TS, endpoint bool) types.TS { + if endpoint { + return endpointSP + } + return windowEnd +} + // getTablesCreationCommitTS resolves the creation commit timestamp for // both tar and base tables. It tries CollectChanges first (fast, // preserves real commit_ts from partition state), falling back to the @@ -2447,8 +2586,14 @@ func decideLCABranchTSFromBranchDAG( return } - tarLineageID := dataBranchLineageTableID(ctx, dag, tblStuff.tarRel) - baseLineageID := dataBranchLineageTableID(ctx, dag, tblStuff.baseRel) + tarLineageID, tarLegacyAlias := dataBranchLineageTableID(ctx, dag, tblStuff.tarRel) + baseLineageID, baseLegacyAlias := dataBranchLineageTableID(ctx, dag, tblStuff.baseRel) + if tarLegacyAlias || baseLegacyAlias { + err = moerr.NewInternalErrorNoCtx( + "data branch: schema-evolution lineage metadata is missing for a legacy ALTER; recreate the branch before DIFF/MERGE", + ) + return + } if lcaTableID, _, _, hasLca := dag.FindLCA(tarLineageID, baseLineageID); hasLca { tarIDs, tarTSs, ok := dag.PathFromAncestor(tarLineageID, lcaTableID) @@ -2465,20 +2610,43 @@ func decideLCABranchTSFromBranchDAG( lcaTableID, baseLineageID) return } - if tarLineageID != tarTableID { - tarIDs[len(tarIDs)-1] = tarTableID - } - if baseLineageID != baseTableID { - baseIDs[len(baseIDs)-1] = baseTableID - } branchInfo.lcaTableId = lcaTableID branchInfo.pathFromLCAToTar = tarIDs branchInfo.pathFromLCAToTarTS = buildTSs(tarTSs) + branchInfo.pathFromLCAToTarLineageOnly = dataBranchPathLineageOnly(dag, tarIDs) branchInfo.pathFromLCAToBase = baseIDs branchInfo.pathFromLCAToBaseTS = buildTSs(baseTSs) + branchInfo.pathFromLCAToBaseLineageOnly = dataBranchPathLineageOnly(dag, baseIDs) return } + wholePath := func(lineageID, endpointID uint64) ([]uint64, []types.TS, []bool, error) { + ids, cloneTSs, ok := dag.PathFromRoot(lineageID) + if !ok { + return []uint64{endpointID}, []types.TS{{}}, []bool{false}, nil + } + if len(ids) == 0 || ids[len(ids)-1] != endpointID { + return nil, nil, nil, moerr.NewInternalErrorNoCtxf( + "data branch: lineage endpoint %d does not match table %d", lineageID, endpointID, + ) + } + return ids, buildTSs(cloneTSs), dataBranchPathLineageOnly(dag, ids), nil + } + if tarTableID != baseTableID { + if branchInfo.pathFromLCAToTar, + branchInfo.pathFromLCAToTarTS, + branchInfo.pathFromLCAToTarLineageOnly, + err = wholePath(tarLineageID, tarTableID); err != nil { + return + } + if branchInfo.pathFromLCAToBase, + branchInfo.pathFromLCAToBaseTS, + branchInfo.pathFromLCAToBaseLineageOnly, + err = wholePath(baseLineageID, baseTableID); err != nil { + return + } + } + // No DAG-derived LCA, but if tar and base resolve to the same // table id we still need to feed downstream tombstone resolution // with a probe snapshot. Synthetic single-node paths give the @@ -2487,21 +2655,35 @@ func decideLCABranchTSFromBranchDAG( branchInfo.lcaTableId = tarTableID branchInfo.pathFromLCAToTar = []uint64{tarTableID} branchInfo.pathFromLCAToTarTS = []types.TS{{}} + branchInfo.pathFromLCAToTarLineageOnly = []bool{false} branchInfo.pathFromLCAToBase = []uint64{baseTableID} branchInfo.pathFromLCAToBaseTS = []types.TS{{}} + branchInfo.pathFromLCAToBaseLineageOnly = []bool{false} } return } -func dataBranchLineageTableID(ctx context.Context, dag *databranchutils.DataBranchDAG, rel engine.Relation) uint64 { - tableID := rel.GetTableID(ctx) +func dataBranchPathLineageOnly(dag *databranchutils.DataBranchDAG, path []uint64) []bool { + ret := make([]bool, len(path)) + for i := 1; i < len(path); i++ { + ret[i] = dag.IsLineageOnly(path[i]) + } + return ret +} + +func dataBranchLineageTableID( + ctx context.Context, + dag *databranchutils.DataBranchDAG, + rel engine.Relation, +) (tableID uint64, legacyLogicalFallback bool) { + tableID = rel.GetTableID(ctx) if dag.Exists(tableID) { - return tableID + return tableID, false } if tblDef := rel.GetTableDef(ctx); tblDef != nil && tblDef.LogicalId != 0 && dag.Exists(tblDef.LogicalId) { - return tblDef.LogicalId + return tblDef.LogicalId, true } - return tableID + return tableID, false } // buildTSs lifts a slice of int64 physical timestamps into the @@ -2539,7 +2721,7 @@ func constructBranchDAG( if sqlRet, err = runSql( sysCtx, ses, bh, fmt.Sprintf( - "select table_id, clone_ts, p_table_id, table_deleted from %s.%s", + "select table_id, clone_ts, p_table_id, level, table_deleted from %s.%s", catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, ), nil, nil, @@ -2552,13 +2734,15 @@ func constructBranchDAG( tblIds := vector.MustFixedColNoTypeCheck[uint64](cols[0]) cloneTS := vector.MustFixedColNoTypeCheck[int64](cols[1]) pTblIds := vector.MustFixedColNoTypeCheck[uint64](cols[2]) - tableDeleted := vector.MustFixedColNoTypeCheck[bool](cols[3]) + levels := executor.GetStringRows(cols[3]) + tableDeleted := vector.MustFixedColNoTypeCheck[bool](cols[4]) for i := range tblIds { rowData = append(rowData, databranchutils.DataBranchMetadata{ TableID: tblIds[i], CloneTS: cloneTS[i], PTableID: pTblIds[i], TableDeleted: tableDeleted[i], + LineageOnly: databranchutils.IsAlterLineageLevel(levels[i]), }) } return true diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 5e9e94d05d5e9..bbeed9c26f23f 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -33,6 +33,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/frontend/databranchutils" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/objectio/mergeutil" "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" @@ -49,6 +50,34 @@ type lcaProbeLayout struct { targetIdxes []int } +func lcaProbeResultTargetIndexes( + layout lcaProbeLayout, + targetColumnCount int, + resultColumnCount int, + fullTargetLayout bool, +) ([]int, error) { + if fullTargetLayout { + if resultColumnCount != targetColumnCount { + return nil, moerr.NewInternalErrorNoCtxf( + "unexpected LCA probe result width %d for full target layout with %d columns", + resultColumnCount, targetColumnCount, + ) + } + idxes := make([]int, targetColumnCount) + for i := range idxes { + idxes[i] = i + } + return idxes, nil + } + if resultColumnCount != len(layout.targetIdxes) { + return nil, moerr.NewInternalErrorNoCtxf( + "unexpected LCA probe result width %d for projected layout with %d columns", + resultColumnCount, len(layout.targetIdxes), + ) + } + return append([]int(nil), layout.targetIdxes...), nil +} + // lcaProbeColumnLayout selects only columns that exist in both the LCA and // target layouts. The caller reconstructs omitted target-only columns as NULL, // because they cannot exist in an ancestor snapshot. @@ -99,7 +128,8 @@ func handleDelsOnLCA( } var ( - sqlRet executor.Result + sqlRet executor.Result + fullTargetLayout bool lcaTblDef = tblStuff.lcaRel.GetTableDef(ctx) baseTblDef = tblStuff.baseRel.GetTableDef(ctx) @@ -116,6 +146,7 @@ func handleDelsOnLCA( if err != nil { return nil, err } + fullTargetLayout = true } else { sqlBuf := acquireBuffer(tblStuff.bufPool) valsBuf := acquireBuffer(tblStuff.bufPool) @@ -267,6 +298,7 @@ func handleDelsOnLCA( ) return nil, err } + fullTargetLayout = true logutil.Info( "DataBranch-LCA-SQL-Fallback-Done", zap.Uint64("table-id", tblStuff.lcaRel.GetTableID(ctx)), @@ -279,6 +311,7 @@ func handleDelsOnLCA( } } } + defer sqlRet.Close() if forceReaderProbe { logutil.Debug( @@ -302,12 +335,28 @@ func handleDelsOnLCA( } dBat = tblStuff.retPool.acquireRetBatch(tblStuff, false) + defer func() { + if err != nil && dBat != nil { + tblStuff.retPool.releaseRetBatch(dBat, false) + dBat = nil + } + }() sels := make([]int64, 0, 100) joinedRows := 0 lcaHitRows := 0 lcaMissRows := 0 sqlRet.ReadRows(func(rowCnt int, cols []*vector.Vector) bool { + if len(cols) == 0 { + err = moerr.NewInternalErrorNoCtx("LCA probe returned a batch without index column") + return false + } + var resultTargetIdxes []int + if resultTargetIdxes, err = lcaProbeResultTargetIndexes( + lcaLayout, len(tblStuff.def.colNames), len(cols)-1, fullTargetLayout, + ); err != nil { + return false + } joinedRows += rowCnt for i := range rowCnt { if notExist(cols[1:], i) { @@ -317,36 +366,43 @@ func handleDelsOnLCA( continue } - lcaHitRows++ for j := 1; j < len(cols); j++ { - targetIdx := lcaLayout.targetIdxes[j-1] + targetIdx := resultTargetIdxes[j-1] if err = dBat.Vecs[targetIdx].UnionOne(cols[j], int64(i), ses.proc.Mp()); err != nil { return false } } + lcaHitRows++ } return true }) - selectedTargetIdxes := make(map[int]struct{}, len(lcaLayout.targetIdxes)) - for _, targetIdx := range lcaLayout.targetIdxes { - selectedTargetIdxes[targetIdx] = struct{}{} + if err != nil { + return nil, err + } + selectedTargetIdxes := make(map[int]struct{}, len(tblStuff.def.colNames)) + if fullTargetLayout { + for targetIdx := range tblStuff.def.colNames { + selectedTargetIdxes[targetIdx] = struct{}{} + } + } else { + for _, targetIdx := range lcaLayout.targetIdxes { + selectedTargetIdxes[targetIdx] = struct{}{} + } } for targetIdx, typ := range tblStuff.def.colTypes { if _, selected := selectedTargetIdxes[targetIdx]; selected { continue } - nullVec := vector.NewConstNull(typ, dBat.Vecs[0].Length(), ses.proc.Mp()) - if err = dBat.Vecs[targetIdx].UnionBatch(nullVec, 0, dBat.Vecs[0].Length(), nil, ses.proc.Mp()); err != nil { + nullVec := vector.NewConstNull(typ, lcaHitRows, ses.proc.Mp()) + if err = dBat.Vecs[targetIdx].UnionBatch(nullVec, 0, lcaHitRows, nil, ses.proc.Mp()); err != nil { nullVec.Free(ses.proc.Mp()) return nil, err } nullVec.Free(ses.proc.Mp()) } - dBat.SetRowCount(dBat.Vecs[0].Length()) - - sqlRet.Close() + dBat.SetRowCount(lcaHitRows) logutil.Debug( "DataBranch-LCA-Join-Result", zap.Uint64("table-id", tblStuff.lcaRel.GetTableID(ctx)), @@ -2274,7 +2330,9 @@ func buildHashmapForTable( return } - if dataBat != nil && dataBat.RowCount() > 0 && side == "base" && len(tblStuff.def.tarOnlyIdxes) > 0 { + if dataBat != nil && dataBat.RowCount() > 0 && side == "base" && + len(tblStuff.def.tarOnlyIdxes) > 0 && + !dataBranchBatchHasTargetLayout(dataBat, tblStuff) { projected := projectBaseBatchToTarget(dataBat, tblStuff, mp) dataBat = projected // projected will be cleaned in putVectors } @@ -2502,26 +2560,287 @@ func projectBaseBatchToTarget( tblStuff *tableStuff, mp *mpool.MPool, ) *batch.Batch { - out := batch.NewWithSize(len(tblStuff.def.colNames) + 1) - out.Vecs[0] = baseBat.Vecs[0] // RowID - baseBat.Vecs[0] = nil - baseColCount := baseBat.VectorCount() - 1 // subtract RowID - if baseColCount > len(tblStuff.def.baseColToTarIdx) { - baseColCount = len(tblStuff.def.baseColToTarIdx) - } - for baseColIdx := 0; baseColIdx < baseColCount; baseColIdx++ { - tarColIdx := tblStuff.def.baseColToTarIdx[baseColIdx] + return projectDataBranchBatchToTarget( + baseBat, tblStuff, tblStuff.def.baseColToTarIdx, mp, + ) +} + +func dataBranchSourceColToTargetIdx( + sourceDef, targetDef *plan2.TableDef, + targetColNames []string, +) ([]int, error) { + if sourceDef == nil || targetDef == nil { + return nil, moerr.NewInternalErrorNoCtx("missing schema for historical data branch projection") + } + if err := checkDataBranchPrimaryKeyCompatibility(targetDef, sourceDef); err != nil { + return nil, moerr.NewNotSupportedNoCtxf( + "historical data branch primary key is incompatible with the endpoint schema: %s", + err.Error(), + ) + } + if sourceDef.Pkey.PkeyColName == catalog.FakePrimaryKeyColName { + sourceNames := make([]string, 0, len(sourceDef.Cols)) + targetNames := make([]string, 0, len(targetDef.Cols)) + for _, col := range sourceDef.Cols { + if col.Name != catalog.Row_ID { + sourceNames = append(sourceNames, col.Name) + } + } + for _, col := range targetDef.Cols { + if col.Name != catalog.Row_ID { + targetNames = append(targetNames, col.Name) + } + } + if len(sourceNames) != len(targetNames) { + return nil, moerr.NewNotSupportedNoCtx( + "historical data branch fake primary key schema differs from the endpoint schema", + ) + } + for i := range sourceNames { + if !strings.EqualFold(sourceNames[i], targetNames[i]) { + return nil, moerr.NewNotSupportedNoCtx( + "historical data branch fake primary key schema differs from the endpoint schema", + ) + } + } + } + targetCols := make(map[string]*plan2.ColDef, len(targetDef.Cols)) + for _, col := range targetDef.Cols { + targetCols[strings.ToLower(col.Name)] = col + } + mapping := make([]int, 0, len(sourceDef.Cols)) + for _, col := range sourceDef.Cols { + if col.Name == catalog.Row_ID { + continue + } + targetIdx := dataBranchColumnIndexByName(targetColNames, col.Name) + mapping = append(mapping, targetIdx) + if targetIdx < 0 { + continue + } + targetCol, ok := targetCols[strings.ToLower(col.Name)] + if !ok || col.Typ.Id != targetCol.Typ.Id || + !dataBranchColumnTypeAttributesEqual(col.Typ, targetCol.Typ) { + return nil, moerr.NewNotSupportedNoCtxf( + "historical data branch column %s has a different type from the endpoint schema", + col.Name, + ) + } + } + return mapping, nil +} + +func dataBranchTargetLayoutAttrs(tblStuff *tableStuff, hasCommitTS bool) []string { + attrs := make([]string, 0, len(tblStuff.def.colNames)+2) + attrs = append(attrs, catalog.Row_ID) + attrs = append(attrs, tblStuff.def.colNames...) + if hasCommitTS { + attrs = append(attrs, objectio.DefaultCommitTS_Attr) + } + return attrs +} + +func dataBranchBatchHasTargetLayout(bat *batch.Batch, tblStuff *tableStuff) bool { + if bat == nil || len(bat.Attrs) != bat.VectorCount() { + return false + } + hasCommitTS := bat.VectorCount() == len(tblStuff.def.colNames)+2 + if !hasCommitTS && bat.VectorCount() != len(tblStuff.def.colNames)+1 { + return false + } + want := dataBranchTargetLayoutAttrs(tblStuff, hasCommitTS) + for i := range want { + if !strings.EqualFold(bat.Attrs[i], want[i]) { + return false + } + } + return true +} + +func overlayDataBranchProbeResult( + projected *batch.Batch, + probe executor.Result, + pkTargetIdx int, + mp *mpool.MPool, +) (err error) { + if projected == nil || projected.RowCount() == 0 { + return nil + } + prepared := false + probe.ReadRows(func(rowCount int, cols []*vector.Vector) bool { + if len(cols) < 2 || pkTargetIdx < 0 || pkTargetIdx+1 >= len(cols) { + err = moerr.NewInternalErrorNoCtx("invalid endpoint probe layout for historical data branch batch") + return false + } + targetColCount := len(cols) - 1 + if projected.VectorCount() < targetColCount+1 { + err = moerr.NewInternalErrorNoCtxf( + "historical data branch batch has %d vectors for %d target columns", + projected.VectorCount(), targetColCount, + ) + return false + } + if !prepared { + // Projection represents columns absent from the historical schema as + // constant-NULL vectors. Vector.Copy requires a writable flat + // destination, so materialize only those constant data columns before + // overlaying endpoint values. RowID and commit_ts stay untouched. + for targetIdx := 0; targetIdx < targetColCount; targetIdx++ { + dst := projected.Vecs[targetIdx+1] + if !dst.IsConst() { + continue + } + flat := vector.NewVec(*dst.GetType()) + if err = flat.UnionBatch(dst, 0, projected.RowCount(), nil, mp); err != nil { + flat.Free(mp) + return false + } + dst.Free(mp) + projected.Vecs[targetIdx+1] = flat + } + prepared = true + } + for row := 0; row < rowCount; row++ { + projectedRow := vector.GetFixedAtNoTypeCheck[int64](cols[0], row) + if projectedRow < 0 || projectedRow >= int64(projected.RowCount()) { + err = moerr.NewInternalErrorNoCtxf( + "endpoint probe row index %d out of range %d", + projectedRow, projected.RowCount(), + ) + return false + } + if cols[pkTargetIdx+1].IsNull(uint64(row)) { + continue + } + for targetIdx := 0; targetIdx < targetColCount; targetIdx++ { + if err = projected.Vecs[targetIdx+1].Copy( + cols[targetIdx+1], projectedRow, int64(row), mp, + ); err != nil { + return false + } + } + } + return true + }) + return err +} + +func hydrateHistoricalDataBranchBatch( + ctx context.Context, + ses *Session, + projected *batch.Batch, + tblStuff tableStuff, + endpointRel engine.Relation, + endpointSnapshot types.TS, +) (err error) { + if projected == nil || projected.RowCount() == 0 { + return nil + } + pkVecIdx := tblStuff.def.pkColIdx + 1 // projected data keeps RowID at Vec[0] + if pkVecIdx <= 0 || pkVecIdx >= projected.VectorCount() { + return moerr.NewInternalErrorNoCtxf( + "historical data branch PK index %d out of range", pkVecIdx, + ) + } + tBat := batch.NewWithSize(1) + tBat.Vecs[0] = vector.NewVec(*projected.Vecs[pkVecIdx].GetType()) + defer tBat.Clean(ses.proc.Mp()) + if err = tBat.Vecs[0].UnionBatch( + projected.Vecs[pkVecIdx], 0, projected.RowCount(), nil, ses.proc.Mp(), + ); err != nil { + return err + } + tBat.SetRowCount(projected.RowCount()) + + probeStuff := tblStuff + probeStuff.lcaRel = endpointRel + probe, err := runLCAProbeWithReaderFallback( + ctx, ses, tBat, probeStuff, endpointSnapshot, + ) + if err != nil { + return err + } + defer probe.Close() + return overlayDataBranchProbeResult( + projected, probe, tblStuff.def.pkColIdx, ses.proc.Mp(), + ) +} + +type historicalDataBranchChangesHandle struct { + inner engine.ChangesHandle + sourceMapping []int + tblStuff tableStuff + ses *Session + endpointRel engine.Relation + endpointSnapshot types.TS +} + +func (h *historicalDataBranchChangesHandle) Next( + ctx context.Context, + mp *mpool.MPool, +) (*batch.Batch, *batch.Batch, engine.ChangesHandle_Hint, error) { + data, tombstone, hint, err := h.inner.Next(ctx, mp) + if err != nil || data == nil || data.RowCount() == 0 { + return data, tombstone, hint, err + } + data = projectDataBranchBatchToTarget(data, &h.tblStuff, h.sourceMapping, mp) + if err = hydrateHistoricalDataBranchBatch( + ctx, h.ses, data, h.tblStuff, h.endpointRel, h.endpointSnapshot, + ); err != nil { + data.Clean(mp) + if tombstone != nil { + tombstone.Clean(mp) + } + return nil, nil, hint, err + } + return data, tombstone, hint, nil +} + +func (h *historicalDataBranchChangesHandle) Close() error { + return h.inner.Close() +} + +func projectDataBranchBatchToTarget( + sourceBat *batch.Batch, + tblStuff *tableStuff, + sourceColToTargetIdx []int, + mp *mpool.MPool, +) *batch.Batch { + // CollectChanges data batches are laid out as [RowID, data..., commit_ts]. + // Identify the optional trailing vector from the schema-derived data-column + // count so that it cannot be mistaken for a column that needs projection. + hasCommitTS := sourceBat.VectorCount() == len(sourceColToTargetIdx)+2 + outColCount := len(tblStuff.def.colNames) + 1 + if hasCommitTS { + outColCount++ + } + out := batch.NewWithSize(outColCount) + out.Vecs[0] = sourceBat.Vecs[0] // RowID + sourceBat.Vecs[0] = nil + sourceColCount := sourceBat.VectorCount() - 1 // subtract RowID + if hasCommitTS { + sourceColCount-- + commitTSIdx := sourceBat.VectorCount() - 1 + out.Vecs[out.VectorCount()-1] = sourceBat.Vecs[commitTSIdx] + sourceBat.Vecs[commitTSIdx] = nil + } + if sourceColCount > len(sourceColToTargetIdx) { + sourceColCount = len(sourceColToTargetIdx) + } + for sourceColIdx := 0; sourceColIdx < sourceColCount; sourceColIdx++ { + tarColIdx := sourceColToTargetIdx[sourceColIdx] if tarColIdx >= 0 && tarColIdx < len(tblStuff.def.colNames) { - out.Vecs[tarColIdx+1] = baseBat.Vecs[baseColIdx+1] - baseBat.Vecs[baseColIdx+1] = nil + out.Vecs[tarColIdx+1] = sourceBat.Vecs[sourceColIdx+1] + sourceBat.Vecs[sourceColIdx+1] = nil } } for i := range tblStuff.def.colNames { if out.Vecs[i+1] == nil { - out.Vecs[i+1] = vector.NewConstNull(tblStuff.def.colTypes[i], baseBat.RowCount(), mp) + out.Vecs[i+1] = vector.NewConstNull(tblStuff.def.colTypes[i], sourceBat.RowCount(), mp) } } - out.SetRowCount(baseBat.RowCount()) - baseBat.Clean(mp) + out.SetRowCount(sourceBat.RowCount()) + out.SetAttributes(dataBranchTargetLayoutAttrs(tblStuff, hasCommitTS)) + sourceBat.Clean(mp) return out } diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index ee5764a9b0b40..e8f9ad7c27b94 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -37,6 +37,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/util/executor" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/readutil" "github.com/panjf2000/ants/v2" @@ -675,7 +676,7 @@ func TestBuildHashmapForTableProjectsBaseRowsBeforeKeying(t *testing.T) { }, -1)) } -func TestDataBranchLineageTableIDFallsBackToLogicalID(t *testing.T) { +func TestDataBranchLineageTableIDReportsLegacyLogicalFallback(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -689,7 +690,9 @@ func TestDataBranchLineageTableIDFallsBackToLogicalID(t *testing.T) { LogicalId: 10, }).AnyTimes() - require.Equal(t, uint64(10), dataBranchLineageTableID(context.Background(), dag, rel)) + tableID, legacyFallback := dataBranchLineageTableID(context.Background(), dag, rel) + require.Equal(t, uint64(10), tableID) + require.True(t, legacyFallback) } func TestDiffDataHelper_ConflictAcceptExpandUpdate(t *testing.T) { @@ -1040,6 +1043,21 @@ func TestLCAProbeColumnLayoutExcludesTargetOnlyColumns(t *testing.T) { require.Equal(t, []types.T{types.T_int64, types.T_varchar}, []types.T{layout.types[0].Oid, layout.types[1].Oid}) } +func TestLCAProbeResultTargetIndexes(t *testing.T) { + layout := lcaProbeLayout{targetIdxes: []int{1, 2}} + + got, err := lcaProbeResultTargetIndexes(layout, 3, 2, false) + require.NoError(t, err) + require.Equal(t, []int{1, 2}, got, "SQL probe returns only LCA/common columns") + + got, err = lcaProbeResultTargetIndexes(layout, 3, 3, true) + require.NoError(t, err) + require.Equal(t, []int{0, 1, 2}, got, "reader fallback returns the full target layout") + + _, err = lcaProbeResultTargetIndexes(layout, 3, 1, false) + require.ErrorContains(t, err, "unexpected LCA probe result width") +} + func makeTestBranchTableStuffFakePK(tblStuff *tableStuff) { tblStuff.def.pkKind = fakeKind tblStuff.def.pkColIdxes = []int{0, 1} @@ -1121,6 +1139,77 @@ func decodeCapturedRows(t *testing.T, bat *batch.Batch, colTypes []types.Type) [ func TestHandleDelsOnLCA_SQLPaths(t *testing.T) { ses := newValidateSession(t) + t.Run("target-only first column preserves LCA delete", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.lcaRel = mock_frontend.NewMockRelation(ctrl) + tblStuff.def.colNames = []string{"added", "id", "name"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_int64.ToType(), + types.T_varchar.ToType(), + } + tblStuff.def.visibleIdxes = []int{0, 1, 2} + tblStuff.def.pkColIdx = 1 + tblStuff.def.pkColIdxes = []int{1} + tblStuff.retPool = &retBatchList{} + lcaDef := &plan.TableDef{ + DbName: "db1", + Name: "lca_tbl", + Pkey: &plan.PrimaryKeyDef{ + Names: []string{"id"}, + PkeyColName: "id", + }, + Cols: []*plan.ColDef{ + {Name: "id", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "name", Typ: plan.Type{Id: int32(types.T_varchar)}}, + }, + } + tblStuff.lcaRel.(*mock_frontend.MockRelation).EXPECT().GetTableDef(gomock.Any()).Return(lcaDef).AnyTimes() + tblStuff.lcaRel.(*mock_frontend.MockRelation).EXPECT().GetTableID(gomock.Any()).Return(uint64(76)).AnyTimes() + + mrs := &MysqlResultSet{} + for _, col := range []struct { + name string + typ defines.MysqlType + }{ + {name: "__idx_", typ: defines.MYSQL_TYPE_LONGLONG}, + {name: "id", typ: defines.MYSQL_TYPE_LONGLONG}, + {name: "name", typ: defines.MYSQL_TYPE_VARCHAR}, + } { + mysqlCol := &MysqlColumn{} + mysqlCol.SetName(col.name) + mysqlCol.SetColumnType(col.typ) + mrs.AddColumn(mysqlCol) + } + mrs.AddRow([]interface{}{int64(0), int64(1), "alice"}) + + bh := mock_frontend.NewMockBackgroundExec(ctrl) + bh.EXPECT().Exec(gomock.Any(), gomock.Any()).Return(nil).Times(1) + bh.EXPECT().GetExecResultSet().Return([]interface{}{mrs}).Times(1) + bh.EXPECT().ClearExecResultSet().Times(1) + + tBat := batch.NewWithSize(1) + tBat.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixed(tBat.Vecs[0], int64(1), false, ses.proc.Mp())) + tBat.SetRowCount(1) + defer tBat.Clean(ses.proc.Mp()) + + dBat, err := handleDelsOnLCA( + context.Background(), ses, bh, tBat, tblStuff, + types.BuildTS(10, 0).ToTimestamp(), + ) + require.NoError(t, err) + require.Equal(t, 1, dBat.RowCount()) + require.True(t, dBat.Vecs[0].IsNull(0)) + require.Equal(t, []int64{1}, vector.MustFixedColWithTypeCheck[int64](dBat.Vecs[1])) + require.Equal(t, "alice", string(dBat.Vecs[2].GetBytesAt(0))) + require.Zero(t, tBat.RowCount()) + tblStuff.retPool.releaseRetBatch(dBat, false) + }) + t.Run("sql result keeps hits and leaves misses in tombstone batch", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -2196,6 +2285,161 @@ func TestProjectBaseBatchToTarget(t *testing.T) { projected.Clean(mp) } +func TestProjectBaseBatchToTargetPreservesTrailingCommitTS(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.def.colNames = []string{"a", "c", "b"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_int64.ToType(), + types.T_varchar.ToType(), + } + tblStuff.def.baseColToTarIdx = []int{0, 2} + tblStuff.def.tarOnlyIdxes = []int{1} + + mp := ses.proc.Mp() + baseline := mp.CurrNB() + baseBat := batch.NewWithSize(4) + baseBat.SetAttributes([]string{catalog.Row_ID, "a", "b", objectio.DefaultCommitTS_Attr}) + baseBat.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + baseBat.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + baseBat.Vecs[2] = vector.NewVec(types.T_varchar.ToType()) + baseBat.Vecs[3] = vector.NewVec(types.T_TS.ToType()) + + uid, err := types.BuildUuid() + require.NoError(t, err) + blkID := objectio.NewBlockid(&uid, 0, 1) + commitTS := types.BuildTS(42, 7) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[0], types.NewRowid(blkID, 0), false, mp)) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[1], int64(11), false, mp)) + require.NoError(t, vector.AppendBytes(baseBat.Vecs[2], []byte("base"), false, mp)) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[3], commitTS, false, mp)) + baseBat.SetRowCount(1) + + rowIDVec := baseBat.Vecs[0] + aVec := baseBat.Vecs[1] + bVec := baseBat.Vecs[2] + commitTSVec := baseBat.Vecs[3] + projected := projectBaseBatchToTarget(baseBat, &tblStuff, mp) + + // CollectChanges data batches are [RowID, data..., commit_ts]. Projection + // changes only the data-column layout and retains both boundary vectors. + require.Equal(t, 5, projected.VectorCount()) + require.True(t, dataBranchBatchHasTargetLayout(projected, &tblStuff)) + require.Same(t, rowIDVec, projected.Vecs[0]) + require.Same(t, aVec, projected.Vecs[1]) + require.True(t, projected.Vecs[2].IsConstNull()) + require.Same(t, bVec, projected.Vecs[3]) + require.Same(t, commitTSVec, projected.Vecs[4]) + require.Equal(t, commitTS, vector.GetFixedAtNoTypeCheck[types.TS](projected.Vecs[4], 0)) + + projected.Clean(mp) + require.Equal(t, baseline, mp.CurrNB(), "projection must transfer or clean every input vector") +} + +func TestDataBranchSourceColToTargetIdxRejectsHistoricalTypeDrift(t *testing.T) { + sourceDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, + } + targetDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int32)}}, + }, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, + } + + _, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a"}) + require.Error(t, err) + require.Contains(t, err.Error(), "historical data branch column a has a different type") +} + +func TestDataBranchSourceColToTargetIdxRejectsHistoricalPrimaryKeyDrift(t *testing.T) { + cols := []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + } + sourceDef := &plan.TableDef{ + Cols: cols, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "b", Names: []string{"b"}}, + } + targetDef := &plan.TableDef{ + Cols: cols, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, + } + + _, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a", "b"}) + require.Error(t, err) + require.Contains(t, err.Error(), "historical data branch primary key is incompatible") +} + +func TestDataBranchSourceColToTargetIdxRejectsHistoricalFakePrimaryKeyDrift(t *testing.T) { + fakePK := &plan.PrimaryKeyDef{PkeyColName: catalog.FakePrimaryKeyColName} + sourceDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: catalog.FakePrimaryKeyColName, Typ: plan.Type{Id: int32(types.T_varchar)}}, + }, + Pkey: fakePK, + } + targetDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: catalog.FakePrimaryKeyColName, Typ: plan.Type{Id: int32(types.T_varchar)}}, + }, + Pkey: fakePK, + } + + _, err := dataBranchSourceColToTargetIdx( + sourceDef, targetDef, []string{"a", "b", catalog.FakePrimaryKeyColName}, + ) + require.Error(t, err) + require.Contains(t, err.Error(), "historical data branch fake primary key schema differs") +} + +func TestOverlayDataBranchProbeResultHydratesTargetDefaults(t *testing.T) { + ses := newValidateSession(t) + mp := ses.proc.Mp() + + projected := batch.NewWithSize(5) + projected.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + projected.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + projected.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + projected.Vecs[3] = vector.NewConstNull(types.T_int64.ToType(), 1, mp) + projected.Vecs[4] = vector.NewVec(types.T_TS.ToType()) + rowID := buildHashDiffRowID(t, 1) + require.NoError(t, vector.AppendFixed(projected.Vecs[0], rowID, false, mp)) + require.NoError(t, vector.AppendFixed(projected.Vecs[1], int64(1), false, mp)) + require.NoError(t, vector.AppendFixed(projected.Vecs[2], int64(11), false, mp)) + commitTS := types.BuildTS(20, 1) + require.NoError(t, vector.AppendFixed(projected.Vecs[4], commitTS, false, mp)) + projected.SetRowCount(1) + defer projected.Clean(mp) + + probeBatch := batch.NewWithSize(4) + probeBatch.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + probeBatch.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + probeBatch.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + probeBatch.Vecs[3] = vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixed(probeBatch.Vecs[0], int64(0), false, mp)) + require.NoError(t, vector.AppendFixed(probeBatch.Vecs[1], int64(1), false, mp)) + require.NoError(t, vector.AppendFixed(probeBatch.Vecs[2], int64(11), false, mp)) + require.NoError(t, vector.AppendFixed(probeBatch.Vecs[3], int64(0), false, mp)) + probeBatch.SetRowCount(1) + probe := executor.Result{Mp: mp, Batches: []*batch.Batch{probeBatch}} + defer probe.Close() + + require.NoError(t, overlayDataBranchProbeResult(projected, probe, 0, mp)) + require.Equal(t, []int64{0}, vector.MustFixedColWithTypeCheck[int64](projected.Vecs[3])) + require.Equal(t, []types.TS{commitTS}, vector.MustFixedColWithTypeCheck[types.TS](projected.Vecs[4])) +} + func TestProjectBaseBatchToTargetMovesHiddenKeyColumns(t *testing.T) { ses := newValidateSession(t) ctrl := gomock.NewController(t) diff --git a/pkg/frontend/data_branch_privilege.go b/pkg/frontend/data_branch_privilege.go index 3ced1ade8bedd..93859e8c9bb24 100644 --- a/pkg/frontend/data_branch_privilege.go +++ b/pkg/frontend/data_branch_privilege.go @@ -25,6 +25,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/frontend/databranchutils" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/util/executor" "github.com/matrixorigin/matrixone/pkg/util/trace/impl/motrace/statistic" @@ -743,9 +744,10 @@ func validateActiveBranchChildTableIDs( } sql := fmt.Sprintf( - "select table_id from %s.%s where table_deleted = false and table_id in (%s)", + "select table_id from %s.%s where table_deleted = false and level != '%s' and table_id in (%s)", catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, + databranchutils.AlterLineageLevel, formatUintList(idList[start:end]), ) sqlRet, err := runSql(sysCtx, ses, bh, sql, nil, nil) diff --git a/pkg/frontend/data_branch_snapshot_test.go b/pkg/frontend/data_branch_snapshot_test.go index aee1dab53c1f4..af8cd59898f5b 100644 --- a/pkg/frontend/data_branch_snapshot_test.go +++ b/pkg/frontend/data_branch_snapshot_test.go @@ -144,6 +144,20 @@ func TestSubtreeAllDeleted_Linear(t *testing.T) { require.True(t, dag.SubtreeAllDeleted(2)) } +func TestSubtreeHasLiveNodeThroughDeletedGeneration(t *testing.T) { + dag := databranchutils.NewBranchReclaimDag([]databranchutils.DataBranchMetadata{ + {TableID: 2, PTableID: 1, TableDeleted: true}, + {TableID: 3, PTableID: 2, TableDeleted: false}, + }) + + require.True(t, dag.SubtreeHasLiveNode(1)) + require.True(t, dag.SubtreeHasLiveNode(2)) + require.True(t, dag.SubtreeHasLiveNode(3)) + + dag.Info[3] = databranchutils.BranchReclaimNode{ParentTableID: 2, Deleted: true} + require.False(t, dag.SubtreeHasLiveNode(1)) +} + // --------------------------------------------------------------------------- // UT-U4 — SubtreeAllDeleted on a branching DAG t1 -> {t2, t3}, t2 -> t4 // --------------------------------------------------------------------------- diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index 993cbe509107a..882103f3afe22 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -46,6 +46,13 @@ func TestDataBranchUserVisibleColumn(t *testing.T) { require.False(t, isDataBranchUserVisibleColumn(&plan.ColDef{Name: catalog.Row_ID, Hidden: true})) } +func TestBranchQuotaUsageSQLExcludesRootAlterLineage(t *testing.T) { + require.Equal(t, + "select count(*) from mo_catalog.mo_branch_metadata where creator = 7 and table_deleted = false and level != 'alter'", + branchQuotaUsageSQL(7), + ) +} + func TestDataBranchFakePKColIdxesUseOnlyVisibleColumns(t *testing.T) { tblDef := &plan.TableDef{ Cols: []*plan.ColDef{ @@ -1037,6 +1044,26 @@ func TestCheckSchemaCompatibility_PKChanged(t *testing.T) { require.Contains(t, err.Error(), "primary key column") } +func TestCheckSchemaCompatibility_RejectsChangedPrimaryKeyWithCommonColumns(t *testing.T) { + tarDef := &plan.TableDef{ + Pkey: &plan.PrimaryKeyDef{Names: []string{"b"}, PkeyColName: "b"}, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + baseDef := &plan.TableDef{ + Pkey: &plan.PrimaryKeyDef{Names: []string{"a"}, PkeyColName: "a"}, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + }, + } + + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.ErrorContains(t, err, "primary key columns") +} + func TestCheckSchemaCompatibility_TypeMismatch(t *testing.T) { tarDef := &plan.TableDef{ Name: "target", @@ -1066,6 +1093,46 @@ func TestCheckSchemaCompatibility_TypeMismatch(t *testing.T) { require.Contains(t, err.Error(), "has different types") } +func TestCheckSchemaCompatibility_RejectsDifferentTypeAttributes(t *testing.T) { + tarDef := &plan.TableDef{ + Pkey: &plan.PrimaryKeyDef{Names: []string{"a"}, PkeyColName: "a"}, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "amount", Typ: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 2}}, + }, + } + baseDef := &plan.TableDef{ + Pkey: &plan.PrimaryKeyDef{Names: []string{"a"}, PkeyColName: "a"}, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "amount", Typ: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 0}}, + }, + } + + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.ErrorContains(t, err, "different type attributes") +} + +func TestCheckSchemaCompatibility_RejectsDifferentColumnNullability(t *testing.T) { + tarDef := &plan.TableDef{ + Pkey: &plan.PrimaryKeyDef{Names: []string{"a"}, PkeyColName: "a"}, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}, NotNull: false}, + }, + } + baseDef := &plan.TableDef{ + Pkey: &plan.PrimaryKeyDef{Names: []string{"a"}, PkeyColName: "a"}, + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}, NotNull: true}, + }, + } + + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.ErrorContains(t, err, "different nullability") +} + func TestCheckSchemaCompatibility_BaseOnlyVisibleColumnRejected(t *testing.T) { tarDef := &plan.TableDef{ Name: "target", @@ -1128,7 +1195,7 @@ func TestCheckSchemaCompatibility_CompositePK(t *testing.T) { require.Equal(t, []int{2}, tarOnlyIdxes) } -func TestCheckSchemaCompatibility_FakePKAllowsTargetOnlyColumns(t *testing.T) { +func TestCheckSchemaCompatibility_FakePKRejectsTargetOnlyColumns(t *testing.T) { tarDef := &plan.TableDef{ Name: "target", Pkey: &plan.PrimaryKeyDef{ @@ -1151,9 +1218,33 @@ func TestCheckSchemaCompatibility_FakePKAllowsTargetOnlyColumns(t *testing.T) { }, } - commonIdxes, commonVisibleIdxes, tarOnlyIdxes, err := checkSchemaCompatibility(tarDef, baseDef) - require.NoError(t, err) - require.Equal(t, []int{0, 1}, commonIdxes) - require.Equal(t, []int{0, 1}, commonVisibleIdxes) - require.Equal(t, []int{2}, tarOnlyIdxes) + _, _, _, err := checkSchemaCompatibility(tarDef, baseDef) + require.Error(t, err) + require.Contains(t, err.Error(), "require an explicit primary key") +} + +func TestDataBranchCollectRelationSnapshot(t *testing.T) { + endpointSP := types.BuildTS(300, 7) + transitionTS := types.BuildTS(200, 3) + + require.Equal(t, endpointSP, dataBranchCollectRelationSnapshot(endpointSP, transitionTS, true)) + require.Equal(t, transitionTS, dataBranchCollectRelationSnapshot(endpointSP, transitionTS, false)) +} + +func TestBranchMetaInfoLCASnapshotIgnoresAlterLineageEdges(t *testing.T) { + alterTS := types.BuildTS(200, 0) + forkTS := types.BuildTS(100, 0) + tarSP := types.BuildTS(300, 0) + baseSP := types.BuildTS(250, 0) + info := branchMetaInfo{ + pathFromLCAToTar: []uint64{1, 2}, + pathFromLCAToTarTS: []types.TS{{}, alterTS}, + pathFromLCAToTarLineageOnly: []bool{false, true}, + pathFromLCAToBase: []uint64{1, 3}, + pathFromLCAToBaseTS: []types.TS{{}, forkTS}, + pathFromLCAToBaseLineageOnly: []bool{false, false}, + } + + require.Equal(t, forkTS, info.tarLCASnapshot(baseSP)) + require.Equal(t, forkTS, info.baseLCASnapshot(tarSP)) } diff --git a/pkg/frontend/data_branch_types.go b/pkg/frontend/data_branch_types.go index cf7d2f6dd866b..655e7f9c4e737 100644 --- a/pkg/frontend/data_branch_types.go +++ b/pkg/frontend/data_branch_types.go @@ -131,10 +131,15 @@ type collectRange struct { type branchMetaInfo struct { lcaTableId uint64 - pathFromLCAToTar []uint64 - pathFromLCAToTarTS []types.TS - pathFromLCAToBase []uint64 - pathFromLCAToBaseTS []types.TS + pathFromLCAToTar []uint64 + pathFromLCAToTarTS []types.TS + // LineageOnly[i] marks an ALTER copy-and-swap edge into path[i]. Such + // edges split physical collection ranges but do not represent a branch + // fork for LCA probing/conflict semantics. + pathFromLCAToTarLineageOnly []bool + pathFromLCAToBase []uint64 + pathFromLCAToBaseTS []types.TS + pathFromLCAToBaseLineageOnly []bool } // hasLCA reports whether tar and base share a common ancestor in the @@ -154,26 +159,44 @@ func (m *branchMetaInfo) hasLCA() bool { // point from base: base's first-child CloneTS, or baseSP if base is // also the LCA (case-0 same-table diff). func (m *branchMetaInfo) tarLCASnapshot(baseSP types.TS) types.TS { - if len(m.pathFromLCAToTar) > 1 { - return m.pathFromLCAToTarTS[1] + if ts, ok := firstDataBranchForkTS( + m.pathFromLCAToTarTS, m.pathFromLCAToTarLineageOnly, + ); ok { + return ts } - if len(m.pathFromLCAToBase) > 1 { - return m.pathFromLCAToBaseTS[1] + if ts, ok := firstDataBranchForkTS( + m.pathFromLCAToBaseTS, m.pathFromLCAToBaseLineageOnly, + ); ok { + return ts } return baseSP } // baseLCASnapshot is the mirror of tarLCASnapshot for the base side. func (m *branchMetaInfo) baseLCASnapshot(tarSP types.TS) types.TS { - if len(m.pathFromLCAToBase) > 1 { - return m.pathFromLCAToBaseTS[1] + if ts, ok := firstDataBranchForkTS( + m.pathFromLCAToBaseTS, m.pathFromLCAToBaseLineageOnly, + ); ok { + return ts } - if len(m.pathFromLCAToTar) > 1 { - return m.pathFromLCAToTarTS[1] + if ts, ok := firstDataBranchForkTS( + m.pathFromLCAToTarTS, m.pathFromLCAToTarLineageOnly, + ); ok { + return ts } return tarSP } +func firstDataBranchForkTS(pathTS []types.TS, lineageOnly []bool) (types.TS, bool) { + for i := 1; i < len(pathTS); i++ { + if i < len(lineageOnly) && lineageOnly[i] { + continue + } + return pathTS[i], true + } + return types.TS{}, false +} + // lcaProbeSnapshot is the snapshot used by diffOnBase to fetch the // LCA relation handle when the LCA is a third-party node (neither // tar nor base). It is the earlier of the two per-side snapshots so diff --git a/pkg/frontend/databranchutils/branch_dag.go b/pkg/frontend/databranchutils/branch_dag.go index 1684d2bee3a58..5b2f1cc557131 100644 --- a/pkg/frontend/databranchutils/branch_dag.go +++ b/pkg/frontend/databranchutils/branch_dag.go @@ -18,13 +18,19 @@ type DataBranchMetadata struct { TableID uint64 CloneTS int64 PTableID uint64 + Creator uint64 + Level string TableDeleted bool + LineageOnly bool } type dagNode struct { TableID uint64 CloneTS int64 ParentID uint64 + // LineageOnly marks copy-and-swap generations (for example ALTER) + // that preserve one logical branch rather than creating a new fork. + LineageOnly bool Parent *dagNode Depth int @@ -59,6 +65,7 @@ func NewDAG(rows []DataBranchMetadata) *DataBranchDAG { node1.CloneTS = row.CloneTS node1.ParentID = row.PTableID + node1.LineageOnly = row.LineageOnly if _, ok = dag.nodes[row.PTableID]; !ok { node2 = &dagNode{ @@ -114,6 +121,11 @@ func (d *DataBranchDAG) Exists(tableID uint64) bool { return ok } +func (d *DataBranchDAG) IsLineageOnly(tableID uint64) bool { + node, ok := d.nodes[tableID] + return ok && node.LineageOnly +} + func (d *DataBranchDAG) HasParent(tableID uint64) bool { node, ok := d.nodes[tableID] if !ok { @@ -278,3 +290,31 @@ func (d *DataBranchDAG) PathFromAncestor(descendantID, ancestorID uint64) ( } return nil, nil, false } + +// PathFromRoot returns the complete top-down physical/logical lineage ending +// at descendantID. It is used when two endpoints have no shared LCA: each +// side still has to reconstruct its own state across copy-and-swap ALTER +// generations instead of scanning only the current physical table. +func (d *DataBranchDAG) PathFromRoot(descendantID uint64) ( + tableIDs []uint64, + cloneTSes []int64, + ok bool, +) { + node, exists := d.nodes[descendantID] + if !exists { + return nil, nil, false + } + for steps := 0; node != nil && steps < maxBranchDAGDepth; steps++ { + tableIDs = append(tableIDs, node.TableID) + cloneTSes = append(cloneTSes, node.CloneTS) + node = node.Parent + } + if node != nil { + return nil, nil, false + } + for i, j := 0, len(tableIDs)-1; i < j; i, j = i+1, j-1 { + tableIDs[i], tableIDs[j] = tableIDs[j], tableIDs[i] + cloneTSes[i], cloneTSes[j] = cloneTSes[j], cloneTSes[i] + } + return tableIDs, cloneTSes, true +} diff --git a/pkg/frontend/databranchutils/branch_dag_test.go b/pkg/frontend/databranchutils/branch_dag_test.go index 992f9019cfb91..75817b40981bd 100644 --- a/pkg/frontend/databranchutils/branch_dag_test.go +++ b/pkg/frontend/databranchutils/branch_dag_test.go @@ -195,3 +195,23 @@ func TestDAGFunctionality(t *testing.T) { } }) } + +func TestPathFromRoot(t *testing.T) { + dag := NewDAG([]DataBranchMetadata{ + {TableID: 2, PTableID: 1, CloneTS: 20}, + {TableID: 3, PTableID: 2, CloneTS: 30}, + }) + + ids, cloneTSs, ok := dag.PathFromRoot(3) + if !ok || len(ids) != 3 || ids[0] != 1 || ids[1] != 2 || ids[2] != 3 { + t.Fatalf("unexpected root path: ids=%v ok=%v", ids, ok) + } + if len(cloneTSs) != 3 || cloneTSs[0] != 0 || cloneTSs[1] != 20 || cloneTSs[2] != 30 { + t.Fatalf("unexpected clone timestamps: %v", cloneTSs) + } + + _, _, ok = dag.PathFromRoot(99) + if ok { + t.Fatal("missing node unexpectedly had a root path") + } +} diff --git a/pkg/frontend/databranchutils/branch_protect_snapshot.go b/pkg/frontend/databranchutils/branch_protect_snapshot.go index 2db65d7d07b44..38ea594eff5f0 100644 --- a/pkg/frontend/databranchutils/branch_protect_snapshot.go +++ b/pkg/frontend/databranchutils/branch_protect_snapshot.go @@ -29,6 +29,25 @@ import ( // branch". const BranchSnapshotKind = "branch" +// AlterLineageLevel marks an internal mo_branch_metadata edge created by +// ALTER's copy-and-swap. It preserves physical history but is not a logical +// branch fork. +const AlterLineageLevel = "alter" + +func IsAlterLineageLevel(level string) bool { + return level == AlterLineageLevel || strings.HasPrefix(level, AlterLineageLevel+":") +} + +func NextAlterLineageLevel(parentLevel string) string { + if IsAlterLineageLevel(parentLevel) { + return parentLevel + } + if parentLevel == "" { + return AlterLineageLevel + } + return AlterLineageLevel + ":" + parentLevel +} + // BranchSnapshotSnamePrefix is the sname prefix used by branch-owned snapshot // rows. The suffix is the decimal child table id. Keep this in sync with the // design doc §4.3. @@ -57,6 +76,8 @@ type BranchReclaimDag struct { type BranchReclaimNode struct { ParentTableID uint64 CloneTS int64 + Creator uint64 + Level string Deleted bool } @@ -71,6 +92,8 @@ func NewBranchReclaimDag(rows []DataBranchMetadata) BranchReclaimDag { dag.Info[r.TableID] = BranchReclaimNode{ ParentTableID: r.PTableID, CloneTS: r.CloneTS, + Creator: r.Creator, + Level: r.Level, Deleted: r.TableDeleted, } if r.PTableID != 0 { @@ -99,6 +122,31 @@ func (d BranchReclaimDag) SubtreeAllDeleted(root uint64) bool { return d.subtreeAllDeletedMemo(root, memo, visited) } +// SubtreeHasLiveNode reports whether root or any descendant still represents +// a live physical generation. Unlike SubtreeAllDeleted, it intentionally +// walks children even when root has no metadata row; root tables commonly +// appear only as p_table_id. The walk is cycle-safe for corrupted metadata. +func (d BranchReclaimDag) SubtreeHasLiveNode(root uint64) bool { + visited := make(map[uint64]struct{}, len(d.Info)) + var walk func(uint64) bool + walk = func(tableID uint64) bool { + if _, ok := visited[tableID]; ok { + return false + } + visited[tableID] = struct{}{} + if meta, ok := d.Info[tableID]; ok && !meta.Deleted { + return true + } + for _, childID := range d.Children[tableID] { + if walk(childID) { + return true + } + } + return false + } + return walk(root) +} + func (d BranchReclaimDag) subtreeAllDeletedMemo( root uint64, memo map[uint64]bool, diff --git a/pkg/frontend/feature_limit.go b/pkg/frontend/feature_limit.go index a5cbc421c6085..97060b1f6ecbf 100644 --- a/pkg/frontend/feature_limit.go +++ b/pkg/frontend/feature_limit.go @@ -63,6 +63,16 @@ func checkBranchQuota( return featureLimitChecker(ctx, ses, bh, featureCodeBranch, "", increment) } +func branchQuotaUsageSQL(accountID uint32) string { + return fmt.Sprintf( + "select count(*) from %s.%s where creator = %d and table_deleted = false and level != '%s'", + catalog.MO_CATALOG, + catalog.MO_BRANCH_METADATA, + accountID, + databranchutils.AlterLineageLevel, + ) +} + func featureLimitChecker( ctx context.Context, ses *Session, @@ -117,10 +127,7 @@ func featureLimitChecker( ) } else if featureCode == featureCodeBranch { ctx = defines.AttachAccountId(ctx, sysAccountID) - sql = fmt.Sprintf( - "select count(*) from %s.%s where creator = %d and table_deleted = false", - catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, accId, - ) + sql = branchQuotaUsageSQL(accId) } else { return moerr.NewInternalErrorNoCtxf("no such feature %s with scope %s", featureCode, featureScope) } diff --git a/pkg/sql/compile/alter.go b/pkg/sql/compile/alter.go index 9837d0c67bb9d..dbe9975a9e365 100644 --- a/pkg/sql/compile/alter.go +++ b/pkg/sql/compile/alter.go @@ -19,11 +19,16 @@ import ( "fmt" "slices" "strings" + "time" + "github.com/google/uuid" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/reuse" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/frontend/databranchutils" indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -31,6 +36,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/pb/lock" "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/sql/colexec/table_clone" "github.com/matrixorigin/matrixone/pkg/sql/features" "github.com/matrixorigin/matrixone/pkg/sql/parsers" @@ -43,6 +49,283 @@ import ( "go.uber.org/zap" ) +func buildAlterDataBranchLineageSQL( + oldTableID, newTableID uint64, + cloneTS int64, + creator uint32, + lineageLevel, accountName, databaseName, tableName, snapshotID string, +) (metadataSQL, snapshotSQL string) { + metadataSQL = fmt.Sprintf( + "insert into %s.%s values(%d, %d, %d, %d, '%s', false)", + catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, + newTableID, cloneTS, oldTableID, creator, sqlquote.EscapeString(lineageLevel), + ) + snapshotSQL = fmt.Sprintf( + `insert into %s.%s(snapshot_id, sname, ts, level, account_name, database_name, table_name, obj_id, kind) `+ + `values ('%s', '%s', %d, 'table', '%s', '%s', '%s', %d, '%s')`, + catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, + sqlquote.EscapeString(snapshotID), + databranchutils.BranchSnapshotName(newTableID), + cloneTS, + sqlquote.EscapeString(accountName), + sqlquote.EscapeString(databaseName), + sqlquote.EscapeString(tableName), + oldTableID, + databranchutils.BranchSnapshotKind, + ) + return +} + +type alterDataBranchLineagePlan struct { + enabled bool + preserveHistoricalSource bool + cloneTS int64 + fixedCopyTS bool +} + +func alterCopySQLAtLineageSnapshot(sql string, plan alterDataBranchLineagePlan) string { + if !plan.enabled || !plan.fixedCopyTS { + return sql + } + return sql + fmt.Sprintf(" {MO_TS = %d}", plan.cloneTS) +} + +func alterDataBranchParticipationSQL(oldTableID uint64) string { + return fmt.Sprintf( + "select 1 from %s.%s where table_id = %d or p_table_id = %d limit 1", + catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, oldTableID, oldTableID, + ) +} + +func alterDataBranchHistoricalSourceSQL(accountName, databaseName, tableName string, tableID uint64) string { + accountName = sqlquote.EscapeString(accountName) + databaseName = sqlquote.EscapeString(databaseName) + tableName = sqlquote.EscapeString(tableName) + return fmt.Sprintf( + `select 1 from (`+ + `select level, account_name, database_name, table_name, obj_id from %s.%s where kind = 'user' `+ + `union all `+ + `select level, account_name, database_name, table_name, obj_id from %s.%s where pitr_status = 1`+ + `) as h where h.level = 'cluster' or (`+ + `h.account_name = '%s' and (`+ + `h.level = 'account' or `+ + `(h.level = 'database' and h.database_name = '%s') or `+ + `(h.level = 'table' and (h.obj_id = %d or (h.database_name = '%s' and h.table_name = '%s')))`+ + `)) limit 1`, + catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, + catalog.MO_CATALOG, catalog.MO_PITR, + accountName, databaseName, tableID, databaseName, tableName, + ) +} + +func (c *Compile) alterTableParticipatesInDataBranch(oldTableID uint64) (bool, error) { + probeSQL := alterDataBranchParticipationSQL(oldTableID) + res, err := c.runSqlWithResult(probeSQL, int32(catalog.System_Account)) + if err != nil { + return false, err + } + participates := false + res.ReadRows(func(rows int, _ []*vector.Vector) bool { + participates = rows > 0 + return false + }) + res.Close() + return participates, nil +} + +func (c *Compile) alterTableHasHistoricalBranchSource( + oldTableID uint64, + databaseName, tableName string, +) (bool, error) { + res, err := c.runSqlWithResult( + alterDataBranchHistoricalSourceSQL( + c.proc.GetSessionInfo().Account, databaseName, tableName, oldTableID, + ), + int32(catalog.System_Account), + ) + if err != nil { + return false, err + } + defer res.Close() + hasHistory := false + res.ReadRows(func(rows int, _ []*vector.Vector) bool { + hasHistory = rows > 0 + return false + }) + return hasHistory, nil +} + +func (c *Compile) prepareAlterDataBranchLineage( + oldTableID uint64, + databaseName, tableName string, +) (alterDataBranchLineagePlan, error) { + participates, err := c.alterTableParticipatesInDataBranch(oldTableID) + if err != nil { + return alterDataBranchLineagePlan{}, err + } + hasLiveLineage := false + if participates { + dag, dagErr := c.loadAlterDataBranchDAG(false) + if dagErr != nil { + return alterDataBranchLineagePlan{}, dagErr + } + hasLiveLineage = dag.SubtreeHasLiveNode(oldTableID) + } + preserveHistoricalSource := false + if !hasLiveLineage { + if preserveHistoricalSource, err = c.alterTableHasHistoricalBranchSource( + oldTableID, databaseName, tableName, + ); err != nil { + return alterDataBranchLineagePlan{}, err + } + if !preserveHistoricalSource { + return alterDataBranchLineagePlan{}, nil + } + } + + op := c.proc.GetTxnOperator() + opts := op.TxnOptions() + if err = validateAlterDataBranchLineageTxn( + opts.GetByBegin(), opts.GetAutocommit(), op.Txn().IsPessimistic(), + ); err != nil { + return alterDataBranchLineagePlan{}, err + } + return alterDataBranchLineagePlan{ + enabled: true, + preserveHistoricalSource: preserveHistoricalSource, + }, nil +} + +func validateAlterDataBranchLineageTxn(byBegin, autocommit, _ bool) error { + if byBegin || !autocommit { + return moerr.NewNotSupportedNoCtx( + "ALTER on a data-branch lineage is not supported inside an explicit transaction", + ) + } + return nil +} + +func shouldAdvanceAlterDataBranchLineageSnapshot(pessimistic, rcIsolation bool) bool { + return pessimistic && rcIsolation +} + +func (c *Compile) advanceAlterDataBranchLineageSnapshot() (int64, error) { + op := c.proc.GetTxnOperator() + requested := op.SnapshotTS().PhysicalTime + int64(time.Microsecond) + if err := op.UpdateSnapshot(c.proc.Ctx, timestamp.Timestamp{PhysicalTime: requested}); err != nil { + return 0, err + } + updated := op.SnapshotTS().PhysicalTime + if updated <= requested { + return 0, moerr.NewInternalErrorNoCtx( + "failed to advance ALTER data-branch lineage snapshot", + ) + } + return updated - int64(time.Nanosecond), nil +} + +func (c *Compile) loadAlterDataBranchDAG(forUpdate bool) (databranchutils.BranchReclaimDag, error) { + suffix := "" + if forUpdate { + suffix = " for update" + } + res, err := c.runSqlWithResult( + fmt.Sprintf( + "select table_id, p_table_id, clone_ts, creator, level, table_deleted from %s.%s%s", + catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, suffix, + ), + int32(catalog.System_Account), + ) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + defer res.Close() + rows := make([]databranchutils.DataBranchMetadata, 0, res.AffectedRows) + res.ReadRows(func(rowCount int, cols []*vector.Vector) bool { + if rowCount == 0 { + return true + } + tableIDs := vector.MustFixedColNoTypeCheck[uint64](cols[0]) + parentIDs := vector.MustFixedColNoTypeCheck[uint64](cols[1]) + cloneTSs := vector.MustFixedColNoTypeCheck[int64](cols[2]) + creators := vector.MustFixedColNoTypeCheck[uint64](cols[3]) + levels := executor.GetStringRows(cols[4]) + deleted := vector.MustFixedColNoTypeCheck[bool](cols[5]) + for i := range tableIDs { + rows = append(rows, databranchutils.DataBranchMetadata{ + TableID: tableIDs[i], + PTableID: parentIDs[i], + CloneTS: cloneTSs[i], + Creator: creators[i], + Level: levels[i], + TableDeleted: deleted[i], + }) + } + return true + }) + return databranchutils.NewBranchReclaimDag(rows), nil +} + +func alterDataBranchLineageMetadata( + dag databranchutils.BranchReclaimDag, + oldTableID uint64, +) (creator uint32, level string) { + meta, ok := dag.Info[oldTableID] + if !ok || meta.Deleted { + return catalog.System_Account, databranchutils.AlterLineageLevel + } + return uint32(meta.Creator), databranchutils.NextAlterLineageLevel(meta.Level) +} + +// preserveAlterDataBranchLineage models ALTER's copy-and-swap as a lineage +// edge whenever the old physical table has a live branch descendant. The +// matching snapshot pins the old generation until every later generation and +// descendant has been dropped. +func (c *Compile) preserveAlterDataBranchLineage( + plan alterDataBranchLineagePlan, + oldTableID, newTableID uint64, + databaseName, tableName string, +) error { + if !plan.enabled { + participates, err := c.alterTableParticipatesInDataBranch(oldTableID) + if err != nil || !participates { + return err + } + } + dag, err := c.loadAlterDataBranchDAG(true) + if err != nil { + return err + } + if !dag.SubtreeHasLiveNode(oldTableID) && !plan.preserveHistoricalSource { + return nil + } + if !plan.enabled { + return moerr.NewTxnNeedRetryWithDefChanged(c.proc.Ctx) + } + + snapshotID, err := uuid.NewV7() + if err != nil { + return err + } + creator, lineageLevel := alterDataBranchLineageMetadata(dag, oldTableID) + metadataSQL, snapshotSQL := buildAlterDataBranchLineageSQL( + oldTableID, newTableID, plan.cloneTS, creator, lineageLevel, + c.proc.GetSessionInfo().Account, databaseName, tableName, snapshotID.String(), + ) + if err = c.runSqlWithSystemTenant(metadataSQL); err != nil { + return err + } + if err = c.runSqlWithSystemTenant(snapshotSQL); err != nil { + return err + } + logutil.Info("DataBranch-Alter-Lineage-Preserved", + zap.Uint64("old-table-id", oldTableID), + zap.Uint64("new-table-id", newTableID), + zap.Int64("clone-ts", plan.cloneTS), + ) + return nil +} + func convertDBEOB(ctx context.Context, e error, name string) error { if moerr.IsMoErrCode(e, moerr.OkExpectedEOB) { return moerr.NewBadDB(ctx, name) @@ -297,7 +580,18 @@ func (s *Scope) AlterTableCopy(c *Compile) error { } oldId := originRel.GetTableID(c.proc.Ctx) - if c.proc.GetTxnOperator().Txn().IsPessimistic() { + lineagePlan := alterDataBranchLineagePlan{} + lineageSnapshotAdvanced := false + lineageCloneTS := int64(0) + lineageTxnOp := c.proc.GetTxnOperator() + lineageOriginalSnapshot := timestamp.Timestamp{} + lineageRestoreSnapshot := false + defer func() { + if lineageRestoreSnapshot { + lineageTxnOp.SetSnapshotTS(lineageOriginalSnapshot) + } + }() + if lineageTxnOp.Txn().IsPessimistic() { var retryErr error // 0. lock origin database metadata in catalog if err = lockMoDatabase(c, dbName, lock.LockMode_Shared); err != nil { @@ -360,6 +654,46 @@ func (s *Scope) AlterTableCopy(c *Compile) error { if retryErr != nil { return retryErr } + if shouldAdvanceAlterDataBranchLineageSnapshot( + lineageTxnOp.Txn().IsPessimistic(), lineageTxnOp.Txn().IsRCIsolation(), + ) { + // The source metadata lock excludes new current-source branch clones. + // Under RC, advance the statement snapshot while holding it so a branch + // that committed just before lock acquisition is visible to the lineage + // probe below, even when lock acquisition itself did not wait. + lineageOriginalSnapshot = lineageTxnOp.SnapshotTS() + lineageRestoreSnapshot = true + if lineageCloneTS, err = c.advanceAlterDataBranchLineageSnapshot(); err != nil { + return err + } + lineageSnapshotAdvanced = true + } + } + lineagePlan, err = c.prepareAlterDataBranchLineage(oldId, dbName, tblName) + if err != nil { + return err + } + if lineagePlan.enabled { + if lineageSnapshotAdvanced { + lineagePlan.cloneTS = lineageCloneTS + lineagePlan.fixedCopyTS = true + } else { + // Optimistic mode has no row-lock snapshot barrier. Its statement + // snapshot is nevertheless the exact source view copied by ALTER, so + // record that same boundary without adding a MO_TS override. + lineagePlan.cloneTS = c.proc.GetTxnOperator().SnapshotTS().PhysicalTime + } + } + if lineageSnapshotAdvanced { + // Re-resolve after the lock-held snapshot barrier so ordinary ALTER and + // lineage ALTER both copy from the exact catalog view just validated. + originRel, err = dbSource.Relation(c.proc.Ctx, tblName, nil) + if err != nil { + return err + } + if originRel.GetTableID(c.proc.Ctx) != oldId { + return moerr.NewTxnNeedRetryWithDefChanged(c.proc.Ctx) + } } // 3. create temporary replica table which doesn't have foreign key constraints @@ -417,9 +751,10 @@ func (s *Scope) AlterTableCopy(c *Compile) error { return err } opt := alterCopyStatementOption(alterCopyOpt) + insertTmpDataSQL := alterCopySQLAtLineageSnapshot(qry.InsertTmpDataSql, lineagePlan) err = func() error { if !shouldEnableAlterCopyPipelineFlush(alterCopyOpt) { - return c.runSqlWithOptions(qry.InsertTmpDataSql, opt) + return c.runSqlWithOptions(insertTmpDataSQL, opt) } // Enable pipeline flush only when PK dedup can be skipped or was proven safe @@ -436,14 +771,14 @@ func (s *Scope) AlterTableCopy(c *Compile) error { defer func() { c.proc.Ctx = restoreCtx }() - return c.runSqlWithOptions(qry.InsertTmpDataSql, opt) + return c.runSqlWithOptions(insertTmpDataSQL, opt) }() if err != nil { c.proc.Error(c.proc.Ctx, "insert data to copy table for alter table", zap.String("databaseName", dbName), zap.String("origin tableName", qry.GetTableDef().Name), zap.String("copy tableName", qry.CopyTableDef.Name), - zap.String("InsertTmpDataSql", qry.InsertTmpDataSql), + zap.String("InsertTmpDataSql", insertTmpDataSQL), zap.Error(err)) return err } @@ -455,6 +790,13 @@ func (s *Scope) AlterTableCopy(c *Compile) error { return err } + newId := newRel.GetTableID(c.proc.Ctx) + if err = c.preserveAlterDataBranchLineage( + lineagePlan, oldId, newId, dbName, tblName, + ); err != nil { + return err + } + // 7. drop original table. // ISCP: That will also drop ISCP related jobs and pitr of the original table. dropSql := fmt.Sprintf("drop table `%s`.`%s`", dbName, tblName) @@ -470,7 +812,6 @@ func (s *Scope) AlterTableCopy(c *Compile) error { return err } - newId := newRel.GetTableID(c.proc.Ctx) //------------------------------------------------------------------------- // 8. rename temporary replica table into the original table(Table Id remains unchanged) copyTblName := qry.CopyTableDef.Name diff --git a/pkg/sql/compile/alter_test.go b/pkg/sql/compile/alter_test.go index e965ea9922932..77140524761b7 100644 --- a/pkg/sql/compile/alter_test.go +++ b/pkg/sql/compile/alter_test.go @@ -35,6 +35,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/frontend/databranchutils" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" mock_lock "github.com/matrixorigin/matrixone/pkg/frontend/test/mock_lock" "github.com/matrixorigin/matrixone/pkg/lockservice" @@ -55,6 +56,85 @@ func TestShouldEnableAlterCopyPipelineFlush(t *testing.T) { assert.True(t, shouldEnableAlterCopyPipelineFlush(&plan2.AlterCopyOpt{SkipPkDedup: true})) } +func TestBuildAlterDataBranchLineageSQL(t *testing.T) { + metadataSQL, snapshotSQL := buildAlterDataBranchLineageSQL( + 11, 22, 123456, 7, + "alter:table", "tenant'o", "db'x", "tbl'y", "snapshot-id", + ) + + require.Equal(t, + "insert into mo_catalog.mo_branch_metadata values(22, 123456, 11, 7, 'alter:table', false)", + metadataSQL, + ) + require.Contains(t, snapshotSQL, "insert into mo_catalog.mo_snapshots") + require.Contains(t, snapshotSQL, "'snapshot-id', '__mo_branch_22', 123456") + require.Contains(t, snapshotSQL, "'tenant''o', 'db''x', 'tbl''y', 11, 'branch'") +} + +func TestAlterDataBranchHistoricalSourceSQL(t *testing.T) { + sql := alterDataBranchHistoricalSourceSQL("tenant'o", "db'x", "tbl'y", 42) + require.Contains(t, sql, "from mo_catalog.mo_snapshots where kind = 'user'") + require.Contains(t, sql, "from mo_catalog.mo_pitr where pitr_status = 1") + require.Contains(t, sql, "h.account_name = 'tenant''o'") + require.Contains(t, sql, "h.database_name = 'db''x'") + require.Contains(t, sql, "h.table_name = 'tbl''y'") + require.Contains(t, sql, "h.obj_id = 42") +} + +func TestAlterDataBranchLineageMetadata(t *testing.T) { + dag := databranchutils.NewBranchReclaimDag([]databranchutils.DataBranchMetadata{ + {TableID: 2, PTableID: 1, Creator: 9, Level: "table", TableDeleted: false}, + }) + + creator, level := alterDataBranchLineageMetadata(dag, 2) + require.Equal(t, uint32(9), creator) + require.Equal(t, "alter:table", level) + + creator, level = alterDataBranchLineageMetadata(dag, 1) + require.Equal(t, uint32(catalog.System_Account), creator) + require.Equal(t, "alter", level) +} + +func TestValidateAlterDataBranchLineageTxn(t *testing.T) { + require.NoError(t, validateAlterDataBranchLineageTxn(false, true, true)) + require.NoError(t, validateAlterDataBranchLineageTxn(false, true, false)) + + for _, tc := range []struct { + name string + byBegin bool + autocommit bool + pessimistic bool + want string + }{ + { + name: "explicit begin", + byBegin: true, + autocommit: true, + pessimistic: true, + want: "not supported inside an explicit transaction", + }, + { + name: "autocommit disabled", + autocommit: false, + pessimistic: true, + want: "not supported inside an explicit transaction", + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := validateAlterDataBranchLineageTxn(tc.byBegin, tc.autocommit, tc.pessimistic) + require.Error(t, err) + require.Contains(t, err.Error(), tc.want) + }) + } +} + +func TestShouldAdvanceAlterDataBranchLineageSnapshot(t *testing.T) { + require.True(t, shouldAdvanceAlterDataBranchLineageSnapshot(true, true)) + require.False(t, shouldAdvanceAlterDataBranchLineageSnapshot(true, false)) + require.False(t, shouldAdvanceAlterDataBranchLineageSnapshot(false, true)) + require.False(t, shouldAdvanceAlterDataBranchLineageSnapshot(false, false)) +} + type alterCopyInsertSpyExecutor struct { insertSQL string insertErr error @@ -446,6 +526,8 @@ func TestScopeAlterTableCopyPrecheckPrimaryKeyThenSkipDedup(t *testing.T) { require.True(t, spyExec.insertOption.AlterCopyDedupOpt().SkipPkDedup) require.Equal(t, alterTable.Options.TargetTableName, spyExec.insertOption.AlterCopyDedupOpt().TargetTableName) assert.Equal(t, []string{ + alterDataBranchParticipationSQL(1), + alterDataBranchHistoricalSourceSQL("", "test", "dept", 1), alterTable.CreateTmpTableSql, alterCopyTestPkNullCheckSQL, alterCopyTestPkDuplicateCheckSQL, diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index 85c6d7ca8ddfd..80cab16ddd57b 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -1,6 +1,14 @@ drop database if exists test; create database test; use test; +create table a(id int primary key, f0 double); +data branch create table b from a; +alter table b add column f1 double default 0.0; +data branch diff b against a; +diff b against a flag id f0 f1 +data branch merge b into a; +drop table a; +drop table b; create table t0(a int, b int, primary key(a)); insert into t0 values(1,1),(2,2),(3,3); create snapshot sp0 for table test t0; @@ -21,11 +29,13 @@ insert into t0 values(1,1),(2,2); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; alter table t1 add column c int default 0; +update t1 set b=22 where a=2; alter table t1 add column d varchar(20) default 'x'; insert into t1 values(3,3,30,'new'); create snapshot sp1 for table test t1; data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; diff t1 against t0 flag a b c d +t1 UPDATE 2 22 0 x t1 INSERT 3 3 30 new drop snapshot sp1; drop snapshot sp0; @@ -106,8 +116,8 @@ data branch create table t1 from t0{snapshot="sp0"}; alter table t1 drop column a; create snapshot sp1 for table test t1; data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; --- @regex("schema compatibility check: primary key column 'a' is not present in common columns", true) -internal error: schema compatibility check: primary key column 'a' is not present in common columns +-- @regex("schema compatibility check: target primary key columns .* do not match base primary key columns", true) +internal error: schema compatibility check: target primary key columns (__mo_fake_pk_col) do not match base primary key columns (a) drop snapshot sp1; drop snapshot sp0; drop table t0; @@ -129,13 +139,18 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; -create table t0(a int, b int) cluster by(a,b); +create table t0(a int primary key, b int) cluster by(a,b); +insert into t0 values(1,1),(2,2); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; alter table t1 add column c int default 0 after a; +update t1 set b=11 where a=1; +insert into t1(a,c,b) values(3,30,3); create snapshot sp1 for table test t1; data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; diff t1 against t0 flag a c b +t1 UPDATE 1 0 11 +t1 INSERT 3 30 3 drop snapshot sp1; drop snapshot sp0; drop table t0; @@ -153,4 +168,128 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t1 add column c int default 0; +data branch diff t1 against t0; +diff t1 against t0 flag a b c +t1 UPDATE 1 11 0 +drop table t0; +drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +delete from t1 where a=2; +alter table t1 add column c int default 0 first; +data branch diff t1 against t0; +diff t1 against t0 flag c a b +t1 DELETE null 2 2 +drop table t0; +drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 0; +update t0 set b=77 where a=1; +data branch diff t1 against t0; +diff t1 against t0 flag a b c +t0 UPDATE 1 77 null +drop table t0; +drop table t1; +create table t0(a int, b int); +insert into t0 values(1,1); +data branch create table t1 from t0; +alter table t1 add column c int default 0; +data branch diff t1 against t0; +-- @regex("schema compatibility check: target-only columns require an explicit primary key", true) +internal error: schema compatibility check: target-only columns require an explicit primary key +drop table t0; +drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t1 add column c int default 7; +update t1 set b=12 where a=1; +data branch diff t1 against t0; +diff t1 against t0 flag a b c +t1 UPDATE 1 12 7 +drop table t0; +drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +begin; +update t1 set b=11 where a=1; +alter table t1 add column c int default 0; +-- @regex("ALTER on a data-branch lineage is not supported inside an explicit transaction", true) +not supported: ALTER on a data-branch lineage is not supported inside an explicit transaction +rollback; +data branch diff t1 against t0; +diff t1 against t0 flag a b +drop table t0; +drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +drop snapshot if exists s0; +drop snapshot if exists s1; +create snapshot s0 for account sys; +update t1 set b=11 where a=1; +alter table t1 add column c int default 7; +update t1 set b=22 where a=2; +create snapshot s1 for account sys; +data branch diff t1{snapshot="s1"} against t1{snapshot="s0"}; +diff t1 against t1 flag a b c +t1 UPDATE 1 11 7 +t1 UPDATE 2 22 7 +drop snapshot s1; +drop snapshot s0; +drop table t0; +drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +drop snapshot if exists s0; +create snapshot s0 for account sys; +update t1 set b=11 where a=1; +alter table t1 add column c int default 7; +data branch create table t2 from t1{snapshot="s0"}; +update t2 set b=22 where a=2; +data branch diff t1 against t2; +diff t1 against t2 flag a b c +t1 UPDATE 1 11 7 +t2 UPDATE 2 22 null +drop snapshot s0; +drop table t2; +drop table t1; +drop table t0; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +drop snapshot if exists s0; +create snapshot s0 for account sys; +update t0 set b=11 where a=1; +alter table t0 add column c int default 7; +data branch create table t1 from t0{snapshot="s0"}; +update t1 set b=22 where a=2; +data branch diff t0 against t1; +diff t0 against t1 flag a b c +t0 UPDATE 1 11 7 +t1 UPDATE 2 22 null +drop snapshot s0; +drop table t1; +drop table t0; +create table t0(a int, b int); +insert into t0 values(1,1); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t0 add column c int default 0; +alter table t1 add column c int default 0; +data branch diff t1 against t0; +-- @regex("historical data branch fake primary key schema differs from the endpoint schema", true) +not supported: historical data branch fake primary key schema differs from the endpoint schema +drop table t1; +drop table t0; drop database test; diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 5b9eb90b6efee..9839d7722a1c0 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -2,6 +2,17 @@ drop database if exists test; create database test; use test; +-- ===================================================== +-- Case 0: exact regression for issue #24549 +-- ===================================================== +create table a(id int primary key, f0 double); +data branch create table b from a; +alter table b add column f1 double default 0.0; +data branch diff b against a; +data branch merge b into a; +drop table a; +drop table b; + -- ===================================================== -- Case 1: basic schema evolution - target has one extra column -- base: [a, b] @@ -37,6 +48,7 @@ create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; alter table t1 add column c int default 0; +update t1 set b=22 where a=2; alter table t1 add column d varchar(20) default 'x'; insert into t1 values(3,3,30,'new'); create snapshot sp1 for table test t1; @@ -167,7 +179,7 @@ data branch create table t1 from t0{snapshot="sp0"}; alter table t1 drop column a; create snapshot sp1 for table test t1; --- @regex("schema compatibility check: primary key column 'a' is not present in common columns", true) +-- @regex("schema compatibility check: target primary key columns .* do not match base primary key columns", true) data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; drop snapshot sp1; @@ -202,13 +214,17 @@ drop table t1; -- ===================================================== -- Case 10: cluster-by hidden helper columns stay out of SQL/output columns -- ===================================================== -create table t0(a int, b int) cluster by(a,b); +create table t0(a int primary key, b int) cluster by(a,b); +insert into t0 values(1,1),(2,2); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; alter table t1 add column c int default 0 after a; +update t1 set b=11 where a=1; +insert into t1(a,c,b) values(3,30,3); create snapshot sp1 for table test t1; +-- Common-column UPDATE and INSERT must not expose the hidden cluster-by helper. data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; drop snapshot sp1; @@ -235,4 +251,152 @@ drop snapshot sp0; drop table t0; drop table t1; +-- ===================================================== +-- Case 12: common-column UPDATE before ALTER is preserved +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t1 add column c int default 0; +data branch diff t1 against t0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 13: DELETE before ADD FIRST is preserved +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +delete from t1 where a=2; +alter table t1 add column c int default 0 first; +data branch diff t1 against t0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 14: base-side UPDATE after target ALTER keeps commit_ts +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 0; +update t0 set b=77 where a=1; +data branch diff t1 against t0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 15: fake-PK hash is incompatible across added columns +-- ===================================================== +create table t0(a int, b int); +insert into t0 values(1,1); +data branch create table t1 from t0; +alter table t1 add column c int default 0; +-- @regex("schema compatibility check: target-only columns require an explicit primary key", true) +data branch diff t1 against t0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 16: same PK updated before and after ALTER is emitted once +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t1 add column c int default 7; +update t1 set b=12 where a=1; +data branch diff t1 against t0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 17: ALTER on live data-branch lineage rejects explicit transactions +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +begin; +update t1 set b=11 where a=1; +-- @regex("ALTER on a data-branch lineage is not supported inside an explicit transaction", true) +alter table t1 add column c int default 0; +rollback; +data branch diff t1 against t0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 18: snapshot diff spans an ALTER physical generation +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +drop snapshot if exists s0; +drop snapshot if exists s1; +create snapshot s0 for account sys; +update t1 set b=11 where a=1; +alter table t1 add column c int default 7; +update t1 set b=22 where a=2; +create snapshot s1 for account sys; +data branch diff t1{snapshot="s1"} against t1{snapshot="s0"}; +drop snapshot s1; +drop snapshot s0; +drop table t0; +drop table t1; + +-- ===================================================== +-- Case 19: branch-from-snapshot records the snapshot physical parent +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +drop snapshot if exists s0; +create snapshot s0 for account sys; +update t1 set b=11 where a=1; +alter table t1 add column c int default 7; +data branch create table t2 from t1{snapshot="s0"}; +update t2 set b=22 where a=2; +-- t2 must fork from t1's old physical ID captured by s0, not current t1. +data branch diff t1 against t2; +drop snapshot s0; +drop table t2; +drop table t1; +drop table t0; + +-- ===================================================== +-- Case 20: root ALTER is retained for a future snapshot branch +-- ===================================================== +-- No branch metadata exists when s0 is created. ALTER must still retain +-- old->new lineage because the user snapshot can become a branch parent. +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +drop snapshot if exists s0; +create snapshot s0 for account sys; +update t0 set b=11 where a=1; +alter table t0 add column c int default 7; +data branch create table t1 from t0{snapshot="s0"}; +update t1 set b=22 where a=2; +data branch diff t0 against t1; +drop snapshot s0; +drop table t1; +drop table t0; + +-- ===================================================== +-- Case 21: historical fake-PK generations fail closed +-- ===================================================== +-- Endpoint schemas match, but old fake PKs hash (a,b) while endpoint fake +-- PKs hash (a,b,c); historical rows cannot be matched safely. +create table t0(a int, b int); +insert into t0 values(1,1); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t0 add column c int default 0; +alter table t1 add column c int default 0; +-- @regex("historical data branch fake primary key schema differs from the endpoint schema", true) +data branch diff t1 against t0; +drop table t1; +drop table t0; + drop database test; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result index bae92b6d051b3..54744d2ade456 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result @@ -68,10 +68,8 @@ update both_add_left set c = 'left' where a = 2; update both_add_right set c = 'right' where a = 3; data branch diff both_add_left against both_add_right columns (a, c); diff both_add_left against both_add_right flag a c -both_add_left INSERT 2 left -both_add_right INSERT 2 same -both_add_left INSERT 3 same -both_add_right INSERT 3 right +both_add_left UPDATE 2 left +both_add_right UPDATE 3 right data branch create table type_left from base; data branch create table type_right from base; alter table type_left add column c int default 0; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql index 70915d3709644..1ca2b51b6761a 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql @@ -70,6 +70,8 @@ alter table both_add_left add column c varchar(20) default 'same'; alter table both_add_right add column c varchar(20) default 'same'; update both_add_left set c = 'left' where a = 2; update both_add_right set c = 'right' where a = 3; +-- ALTER preserves branch lineage: changes to inherited rows remain UPDATEs, +-- and only the side that changed each row is emitted. data branch diff both_add_left against both_add_right columns (a, c); data branch create table type_left from base; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index 3929b91aeb3d9..ff2ea6a271c48 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -116,16 +116,36 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; -create table t0(a int, b int) cluster by(a,b); +create table t0(a int primary key, b int) cluster by(a,b); +insert into t0 values(1,1),(2,2); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; alter table t1 add column c int default 0 after a; +update t1 set b=11 where a=1; +insert into t1(a,c,b) values(3,30,3); create snapshot sp1 for table test t1; data branch merge t1 into t0 when conflict accept; select * from t0 order by a; a b +1 11 +2 2 +3 3 drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2),(3,3); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t1 add column c int default 0; +delete from t1 where a=2; +alter table t1 add column d varchar(20) default 'x'; +data branch merge t1 into t0; +select * from t0 order by a; +a b +1 11 +3 3 +drop table t0; +drop table t1; drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index 0690723048070..a6f8dd1b052b6 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -151,11 +151,14 @@ drop table t1; -- ===================================================== -- Case 7: cluster-by hidden helper columns stay out of SQL/apply columns -- ===================================================== -create table t0(a int, b int) cluster by(a,b); +create table t0(a int primary key, b int) cluster by(a,b); +insert into t0 values(1,1),(2,2); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; alter table t1 add column c int default 0 after a; +update t1 set b=11 where a=1; +insert into t1(a,c,b) values(3,30,3); create snapshot sp1 for table test t1; data branch merge t1 into t0 when conflict accept; @@ -166,4 +169,21 @@ drop snapshot sp0; drop table t0; drop table t1; +-- ===================================================== +-- Case 8: DML across multiple ALTER generations is preserved +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2),(3,3); +data branch create table t1 from t0; +update t1 set b=11 where a=1; +alter table t1 add column c int default 0; +delete from t1 where a=2; +alter table t1 add column d varchar(20) default 'x'; + +data branch merge t1 into t0; +select * from t0 order by a; + +drop table t0; +drop table t1; + drop database test; diff --git a/test/distributed/cases/git4data/branch/pick/pick_9.result b/test/distributed/cases/git4data/branch/pick/pick_9.result index 6ac893980c30a..6c9ad7bee132e 100644 --- a/test/distributed/cases/git4data/branch/pick/pick_9.result +++ b/test/distributed/cases/git4data/branch/pick/pick_9.result @@ -125,4 +125,72 @@ drop snapshot sp1; drop snapshot sp2; drop table t1; drop table t2; +drop snapshot if exists sp_schema_from; +drop snapshot if exists sp_schema_to; +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1); +data branch create table t1 from t0; +create snapshot sp_schema_from for account sys; +update t1 set b=10 where a=1; +alter table t1 add column c int default 7; +create snapshot sp_schema_to for account sys; +update t1 set b=20 where a=1; +data branch pick t1 into t0 between snapshot sp_schema_from and sp_schema_to; +select * from t0 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] 𝄀 +1 ¦ 10 +select * from t1 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] ¦ c[4,32,0] 𝄀 +1 ¦ 20 ¦ 7 +drop snapshot sp_schema_from; +drop snapshot sp_schema_to; +drop table t0; +drop table t1; +drop snapshot if exists sp_schema_from; +drop snapshot if exists sp_schema_to; +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1); +data branch create table t1 from t0; +create snapshot sp_schema_from for account sys; +update t1 set b=15 where a=1; +create snapshot sp_schema_to for account sys; +alter table t1 add column c int default 7; +update t1 set b=25 where a=1; +data branch pick t1 into t0 between snapshot sp_schema_from and sp_schema_to; +select * from t0 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] 𝄀 +1 ¦ 15 +select * from t1 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] ¦ c[4,32,0] 𝄀 +1 ¦ 25 ¦ 7 +drop snapshot sp_schema_from; +drop snapshot sp_schema_to; +drop table t0; +drop table t1; +drop snapshot if exists sp_nolca_from; +drop snapshot if exists sp_nolca_to; +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1),(2,2); +data branch create table t1 from t0; +create table t2 (a int, b int, c int default 7, primary key(a)); +insert into t2 values (1,1,7),(2,200,7); +create snapshot sp_nolca_from for account sys; +update t1 set b=15 where a=1; +create snapshot sp_nolca_to for account sys; +alter table t1 add column c int default 7; +update t1 set b=25 where a=1; +data branch pick t1 into t2 between snapshot sp_nolca_from and sp_nolca_to when conflict accept; +select * from t2 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] ¦ c[4,32,0] 𝄀 +1 ¦ 15 ¦ null 𝄀 +2 ¦ 200 ¦ 7 +select * from t1 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] ¦ c[4,32,0] 𝄀 +1 ¦ 25 ¦ 7 𝄀 +2 ¦ 2 ¦ 7 +drop snapshot sp_nolca_from; +drop snapshot sp_nolca_to; +drop table t2; +drop table t1; +drop table t0; drop database test; diff --git a/test/distributed/cases/git4data/branch/pick/pick_9.sql b/test/distributed/cases/git4data/branch/pick/pick_9.sql index 5de885fdccf1a..96e6733c359e7 100644 --- a/test/distributed/cases/git4data/branch/pick/pick_9.sql +++ b/test/distributed/cases/git4data/branch/pick/pick_9.sql @@ -215,4 +215,100 @@ drop snapshot sp2; drop table t1; drop table t2; +-- ---------------------------------------------------------------- +-- case g: BETWEEN upper bound after ALTER uses the bounded row image +-- ---------------------------------------------------------------- +-- The same PK changes in the old physical generation and again after +-- ALTER. PICK must hydrate the old change at sp_schema_to, not from the +-- current source row written after that snapshot. + +drop snapshot if exists sp_schema_from; +drop snapshot if exists sp_schema_to; + +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1); + +data branch create table t1 from t0; + +create snapshot sp_schema_from for account sys; +update t1 set b=10 where a=1; +alter table t1 add column c int default 7; +create snapshot sp_schema_to for account sys; +update t1 set b=20 where a=1; + +data branch pick t1 into t0 between snapshot sp_schema_from and sp_schema_to; +select * from t0 order by a asc; +-- expect destination b=10 at the upper bound; current source b=20 is later +select * from t1 order by a asc; + +drop snapshot sp_schema_from; +drop snapshot sp_schema_to; +drop table t0; +drop table t1; + +-- ---------------------------------------------------------------- +-- case h: BETWEEN upper bound before ALTER scans only the old generation +-- ---------------------------------------------------------------- +-- The current physical table does not exist at sp_schema_to. PICK must +-- resolve the old physical generation and apply its b=15 row image. + +drop snapshot if exists sp_schema_from; +drop snapshot if exists sp_schema_to; + +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1); + +data branch create table t1 from t0; + +create snapshot sp_schema_from for account sys; +update t1 set b=15 where a=1; +create snapshot sp_schema_to for account sys; +alter table t1 add column c int default 7; +update t1 set b=25 where a=1; + +data branch pick t1 into t0 between snapshot sp_schema_from and sp_schema_to; +select * from t0 order by a asc; +-- expect destination b=15; ALTER and b=25 are outside the range +select * from t1 order by a asc; + +drop snapshot sp_schema_from; +drop snapshot sp_schema_to; +drop table t0; +drop table t1; + +-- ---------------------------------------------------------------- +-- case i: no-LCA BETWEEN across ALTER excludes materialization +-- ---------------------------------------------------------------- +-- t2 is schema-compatible with current t1 but independent of its DAG. +-- Only a=1 changes inside the bounded old-generation window; a=2 must +-- keep its independent destination value even though ALTER copies it. + +drop snapshot if exists sp_nolca_from; +drop snapshot if exists sp_nolca_to; + +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1),(2,2); +data branch create table t1 from t0; + +create table t2 (a int, b int, c int default 7, primary key(a)); +insert into t2 values (1,1,7),(2,200,7); + +create snapshot sp_nolca_from for account sys; +update t1 set b=15 where a=1; +create snapshot sp_nolca_to for account sys; +alter table t1 add column c int default 7; +update t1 set b=25 where a=1; + +data branch pick t1 into t2 between snapshot sp_nolca_from and sp_nolca_to when conflict accept; +select * from t2 order by a asc; +-- expect a=1 at bounded value 15 with c=NULL (column not yet present); +-- a=2 remains independent value 200 +select * from t1 order by a asc; + +drop snapshot sp_nolca_from; +drop snapshot sp_nolca_to; +drop table t2; +drop table t1; +drop table t0; + drop database test; From f0a5cb88557178887475e94ffc26025dcf0e6922 Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 15 Jul 2026 14:03:53 +0800 Subject: [PATCH 15/31] test(frontend): cover sparse schema conflict updates --- pkg/frontend/data_branch_hashdiff_test.go | 67 +++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index e8f9ad7c27b94..b2960ce9cfaff 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -748,6 +748,73 @@ func TestDiffDataHelper_ConflictAcceptExpandUpdate(t *testing.T) { require.Equal(t, [][]any{{int64(7), "before", "base"}}, got["DELETE-2"]) } +func TestDiffDataHelper_ConflictAcceptExpandUpdateWithSparseCommonIdxes(t *testing.T) { + ses := newValidateSession(t) + mp := ses.proc.Mp() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + tblStuff.def.colNames = []string{"id", "target_only", "name"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), + types.T_varchar.ToType(), + types.T_varchar.ToType(), + } + tblStuff.def.visibleIdxes = []int{0, 1, 2} + tblStuff.def.commonIdxes = []int{0, 2} + tblStuff.def.commonVisibleIdxes = []int{0, 2} + tblStuff.def.tarOnlyIdxes = []int{1} + + // Both sides changed the same PK after branching. The target-only value is + // identical, while the trailing common column differs; conflict detection + // must therefore compare target physical index 2, not dense ordinal 1. + tarHashmap := buildTestBranchHashmap( + t, mp, tblStuff.def.colTypes, + [][]any{{int64(7), "same-extra", "target-update"}}, + ) + defer func() { + require.NoError(t, tarHashmap.Close()) + }() + baseHashmap := buildTestBranchHashmap( + t, mp, tblStuff.def.colTypes, + [][]any{{int64(7), "same-extra", "base-update"}}, + ) + defer func() { + require.NoError(t, baseHashmap.Close()) + }() + + got := make(map[string][][]any) + var mu sync.Mutex + err := diffDataHelper( + context.Background(), + ses, + compositeOption{ + conflictOpt: &tree.ConflictOpt{Opt: tree.CONFLICT_ACCEPT}, + expandUpdate: true, + }, + tblStuff, + func(w batchWithKind) (bool, error) { + rows := decodeCapturedRows(t, w.batch, tblStuff.def.colTypes) + tblStuff.retPool.releaseRetBatch(w.batch, false) + if len(rows) == 0 { + return false, nil + } + + mu.Lock() + key := fmt.Sprintf("%s-%d", w.kind, w.side) + got[key] = append(got[key], rows...) + mu.Unlock() + return false, nil + }, + tarHashmap, + baseHashmap, + ) + require.NoError(t, err) + require.Equal(t, [][]any{{int64(7), "same-extra", "target-update"}}, got["INSERT-1"]) + require.Equal(t, [][]any{{int64(7), "same-extra", "base-update"}}, got["DELETE-2"]) +} + type closeTrackingBranchHashmap struct { migrated databranchutils.BranchHashmap migrateCalls int From e8ff378412fa91a18c103ba38d8c6631eb81cbcf Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 15 Jul 2026 14:23:27 +0800 Subject: [PATCH 16/31] test(frontend): fix cluster-by schema evolution BVT --- .../branch/diff/diff_schema_evolve.result | 17 +++++------------ .../branch/diff/diff_schema_evolve.sql | 19 ++++++++----------- .../branch/merge/merge_schema_evolve.result | 17 ++++------------- .../branch/merge/merge_schema_evolve.sql | 17 +++++++---------- 4 files changed, 24 insertions(+), 46 deletions(-) diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index 80cab16ddd57b..2447671eb188a 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -139,20 +139,13 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; -create table t0(a int primary key, b int) cluster by(a,b); +create table t0(a int, b int) cluster by(a,b); insert into t0 values(1,1),(2,2); -create snapshot sp0 for table test t0; -data branch create table t1 from t0{snapshot="sp0"}; +data branch create table t1 from t0; alter table t1 add column c int default 0 after a; -update t1 set b=11 where a=1; -insert into t1(a,c,b) values(3,30,3); -create snapshot sp1 for table test t1; -data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; -diff t1 against t0 flag a c b -t1 UPDATE 1 0 11 -t1 INSERT 3 30 3 -drop snapshot sp1; -drop snapshot sp0; +data branch diff t1 against t0; +-- @regex("schema compatibility check: target-only columns require an explicit primary key", true) +internal error: schema compatibility check: target-only columns require an explicit primary key drop table t0; drop table t1; create table t0(a int, b int, primary key(a)); diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 9839d7722a1c0..63ae77c35f15d 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -212,23 +212,20 @@ drop table t0; drop table t1; -- ===================================================== --- Case 10: cluster-by hidden helper columns stay out of SQL/output columns +-- Case 10: cluster-by + added column reaches the fake-PK rejection boundary -- ===================================================== -create table t0(a int primary key, b int) cluster by(a,b); +-- MatrixOne does not allow an explicit primary key together with CLUSTER BY. +-- A legal cluster-by table therefore uses the fake PK, and adding a target-only +-- column is rejected before DIFF output can expose any hidden helper column. +create table t0(a int, b int) cluster by(a,b); insert into t0 values(1,1),(2,2); -create snapshot sp0 for table test t0; -data branch create table t1 from t0{snapshot="sp0"}; +data branch create table t1 from t0; alter table t1 add column c int default 0 after a; -update t1 set b=11 where a=1; -insert into t1(a,c,b) values(3,30,3); -create snapshot sp1 for table test t1; --- Common-column UPDATE and INSERT must not expose the hidden cluster-by helper. -data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; +-- @regex("schema compatibility check: target-only columns require an explicit primary key", true) +data branch diff t1 against t0; -drop snapshot sp1; -drop snapshot sp0; drop table t0; drop table t1; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index ff2ea6a271c48..801e55a41727d 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -116,22 +116,13 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; -create table t0(a int primary key, b int) cluster by(a,b); +create table t0(a int, b int) cluster by(a,b); insert into t0 values(1,1),(2,2); -create snapshot sp0 for table test t0; -data branch create table t1 from t0{snapshot="sp0"}; +data branch create table t1 from t0; alter table t1 add column c int default 0 after a; -update t1 set b=11 where a=1; -insert into t1(a,c,b) values(3,30,3); -create snapshot sp1 for table test t1; data branch merge t1 into t0 when conflict accept; -select * from t0 order by a; -a b -1 11 -2 2 -3 3 -drop snapshot sp1; -drop snapshot sp0; +-- @regex("schema compatibility check: target-only columns require an explicit primary key", true) +internal error: schema compatibility check: target-only columns require an explicit primary key drop table t0; drop table t1; create table t0(a int primary key, b int); diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index a6f8dd1b052b6..be019390d784b 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -149,23 +149,20 @@ drop table t0; drop table t1; -- ===================================================== --- Case 7: cluster-by hidden helper columns stay out of SQL/apply columns +-- Case 7: cluster-by + added column reaches the fake-PK rejection boundary -- ===================================================== -create table t0(a int primary key, b int) cluster by(a,b); +-- MatrixOne does not allow an explicit primary key together with CLUSTER BY. +-- A legal cluster-by table therefore uses the fake PK, and adding a target-only +-- column is rejected before MERGE apply can expose any hidden helper column. +create table t0(a int, b int) cluster by(a,b); insert into t0 values(1,1),(2,2); -create snapshot sp0 for table test t0; -data branch create table t1 from t0{snapshot="sp0"}; +data branch create table t1 from t0; alter table t1 add column c int default 0 after a; -update t1 set b=11 where a=1; -insert into t1(a,c,b) values(3,30,3); -create snapshot sp1 for table test t1; +-- @regex("schema compatibility check: target-only columns require an explicit primary key", true) data branch merge t1 into t0 when conflict accept; -select * from t0 order by a; -drop snapshot sp1; -drop snapshot sp0; drop table t0; drop table t1; From 31713c17b218835d77f7252832d6efe76d46130f Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 15 Jul 2026 14:41:54 +0800 Subject: [PATCH 17/31] test(frontend): isolate lineage alter transaction BVT --- .../cases/git4data/branch/diff/diff_schema_evolve.result | 1 - .../cases/git4data/branch/diff/diff_schema_evolve.sql | 1 - 2 files changed, 2 deletions(-) diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index 2447671eb188a..a498db5d03e0a 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -215,7 +215,6 @@ create table t0(a int primary key, b int); insert into t0 values(1,1),(2,2); data branch create table t1 from t0; begin; -update t1 set b=11 where a=1; alter table t1 add column c int default 0; -- @regex("ALTER on a data-branch lineage is not supported inside an explicit transaction", true) not supported: ALTER on a data-branch lineage is not supported inside an explicit transaction diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 63ae77c35f15d..91c288e3fb749 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -316,7 +316,6 @@ create table t0(a int primary key, b int); insert into t0 values(1,1),(2,2); data branch create table t1 from t0; begin; -update t1 set b=11 where a=1; -- @regex("ALTER on a data-branch lineage is not supported inside an explicit transaction", true) alter table t1 add column c int default 0; rollback; From 7f51dafc2167eb5bfa672418e8290abe01f1c301 Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 15 Jul 2026 14:46:23 +0800 Subject: [PATCH 18/31] test(frontend): restore lineage transaction update case --- .../cases/git4data/branch/diff/diff_schema_evolve.result | 1 + .../cases/git4data/branch/diff/diff_schema_evolve.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index a498db5d03e0a..2447671eb188a 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -215,6 +215,7 @@ create table t0(a int primary key, b int); insert into t0 values(1,1),(2,2); data branch create table t1 from t0; begin; +update t1 set b=11 where a=1; alter table t1 add column c int default 0; -- @regex("ALTER on a data-branch lineage is not supported inside an explicit transaction", true) not supported: ALTER on a data-branch lineage is not supported inside an explicit transaction diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 91c288e3fb749..63ae77c35f15d 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -316,6 +316,7 @@ create table t0(a int primary key, b int); insert into t0 values(1,1),(2,2); data branch create table t1 from t0; begin; +update t1 set b=11 where a=1; -- @regex("ALTER on a data-branch lineage is not supported inside an explicit transaction", true) alter table t1 add column c int default 0; rollback; From e0cc76709eb57b46bbf4e52faca53606b6e7e5c0 Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 15 Jul 2026 16:08:55 +0800 Subject: [PATCH 19/31] fix(frontend): use live context for branch clone lock --- pkg/frontend/clone.go | 39 ++++++++++++++++++++++++++------------ pkg/frontend/clone_test.go | 20 +++++++++++++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/pkg/frontend/clone.go b/pkg/frontend/clone.go index 14d18f26b7764..2531d0ccb323b 100644 --- a/pkg/frontend/clone.go +++ b/pkg/frontend/clone.go @@ -80,6 +80,19 @@ type cloneReceipt struct { type dataBranchCloneLockCtxKey struct{} +func withDataBranchCloneLockContext( + proc *process.Process, + ctx context.Context, + lockRows func() error, +) error { + oldCtx := proc.Ctx + proc.Ctx = ctx + defer func() { + proc.Ctx = oldCtx + }() + return lockRows() +} + func lockDataBranchCloneSource( ctx context.Context, ses *Session, @@ -118,18 +131,20 @@ func lockDataBranchCloneSource( // ALTER locks this exact mo_tables composite key exclusively. A shared // catalog-row lock serializes source-ID/snapshot selection with ALTER while // allowing source-table DML and sibling branch clones to continue. - return lockop.LockRows( - eng, - ses.proc, - rel, - rel.GetTableID(sourceCtx), - lockBat, - 0, - *lockBat.Vecs[0].GetType(), - lock.LockMode_Shared, - lock.Sharding_None, - fromAccountID, - ) + return withDataBranchCloneLockContext(ses.proc, sourceCtx, func() error { + return lockop.LockRows( + eng, + ses.proc, + rel, + rel.GetTableID(sourceCtx), + lockBat, + 0, + *lockBat.Vecs[0].GetType(), + lock.LockMode_Shared, + lock.Sharding_None, + fromAccountID, + ) + }) } func dataBranchCloneCatalogLockBatch( diff --git a/pkg/frontend/clone_test.go b/pkg/frontend/clone_test.go index 541251c97c2f4..e09bac27250fb 100644 --- a/pkg/frontend/clone_test.go +++ b/pkg/frontend/clone_test.go @@ -15,6 +15,8 @@ package frontend import ( + "context" + "errors" "testing" "github.com/stretchr/testify/require" @@ -23,6 +25,24 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) +func TestWithDataBranchCloneLockContext(t *testing.T) { + proc := newValidateSession(t).proc + oldCtx, cancel := context.WithCancel(context.Background()) + cancel() + proc.Ctx = oldCtx + + lockCtx := context.WithValue(context.Background(), struct{}{}, "current") + wantErr := errors.New("lock failed") + err := withDataBranchCloneLockContext(proc, lockCtx, func() error { + require.Same(t, lockCtx, proc.Ctx) + require.NoError(t, proc.Ctx.Err()) + return wantErr + }) + + require.ErrorIs(t, err, wantErr) + require.Same(t, oldCtx, proc.Ctx) +} + func TestDataBranchCloneCatalogLockBatch(t *testing.T) { ses := newValidateSession(t) mp := ses.proc.Mp() From 11def753298ff3fb21d9d0c5517cc073f5d8e2a6 Mon Sep 17 00:00:00 2001 From: ULookup Date: Wed, 15 Jul 2026 17:47:51 +0800 Subject: [PATCH 20/31] fix(frontend): preserve ancestor branch updates --- pkg/frontend/data_branch.go | 26 ++++++++++++++++------- pkg/frontend/data_branch_hashdiff.go | 17 +++++++++++++++ pkg/frontend/data_branch_hashdiff_test.go | 9 ++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index ad874dc13c66a..bbe02930c19e0 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -1558,14 +1558,19 @@ func constructChangeHandle( for i := range tarRange.rel { collectStart := time.Now() - var sourceMapping []int - isHistorical := tarRange.rel[i].GetTableID(ctx) != tables.tarRel.GetTableID(ctx) - if isHistorical { + var ( + sourceMapping []int + needsProjection bool + ) + if tarRange.rel[i].GetTableID(ctx) != tables.tarRel.GetTableID(ctx) { if sourceMapping, err = dataBranchSourceColToTargetIdx( tarRange.rel[i].GetTableDef(ctx), targetDef, tables.def.colNames, ); err != nil { return } + needsProjection = dataBranchNeedsHistoricalProjection( + sourceMapping, len(tables.def.colNames), + ) } if handle, err = collectFn( ctx, @@ -1586,7 +1591,7 @@ func constructChangeHandle( ) if handle != nil { - if isHistorical { + if needsProjection { handle = &historicalDataBranchChangesHandle{ inner: handle, sourceMapping: sourceMapping, @@ -1602,14 +1607,19 @@ func constructChangeHandle( for i := range baseRange.rel { collectStart := time.Now() - var sourceMapping []int - isHistorical := baseRange.rel[i].GetTableID(ctx) != tables.baseRel.GetTableID(ctx) - if isHistorical { + var ( + sourceMapping []int + needsProjection bool + ) + if baseRange.rel[i].GetTableID(ctx) != tables.baseRel.GetTableID(ctx) { if sourceMapping, err = dataBranchSourceColToTargetIdx( baseRange.rel[i].GetTableDef(ctx), targetDef, tables.def.colNames, ); err != nil { return } + needsProjection = dataBranchNeedsHistoricalProjection( + sourceMapping, len(tables.def.colNames), + ) } if handle, err = collectFn( ctx, @@ -1630,7 +1640,7 @@ func constructChangeHandle( ) if handle != nil { - if isHistorical { + if needsProjection { handle = &historicalDataBranchChangesHandle{ inner: handle, sourceMapping: sourceMapping, diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index bbeed9c26f23f..585fe62964a00 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -2630,6 +2630,23 @@ func dataBranchSourceColToTargetIdx( return mapping, nil } +// dataBranchNeedsHistoricalProjection reports whether change rows from an +// older physical table generation have a different data-column layout from +// the endpoint. A different table ID alone can also mean an ordinary branch +// ancestor with an identical schema; projecting and hydrating those rows would +// collapse intermediate UPDATE versions to endpoint values. +func dataBranchNeedsHistoricalProjection(sourceToTarget []int, targetColCount int) bool { + if len(sourceToTarget) != targetColCount { + return true + } + for sourceIdx, targetIdx := range sourceToTarget { + if sourceIdx != targetIdx { + return true + } + } + return false +} + func dataBranchTargetLayoutAttrs(tblStuff *tableStuff, hasCommitTS bool) []string { attrs := make([]string, 0, len(tblStuff.def.colNames)+2) attrs = append(attrs, catalog.Row_ID) diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index b2960ce9cfaff..f5f443b489196 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -2470,6 +2470,15 @@ func TestDataBranchSourceColToTargetIdxRejectsHistoricalFakePrimaryKeyDrift(t *t require.Contains(t, err.Error(), "historical data branch fake primary key schema differs") } +func TestDataBranchNeedsHistoricalProjection(t *testing.T) { + require.False(t, dataBranchNeedsHistoricalProjection([]int{0, 1}, 2), + "a lineage ancestor with the endpoint layout must keep its raw change rows") + require.True(t, dataBranchNeedsHistoricalProjection([]int{0, 2}, 3), + "a target-only column requires projection and endpoint hydration") + require.True(t, dataBranchNeedsHistoricalProjection([]int{1, 0}, 2), + "a reordered layout requires projection") +} + func TestOverlayDataBranchProbeResultHydratesTargetDefaults(t *testing.T) { ses := newValidateSession(t) mp := ses.proc.Mp() From 04b977c4cd60402125fc22199db5e7acd72a1a73 Mon Sep 17 00:00:00 2001 From: ULookup Date: Thu, 16 Jul 2026 12:50:39 +0800 Subject: [PATCH 21/31] fix(frontend): preserve bounded historical branch updates --- pkg/frontend/data_branch.go | 4 + pkg/frontend/data_branch_hashdiff.go | 37 +++---- pkg/frontend/data_branch_hashdiff_test.go | 98 +++++++++++++++++++ pkg/frontend/data_branch_helpers.go | 50 ++++++++-- pkg/frontend/data_branch_helpers_test.go | 40 ++++++++ .../cases/git4data/branch/pick/pick_9.result | 28 ++++++ .../cases/git4data/branch/pick/pick_9.sql | 36 +++++++ 7 files changed, 261 insertions(+), 32 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index bbe02930c19e0..f60de0dedf0e2 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -1599,6 +1599,8 @@ func constructChangeHandle( ses: ses, endpointRel: tarHydrationRel, endpointSnapshot: tarHydrationSnapshot, + hydrate: tarRange.rel[i].GetTableID(ctx) != + tarHydrationRel.GetTableID(ctx), } } tarHandle = append(tarHandle, handle) @@ -1648,6 +1650,8 @@ func constructChangeHandle( ses: ses, endpointRel: tables.baseRel, endpointSnapshot: baseSnapshot, + hydrate: baseRange.rel[i].GetTableID(ctx) != + tables.baseRel.GetTableID(ctx), } } baseHandle = append(baseHandle, handle) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 585fe62964a00..9b912cb2d3ea8 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -580,12 +580,13 @@ func runLCAProbeWithReaderFallback( }() scanStart := time.Now() - err = scanSnapshotRelationByID( + err = scanSnapshotRelationByIDWithFallback( ctx, "lca-reader-fallback", ses, tblStuff.lcaRel.GetTableID(ctx), snapshotTS, + tblStuff.lcaRel, lcaLayout.attrs, lcaLayout.types, pkFilterExpr, @@ -1253,31 +1254,15 @@ func hashDiffIfNoLCA( copt compositeOption, emit emitFunc, tarDataHashmap databranchutils.BranchHashmap, - tarTombstoneHashmap databranchutils.BranchHashmap, + _ databranchutils.BranchHashmap, baseDataHashmap databranchutils.BranchHashmap, - baseTombstoneHashmap databranchutils.BranchHashmap, + _ databranchutils.BranchHashmap, ) (err error) { - - if err = tarTombstoneHashmap.ForEachShardParallel(func(cursor databranchutils.ShardCursor) error { - return cursor.ForEach(func(key []byte, _ []byte) error { - _, err2 := tarDataHashmap.PopByEncodedKey(key, true) - return err2 - }) - - }, -1); err != nil { - return - } - - if err = baseTombstoneHashmap.ForEachShardParallel(func(cursor databranchutils.ShardCursor) error { - return cursor.ForEach(func(key []byte, _ []byte) error { - _, err2 := baseDataHashmap.PopByEncodedKey(key, true) - return err2 - }) - - }, -1); err != nil { - return - } - + // buildHashmapForTable already reconciles data versions with tombstones by + // commit timestamp. Any data row left for a key is the latest live version. + // In particular, a bounded UPDATE produces a same-commit data+tombstone + // pair; removing the data merely because the marker remains would erase the + // update before the two independent tables are compared. return diffDataHelper(ctx, ses, copt, tblStuff, emit, tarDataHashmap, baseDataHashmap) } @@ -2790,6 +2775,7 @@ type historicalDataBranchChangesHandle struct { ses *Session endpointRel engine.Relation endpointSnapshot types.TS + hydrate bool } func (h *historicalDataBranchChangesHandle) Next( @@ -2801,6 +2787,9 @@ func (h *historicalDataBranchChangesHandle) Next( return data, tombstone, hint, err } data = projectDataBranchBatchToTarget(data, &h.tblStuff, h.sourceMapping, mp) + if !h.hydrate { + return data, tombstone, hint, nil + } if err = hydrateHistoricalDataBranchBatch( ctx, h.ses, data, h.tblStuff, h.endpointRel, h.endpointSnapshot, ); err != nil { diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index f5f443b489196..dc87a4c3966ad 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -1597,6 +1597,66 @@ func TestHashDiff_NoLCAWithStubHandles(t *testing.T) { require.Equal(t, [][]any{{int64(3), "base-only", "hb"}}, got[1].rows) } +func TestHashDiff_NoLCABoundedUpdateKeepsLatestRow(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + worker, err := ants.NewPool(1) + require.NoError(t, err) + defer worker.Release() + tblStuff.worker = worker + tblStuff.maxTombstoneBatchCnt = 1 + tblStuff.hashmapAllocator = newBranchHashmapAllocator(dataBranchHashmapLimitRate) + + updateTS := types.BuildTS(15, 0) + tarData := buildHashDiffDataBatch(t, ses.proc.Mp(), [][]any{ + {int64(1), "bounded", "h1", commitTSBytes(updateTS)}, + }) + tarTombstone := buildHashDiffTombstoneBatch(t, ses.proc.Mp(), [][]any{ + {int64(1), commitTSBytes(updateTS)}, + }) + baseData := buildHashDiffDataBatch(t, ses.proc.Mp(), [][]any{ + {int64(1), "destination", "h1", commitTSBytes(types.BuildTS(5, 0))}, + }) + + var got []capturedBatch + var mu sync.Mutex + err = hashDiff( + context.Background(), ses, nil, tblStuff, branchMetaInfo{}, + compositeOption{ + conflictOpt: &tree.ConflictOpt{Opt: tree.CONFLICT_ACCEPT}, + expandUpdate: true, + }, + func(w batchWithKind) (bool, error) { + rows := decodeCapturedRows(t, w.batch, tblStuff.def.colTypes) + mu.Lock() + if len(rows) > 0 { + got = append(got, capturedBatch{kind: w.kind, side: w.side, rows: rows}) + } + mu.Unlock() + tblStuff.retPool.releaseRetBatch(w.batch, false) + return false, nil + }, + []engine.ChangesHandle{&stubEngineChangesHandle{responses: []stubEngineChangesHandleResponse{{ + data: tarData, tombstone: tarTombstone, + }}}}, + []engine.ChangesHandle{&stubEngineChangesHandle{responses: []stubEngineChangesHandleResponse{{ + data: baseData, + }}}}, + nil, + ) + require.NoError(t, err) + require.Len(t, got, 2) + require.Equal(t, diffDelete, got[0].kind) + require.Equal(t, diffSideBase, got[0].side) + require.Equal(t, [][]any{{int64(1), "destination", "h1"}}, got[0].rows) + require.Equal(t, diffInsert, got[1].kind) + require.Equal(t, diffSideTarget, got[1].side) + require.Equal(t, [][]any{{int64(1), "bounded", "h1"}}, got[1].rows) +} + func TestHashDiff_HasLCANoopUpdateFiltering(t *testing.T) { for _, tt := range []struct { name string @@ -2479,6 +2539,44 @@ func TestDataBranchNeedsHistoricalProjection(t *testing.T) { "a reordered layout requires projection") } +func TestHistoricalDataBranchChangesHandleSkipsHydrationAtBoundedGeneration(t *testing.T) { + ses := newValidateSession(t) + mp := ses.proc.Mp() + + source := batch.NewWithSize(4) + source.SetAttributes([]string{catalog.Row_ID, "a", "b", objectio.DefaultCommitTS_Attr}) + source.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + source.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + source.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + source.Vecs[3] = vector.NewVec(types.T_TS.ToType()) + require.NoError(t, vector.AppendFixed(source.Vecs[0], buildHashDiffRowID(t, 0), false, mp)) + require.NoError(t, vector.AppendFixed(source.Vecs[1], int64(1), false, mp)) + require.NoError(t, vector.AppendFixed(source.Vecs[2], int64(15), false, mp)) + require.NoError(t, vector.AppendFixed(source.Vecs[3], types.BuildTS(20, 0), false, mp)) + source.SetRowCount(1) + + tblStuff := tableStuff{} + tblStuff.def.colNames = []string{"a", "b", "c"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), types.T_int64.ToType(), types.T_int64.ToType(), + } + h := historicalDataBranchChangesHandle{ + inner: &stubEngineChangesHandle{responses: []stubEngineChangesHandleResponse{{data: source}}}, + sourceMapping: []int{0, 1}, + tblStuff: tblStuff, + hydrate: false, + } + + got, tombstone, _, err := h.Next(context.Background(), mp) + require.NoError(t, err) + require.Nil(t, tombstone) + require.NotNil(t, got) + defer got.Clean(mp) + require.Equal(t, 1, got.RowCount()) + require.Equal(t, int64(15), vector.GetFixedAtNoTypeCheck[int64](got.Vecs[2], 0)) + require.True(t, got.Vecs[3].IsNull(0), "column added after the upper bound must stay NULL") +} + func TestOverlayDataBranchProbeResultHydratesTargetDefaults(t *testing.T) { ses := newValidateSession(t) mp := ses.proc.Mp() diff --git a/pkg/frontend/data_branch_helpers.go b/pkg/frontend/data_branch_helpers.go index 33ba8460d6887..42d581f5a96e6 100644 --- a/pkg/frontend/data_branch_helpers.go +++ b/pkg/frontend/data_branch_helpers.go @@ -324,6 +324,29 @@ func scanSnapshotRelationByID( filterExpr *plan.Expr, scanParallelism int, onBatch func(*batch.Batch) error, +) error { + return scanSnapshotRelationByIDWithFallback( + ctx, caller, ses, tableID, snapshotTS, nil, + attrs, colTypes, filterExpr, scanParallelism, onBatch, + ) +} + +// scanSnapshotRelationByIDWithFallback prefers current-view ranges, but can +// fall back to a relation that was already opened at snapshotTS. ALTER drops +// replaced physical generations from the current catalog, while bounded data +// branch operations may still need to hydrate rows from such a generation. +func scanSnapshotRelationByIDWithFallback( + ctx context.Context, + caller string, + ses *Session, + tableID uint64, + snapshotTS types.TS, + fallbackRangeRel engine.Relation, + attrs []string, + colTypes []types.Type, + filterExpr *plan.Expr, + scanParallelism int, + onBatch func(*batch.Batch) error, ) error { if len(attrs) == 0 { return nil @@ -350,15 +373,26 @@ func scanSnapshotRelationByID( zap.Int("scan-parallelism", scanParallelism), ) - _, _, rangeRel, err := storage.GetRelationById(ctx, baseTxnOp, tableID) - if err != nil { - return err - } - if rangeRel == nil { - return moerr.NewInternalErrorNoCtxf( - "scanSnapshotRelationByID: cannot resolve range relation by id %d at current txn view", - tableID, + _, _, rangeRel, resolveErr := storage.GetRelationById(ctx, baseTxnOp, tableID) + if resolveErr != nil || rangeRel == nil { + if fallbackRangeRel == nil { + if resolveErr != nil { + return resolveErr + } + return moerr.NewInternalErrorNoCtxf( + "scanSnapshotRelationByID: cannot resolve range relation by id %d at current txn view", + tableID, + ) + } + logutil.Info( + "DataBranch-SnapshotScan-HistoricalRangeFallback", + zap.String("caller", caller), + zap.Uint64("table-id", tableID), + zap.String("snapshot-ts", snapshotTS.ToString()), + zap.Error(resolveErr), ) + rangeRel = fallbackRangeRel + rangeTS = snapshotTS } rangesParam := engine.DefaultRangesParam diff --git a/pkg/frontend/data_branch_helpers_test.go b/pkg/frontend/data_branch_helpers_test.go index c0c85ff290153..19df499e9b25d 100644 --- a/pkg/frontend/data_branch_helpers_test.go +++ b/pkg/frontend/data_branch_helpers_test.go @@ -307,6 +307,46 @@ func TestScanSnapshotRelationByID_EarlyAndErrorPaths(t *testing.T) { require.ErrorIs(t, err, wantErr) }) + t.Run("uses historical fallback when current relation is gone", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().SnapshotTS().Return(types.BuildTS(10, 0).ToTimestamp()).AnyTimes() + + currentLookupErr := moerr.NewInternalErrorNoCtx("can not find table by id 7") + eng := mock_frontend.NewMockEngine(ctrl) + eng.EXPECT().GetRelationById(gomock.Any(), txnOp, uint64(7)). + Return("", "", nil, currentLookupErr). + Times(1) + + historicalRel := mock_frontend.NewMockRelation(ctrl) + wantErr := moerr.NewInternalErrorNoCtx("historical ranges reached") + historicalRel.EXPECT().Ranges(gomock.Any(), gomock.Any()). + Return(nil, wantErr). + Times(1) + + ses.txnHandler = &TxnHandler{ + storage: eng, + txnOp: txnOp, + } + + err := scanSnapshotRelationByIDWithFallback( + context.Background(), + "unit-test", + ses, + 7, + types.BuildTS(20, 0), + historicalRel, + []string{"id"}, + []types.Type{types.T_int64.ToType()}, + nil, + 0, + func(*batch.Batch) error { return nil }, + ) + require.ErrorIs(t, err, wantErr) + }) + t.Run("returns error when range relation is missing", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/test/distributed/cases/git4data/branch/pick/pick_9.result b/test/distributed/cases/git4data/branch/pick/pick_9.result index 6c9ad7bee132e..1c3ec61e3837f 100644 --- a/test/distributed/cases/git4data/branch/pick/pick_9.result +++ b/test/distributed/cases/git4data/branch/pick/pick_9.result @@ -193,4 +193,32 @@ drop snapshot sp_nolca_to; drop table t2; drop table t1; drop table t0; +drop snapshot if exists sp_double_from; +drop snapshot if exists sp_double_to; +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1),(2,2); +data branch create table t1 from t0; +create table t2 (a int, b int, c int default 7, d int default 9, primary key(a)); +insert into t2 values (1,100,70,90),(2,200,71,91); +create snapshot sp_double_from for account sys; +update t1 set b=10 where a=1; +alter table t1 add column c int default 7; +update t1 set b=15,c=8 where a=1; +create snapshot sp_double_to for account sys; +alter table t1 add column d int default 9; +update t1 set b=25,c=18,d=19 where a=1; +data branch pick t1 into t2 between snapshot sp_double_from and sp_double_to when conflict accept; +select * from t2 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] ¦ c[4,32,0] ¦ d[4,32,0] 𝄀 +1 ¦ 15 ¦ 8 ¦ null 𝄀 +2 ¦ 200 ¦ 71 ¦ 91 +select * from t1 order by a asc; +➤ a[4,32,0] ¦ b[4,32,0] ¦ c[4,32,0] ¦ d[4,32,0] 𝄀 +1 ¦ 25 ¦ 18 ¦ 19 𝄀 +2 ¦ 2 ¦ 7 ¦ 9 +drop snapshot sp_double_from; +drop snapshot sp_double_to; +drop table t2; +drop table t1; +drop table t0; drop database test; diff --git a/test/distributed/cases/git4data/branch/pick/pick_9.sql b/test/distributed/cases/git4data/branch/pick/pick_9.sql index 96e6733c359e7..82da63230255c 100644 --- a/test/distributed/cases/git4data/branch/pick/pick_9.sql +++ b/test/distributed/cases/git4data/branch/pick/pick_9.sql @@ -311,4 +311,40 @@ drop table t2; drop table t1; drop table t0; +-- ---------------------------------------------------------------- +-- case j: no-LCA BETWEEN hydrates through a dropped middle generation +-- ---------------------------------------------------------------- +-- The upper-bound generation is itself replaced by a later ALTER. Rows from +-- the first generation still need its endpoint image, so the reader fallback +-- must use the relation opened at sp_double_to rather than current catalog. + +drop snapshot if exists sp_double_from; +drop snapshot if exists sp_double_to; + +create table t0 (a int, b int, primary key(a)); +insert into t0 values (1,1),(2,2); +data branch create table t1 from t0; + +create table t2 (a int, b int, c int default 7, d int default 9, primary key(a)); +insert into t2 values (1,100,70,90),(2,200,71,91); + +create snapshot sp_double_from for account sys; +update t1 set b=10 where a=1; +alter table t1 add column c int default 7; +update t1 set b=15,c=8 where a=1; +create snapshot sp_double_to for account sys; +alter table t1 add column d int default 9; +update t1 set b=25,c=18,d=19 where a=1; + +data branch pick t1 into t2 between snapshot sp_double_from and sp_double_to when conflict accept; +select * from t2 order by a asc; +-- expect bounded middle-generation row; d did not exist at the upper bound +select * from t1 order by a asc; + +drop snapshot sp_double_from; +drop snapshot sp_double_to; +drop table t2; +drop table t1; +drop table t0; + drop database test; From efb637b02c712c806768f217e8418061978f50ff Mon Sep 17 00:00:00 2001 From: ULookup Date: Fri, 17 Jul 2026 15:35:31 +0800 Subject: [PATCH 22/31] fix: reclaim unowned historical alter lineage --- pkg/frontend/authenticate_test.go | 3 + pkg/frontend/data_branch.go | 6 + pkg/frontend/data_branch_snapshot.go | 263 +++++++++++++++++ pkg/frontend/data_branch_snapshot_test.go | 117 ++++++++ .../databranchutils/branch_dag_test.go | 187 +++++++++++- .../branch_protect_snapshot.go | 219 ++++++++++++++ pkg/frontend/pitr.go | 6 + pkg/frontend/pitr_test.go | 66 +++++ pkg/frontend/snapshot.go | 3 + pkg/sql/compile/alter.go | 268 ++++++++++++++++-- pkg/sql/compile/alter_test.go | 171 ++++++++++- pkg/sql/compile/ddl.go | 22 +- .../test/branch_protect_snapshot_test.go | 164 ++++++++++- .../branch/edge/branch_edge_cases.result | 59 ++++ .../branch/edge/branch_edge_cases.sql | 53 +++- 15 files changed, 1561 insertions(+), 46 deletions(-) diff --git a/pkg/frontend/authenticate_test.go b/pkg/frontend/authenticate_test.go index 1196da76adc6e..a813ae0382723 100644 --- a/pkg/frontend/authenticate_test.go +++ b/pkg/frontend/authenticate_test.go @@ -15099,6 +15099,7 @@ func TestDoDropSnapshot(t *testing.T) { bh := &backgroundExecTest{} bh.init() + registerEmptyHistoricalLineageResults(bh) bhStub := gostub.StubFunc(&NewBackgroundExec, bh) defer bhStub.Reset() @@ -15149,6 +15150,7 @@ func TestDoDropSnapshot(t *testing.T) { err := doDropSnapshot(ctx, ses, ds) convey.So(err, convey.ShouldBeNil) + convey.So(bh.executedSQLs, convey.ShouldContain, historicalAlterLineageMetadataSQL()) }) convey.Convey("doDropSnapshot success", t, func() { @@ -15160,6 +15162,7 @@ func TestDoDropSnapshot(t *testing.T) { bh := &backgroundExecTest{} bh.init() + registerEmptyHistoricalLineageResults(bh) bhStub := gostub.StubFunc(&NewBackgroundExec, bh) defer bhStub.Reset() diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index f60de0dedf0e2..d8bbcdae28b10 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -553,6 +553,9 @@ func dataBranchDeleteTable( if err = reclaimBranchSnapshotsWithBH(execCtx.reqCtx, ses, bh, []uint64{tblID}); err != nil { return } + if err = compactHistoricalAlterLineageWithBH(execCtx.reqCtx, bh, time.Now().UTC()); err != nil { + return + } return nil } @@ -610,6 +613,9 @@ func dataBranchDeleteDatabase( if err = reclaimBranchSnapshotsWithBH(execCtx.reqCtx, ses, bh, tableIDs); err != nil { return } + if err = compactHistoricalAlterLineageWithBH(execCtx.reqCtx, bh, time.Now().UTC()); err != nil { + return + } return nil } diff --git a/pkg/frontend/data_branch_snapshot.go b/pkg/frontend/data_branch_snapshot.go index 1a5a370cfa471..898cae8208ef4 100644 --- a/pkg/frontend/data_branch_snapshot.go +++ b/pkg/frontend/data_branch_snapshot.go @@ -17,6 +17,7 @@ package frontend import ( "context" "fmt" + "time" "github.com/google/uuid" "go.uber.org/zap" @@ -41,6 +42,268 @@ func branchSnapshotName(childTableID uint64) string { // databranchutils.BranchSnapshotKind. const branchSnapshotKind = databranchutils.BranchSnapshotKind +func historicalAlterLineageMetadataSQL() string { + return fmt.Sprintf( + "select table_id, p_table_id, clone_ts, creator, level, table_deleted from %s.%s for update", + catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, + ) +} + +func historicalAlterLineageEdgeSQL() string { + return fmt.Sprintf( + "select sname, ts, account_name, database_name, table_name, obj_id from %s.%s where kind = '%s'", + catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, databranchutils.BranchSnapshotKind, + ) +} + +func historicalSnapshotSourceSQL() string { + return fmt.Sprintf( + "select ts, level, account_name, database_name, table_name, obj_id from %s.%s where kind = 'user'", + catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, + ) +} + +func historicalPitrSourceSQL() string { + return fmt.Sprintf( + "select level, account_name, database_name, table_name, obj_id, pitr_length, pitr_unit from %s.%s where pitr_status = 1", + catalog.MO_CATALOG, catalog.MO_PITR, + ) +} + +func historicalLineageQuery( + ctx context.Context, + bh BackgroundExec, + sql string, +) ([]ExecResult, error) { + bh.ClearExecResultSet() + if err := bh.Exec(ctx, sql); err != nil { + return nil, err + } + return getResultSet(ctx, bh) +} + +func loadHistoricalAlterLineageDAG( + ctx context.Context, + bh BackgroundExec, +) (databranchutils.BranchReclaimDag, error) { + erArray, err := historicalLineageQuery(ctx, bh, historicalAlterLineageMetadataSQL()) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + var rows []databranchutils.DataBranchMetadata + for _, er := range erArray { + for row := uint64(0); row < er.GetRowCount(); row++ { + tableID, err := er.GetUint64(ctx, row, 0) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + parentID, err := er.GetUint64(ctx, row, 1) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + cloneTS, err := er.GetInt64(ctx, row, 2) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + creator, err := er.GetUint64(ctx, row, 3) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + level, err := er.GetString(ctx, row, 4) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + deleted, err := er.GetInt64(ctx, row, 5) + if err != nil { + return databranchutils.BranchReclaimDag{}, err + } + rows = append(rows, databranchutils.DataBranchMetadata{ + TableID: tableID, + PTableID: parentID, + CloneTS: cloneTS, + Creator: creator, + Level: level, + TableDeleted: deleted != 0, + }) + } + } + return databranchutils.NewBranchReclaimDag(rows), nil +} + +func loadHistoricalAlterLineageEdges( + ctx context.Context, + bh BackgroundExec, +) (map[uint64]databranchutils.HistoricalLineageEdge, error) { + erArray, err := historicalLineageQuery(ctx, bh, historicalAlterLineageEdgeSQL()) + if err != nil { + return nil, err + } + edges := make(map[uint64]databranchutils.HistoricalLineageEdge) + for _, er := range erArray { + for row := uint64(0); row < er.GetRowCount(); row++ { + sname, err := er.GetString(ctx, row, 0) + if err != nil { + return nil, err + } + childID, ok := databranchutils.ParseBranchSnapshotName(sname) + if !ok { + continue + } + cloneTS, err := er.GetInt64(ctx, row, 1) + if err != nil { + return nil, err + } + accountName, err := er.GetString(ctx, row, 2) + if err != nil { + return nil, err + } + databaseName, err := er.GetString(ctx, row, 3) + if err != nil { + return nil, err + } + tableName, err := er.GetString(ctx, row, 4) + if err != nil { + return nil, err + } + parentID, err := er.GetUint64(ctx, row, 5) + if err != nil { + return nil, err + } + edges[childID] = databranchutils.HistoricalLineageEdge{ + ChildTableID: childID, + ParentTableID: parentID, + CloneTS: cloneTS, + AccountName: accountName, + DatabaseName: databaseName, + TableName: tableName, + } + } + } + return edges, nil +} + +func loadHistoricalSources( + ctx context.Context, + bh BackgroundExec, + now time.Time, +) ([]databranchutils.HistoricalSource, error) { + snapshotRows, err := historicalLineageQuery(ctx, bh, historicalSnapshotSourceSQL()) + if err != nil { + return nil, err + } + var sources []databranchutils.HistoricalSource + for _, er := range snapshotRows { + for row := uint64(0); row < er.GetRowCount(); row++ { + oldestTS, err := er.GetInt64(ctx, row, 0) + if err != nil { + return nil, err + } + source, err := historicalSourceFromResult(ctx, er, row, 1, oldestTS) + if err != nil { + return nil, err + } + sources = append(sources, source) + } + } + + pitrRows, err := historicalLineageQuery(ctx, bh, historicalPitrSourceSQL()) + if err != nil { + return nil, err + } + for _, er := range pitrRows { + for row := uint64(0); row < er.GetRowCount(); row++ { + length, err := er.GetInt64(ctx, row, 5) + if err != nil { + return nil, err + } + unit, err := er.GetString(ctx, row, 6) + if err != nil { + return nil, err + } + oldestTS, err := databranchutils.PitrRetentionLowerBound(now, int(length), unit) + if err != nil { + return nil, err + } + source, err := historicalSourceFromResult(ctx, er, row, 0, oldestTS) + if err != nil { + return nil, err + } + sources = append(sources, source) + } + } + return sources, nil +} + +func historicalSourceFromResult( + ctx context.Context, + er ExecResult, + row, start uint64, + oldestTS int64, +) (databranchutils.HistoricalSource, error) { + level, err := er.GetString(ctx, row, start) + if err != nil { + return databranchutils.HistoricalSource{}, err + } + accountName, err := er.GetString(ctx, row, start+1) + if err != nil { + return databranchutils.HistoricalSource{}, err + } + databaseName, err := er.GetString(ctx, row, start+2) + if err != nil { + return databranchutils.HistoricalSource{}, err + } + tableName, err := er.GetString(ctx, row, start+3) + if err != nil { + return databranchutils.HistoricalSource{}, err + } + objectID, err := er.GetUint64(ctx, row, start+4) + if err != nil { + return databranchutils.HistoricalSource{}, err + } + return databranchutils.HistoricalSource{ + Level: level, + AccountName: accountName, + DatabaseName: databaseName, + TableName: tableName, + ObjectID: objectID, + OldestTS: oldestTS, + }, nil +} + +func compactHistoricalAlterLineageWithBH( + ctx context.Context, + bh BackgroundExec, + now time.Time, +) error { + sysCtx := defines.AttachAccountId(ctx, sysAccountID) + dag, err := loadHistoricalAlterLineageDAG(sysCtx, bh) + if err != nil { + return err + } + edges, err := loadHistoricalAlterLineageEdges(sysCtx, bh) + if err != nil { + return err + } + sources, err := loadHistoricalSources(sysCtx, bh, now) + if err != nil { + return err + } + plan := databranchutils.ComputeAlterLineageCompactionPlan(dag, edges, sources) + if len(plan.TableIDs) == 0 { + return nil + } + for _, sql := range []string{ + databranchutils.BuildAlterLineageSnapshotDeleteSQL(plan.SnapshotNames), + databranchutils.BuildAlterLineageMetadataDeleteSQL(plan.TableIDs), + } { + bh.ClearExecResultSet() + if err = bh.Exec(sysCtx, sql); err != nil { + return err + } + } + return nil +} + // loadBranchDAGWithBH reads mo_branch_metadata under the sys account and // returns an in-memory DAG. It is used by the frontend reclaim entry point // which has a BackgroundExec available. diff --git a/pkg/frontend/data_branch_snapshot_test.go b/pkg/frontend/data_branch_snapshot_test.go index af8cd59898f5b..b5c4feea7e15c 100644 --- a/pkg/frontend/data_branch_snapshot_test.go +++ b/pkg/frontend/data_branch_snapshot_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/frontend/databranchutils" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" ) @@ -61,6 +62,122 @@ func TestBranchSnapshotName(t *testing.T) { } } +func TestParseBranchSnapshotName(t *testing.T) { + childID, ok := databranchutils.ParseBranchSnapshotName("__mo_branch_42") + require.True(t, ok) + require.Equal(t, uint64(42), childID) + + for _, invalid := range []string{"", "user_snapshot", "__mo_branch_", "__mo_branch_bad"} { + _, ok = databranchutils.ParseBranchSnapshotName(invalid) + require.False(t, ok, invalid) + } +} + +func historicalLineageTestResult(rows [][]interface{}, columnCount int) *MysqlResultSet { + mrs := &MysqlResultSet{} + for i := 0; i < columnCount; i++ { + col := &MysqlColumn{} + col.SetName(fmt.Sprintf("c%d", i)) + col.SetColumnType(defines.MYSQL_TYPE_VARCHAR) + mrs.AddColumn(col) + } + for _, row := range rows { + mrs.AddRow(row) + } + return mrs +} + +func registerEmptyHistoricalLineageResults(bh *backgroundExecTest) { + bh.sql2result[historicalAlterLineageMetadataSQL()] = historicalLineageTestResult(nil, 6) + bh.sql2result[historicalAlterLineageEdgeSQL()] = historicalLineageTestResult(nil, 6) + bh.sql2result[historicalSnapshotSourceSQL()] = historicalLineageTestResult(nil, 6) + bh.sql2result[historicalPitrSourceSQL()] = historicalLineageTestResult(nil, 7) +} + +func newHistoricalLineageBackgroundExec( + metadataRows, edgeRows, snapshotRows, pitrRows [][]interface{}, +) *backgroundExecTest { + bh := &backgroundExecTest{} + bh.init() + bh.sql2result[historicalAlterLineageMetadataSQL()] = historicalLineageTestResult(metadataRows, 6) + bh.sql2result[historicalAlterLineageEdgeSQL()] = historicalLineageTestResult(edgeRows, 6) + bh.sql2result[historicalSnapshotSourceSQL()] = historicalLineageTestResult(snapshotRows, 6) + bh.sql2result[historicalPitrSourceSQL()] = historicalLineageTestResult(pitrRows, 7) + return bh +} + +func TestCompactHistoricalAlterLineageWithBH(t *testing.T) { + bh := newHistoricalLineageBackgroundExec( + [][]interface{}{{uint64(2), uint64(1), int64(100), uint64(0), "alter", false}}, + [][]interface{}{{"__mo_branch_2", int64(100), "acc", "db", "t", uint64(1)}}, + nil, + nil, + ) + + err := compactHistoricalAlterLineageWithBH( + context.Background(), bh, time.Unix(0, 1_000).UTC(), + ) + require.NoError(t, err) + require.Equal(t, []string{ + historicalAlterLineageMetadataSQL(), + historicalAlterLineageEdgeSQL(), + historicalSnapshotSourceSQL(), + historicalPitrSourceSQL(), + "delete from mo_catalog.mo_snapshots where kind = 'branch' and sname in ('__mo_branch_2')", + "delete from mo_catalog.mo_branch_metadata where table_id in (2) and (level = 'alter' or level like 'alter:%')", + }, bh.executedSQLs) +} + +func TestCompactHistoricalAlterLineageWithBHRetainsCoveredEdge(t *testing.T) { + bh := newHistoricalLineageBackgroundExec( + [][]interface{}{{uint64(2), uint64(1), int64(100), uint64(0), "alter", false}}, + [][]interface{}{{"__mo_branch_2", int64(100), "acc", "db", "t", uint64(1)}}, + [][]interface{}{{int64(50), "table", "acc", "db", "t", uint64(1)}}, + nil, + ) + + err := compactHistoricalAlterLineageWithBH( + context.Background(), bh, time.Unix(0, 1_000).UTC(), + ) + require.NoError(t, err) + require.Len(t, bh.executedSQLs, 4) +} + +func TestCompactHistoricalAlterLineageWithBHRetainsPitrCoveredEdge(t *testing.T) { + bh := newHistoricalLineageBackgroundExec( + [][]interface{}{{uint64(2), uint64(1), int64(100), uint64(0), "alter", false}}, + [][]interface{}{{"__mo_branch_2", int64(100), "acc", "db", "t", uint64(1)}}, + nil, + [][]interface{}{{"table", "acc", "db", "t", uint64(1), int64(1), "h"}}, + ) + + err := compactHistoricalAlterLineageWithBH( + context.Background(), bh, time.Unix(3600, 0).UTC(), + ) + require.NoError(t, err) + require.Len(t, bh.executedSQLs, 4) +} + +func TestCompactHistoricalAlterLineageWithBHPropagatesDeleteError(t *testing.T) { + bh := newHistoricalLineageBackgroundExec( + [][]interface{}{{uint64(2), uint64(1), int64(100), uint64(0), "alter", false}}, + [][]interface{}{{"__mo_branch_2", int64(100), "acc", "db", "t", uint64(1)}}, + nil, + nil, + ) + wantErr := errors.New("delete snapshot failed") + deleteSQL := "delete from mo_catalog.mo_snapshots where kind = 'branch' and sname in ('__mo_branch_2')" + bh.sql2err[deleteSQL] = wantErr + + err := compactHistoricalAlterLineageWithBH( + context.Background(), bh, time.Unix(0, 1_000).UTC(), + ) + require.ErrorIs(t, err, wantErr) + require.NotContains(t, bh.executedSQLs, + "delete from mo_catalog.mo_branch_metadata where table_id in (2) and (level = 'alter' or level like 'alter:%')", + ) +} + // --------------------------------------------------------------------------- // UT-U2 — buildDagFromRows (DAG adjacency construction) // --------------------------------------------------------------------------- diff --git a/pkg/frontend/databranchutils/branch_dag_test.go b/pkg/frontend/databranchutils/branch_dag_test.go index 75817b40981bd..1fd86f49107c7 100644 --- a/pkg/frontend/databranchutils/branch_dag_test.go +++ b/pkg/frontend/databranchutils/branch_dag_test.go @@ -14,7 +14,12 @@ package databranchutils -import "testing" +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) func TestDAGFunctionality(t *testing.T) { @@ -215,3 +220,183 @@ func TestPathFromRoot(t *testing.T) { t.Fatal("missing node unexpectedly had a root path") } } + +func TestComputeAlterLineageCompactionPlan(t *testing.T) { + dag := NewBranchReclaimDag([]DataBranchMetadata{ + {TableID: 2, PTableID: 1, CloneTS: 100, Level: "alter"}, + {TableID: 3, PTableID: 2, CloneTS: 200, Level: "alter"}, + }) + edges := map[uint64]HistoricalLineageEdge{ + 2: { + ChildTableID: 2, + ParentTableID: 1, + CloneTS: 100, + AccountName: "acc", + DatabaseName: "db", + TableName: "t", + }, + 3: { + ChildTableID: 3, + ParentTableID: 2, + CloneTS: 200, + AccountName: "acc", + DatabaseName: "db", + TableName: "t", + }, + } + + require.Equal(t, + AlterLineageCompactionPlan{ + TableIDs: []uint64{2, 3}, + SnapshotNames: []string{"__mo_branch_2", "__mo_branch_3"}, + }, + ComputeAlterLineageCompactionPlan(dag, edges, nil), + ) + + sources := []HistoricalSource{{ + Level: "table", + AccountName: "acc", + DatabaseName: "db", + TableName: "t", + OldestTS: 150, + }} + require.Equal(t, + AlterLineageCompactionPlan{ + TableIDs: []uint64{2}, + SnapshotNames: []string{"__mo_branch_2"}, + }, + ComputeAlterLineageCompactionPlan(dag, edges, sources), + ) +} + +func TestComputeAlterLineageCompactionPlanScopeAndOwners(t *testing.T) { + baseRows := []DataBranchMetadata{ + {TableID: 2, PTableID: 1, CloneTS: 100, Level: "alter"}, + } + edges := map[uint64]HistoricalLineageEdge{ + 2: { + ChildTableID: 2, + ParentTableID: 1, + CloneTS: 100, + AccountName: "acc", + DatabaseName: "db", + TableName: "t", + }, + } + + for _, source := range []HistoricalSource{ + {Level: "cluster", OldestTS: 100}, + {Level: "account", AccountName: "acc", OldestTS: 100}, + {Level: "database", AccountName: "acc", DatabaseName: "db", OldestTS: 100}, + {Level: "table", AccountName: "acc", DatabaseName: "db", TableName: "t", OldestTS: 100}, + {Level: "table", AccountName: "acc", DatabaseName: "db", TableName: "renamed", ObjectID: 1, OldestTS: 100}, + {Level: "table", AccountName: "acc", DatabaseName: "db", TableName: "t", ObjectID: 999, OldestTS: 100}, + } { + plan := ComputeAlterLineageCompactionPlan( + NewBranchReclaimDag(baseRows), edges, []HistoricalSource{source}, + ) + require.Empty(t, plan.TableIDs, "covering source %+v must retain the edge", source) + } + + uncovered := []HistoricalSource{ + {Level: "account", AccountName: "other", OldestTS: 0}, + {Level: "database", AccountName: "acc", DatabaseName: "other", OldestTS: 0}, + {Level: "table", AccountName: "acc", DatabaseName: "db", TableName: "t", OldestTS: 101}, + } + plan := ComputeAlterLineageCompactionPlan( + NewBranchReclaimDag(baseRows), edges, uncovered, + ) + require.Equal(t, []uint64{2}, plan.TableIDs) + + withLiveLogicalSibling := NewBranchReclaimDag(append(baseRows, + DataBranchMetadata{TableID: 4, PTableID: 1, CloneTS: 90, Level: "table"}, + )) + plan = ComputeAlterLineageCompactionPlan(withLiveLogicalSibling, edges, nil) + require.Empty(t, plan.TableIDs) + + withDeletedLogicalSibling := NewBranchReclaimDag(append(baseRows, + DataBranchMetadata{ + TableID: 4, PTableID: 1, CloneTS: 90, Level: "table", TableDeleted: true, + }, + )) + plan = ComputeAlterLineageCompactionPlan(withDeletedLogicalSibling, edges, nil) + require.Equal(t, []uint64{2}, plan.TableIDs) + + plan = ComputeAlterLineageCompactionPlan(NewBranchReclaimDag(baseRows), nil, nil) + require.Empty(t, plan.TableIDs, "missing identity must be retained conservatively") +} + +func TestComputeAlterLineageCompactionPlanTableOwnerSurvivesRename(t *testing.T) { + dag := NewBranchReclaimDag([]DataBranchMetadata{ + {TableID: 2, PTableID: 1, CloneTS: 100, Level: "alter"}, + {TableID: 3, PTableID: 2, CloneTS: 200, Level: "alter"}, + }) + edges := map[uint64]HistoricalLineageEdge{ + 2: { + ChildTableID: 2, ParentTableID: 1, CloneTS: 100, + AccountName: "acc", DatabaseName: "db", TableName: "before_rename", + }, + 3: { + ChildTableID: 3, ParentTableID: 2, CloneTS: 200, + AccountName: "acc", DatabaseName: "db", TableName: "after_rename", + }, + } + source := HistoricalSource{ + Level: "table", AccountName: "acc", DatabaseName: "db", + TableName: "before_rename", ObjectID: 1, OldestTS: 50, + } + + plan := ComputeAlterLineageCompactionPlan(dag, edges, []HistoricalSource{source}) + require.Empty(t, plan.TableIDs, + "the source object owns the component, including the newer edge across rename") +} + +func TestComputeAlterLineageCompactionPlanCycleSafe(t *testing.T) { + dag := NewBranchReclaimDag([]DataBranchMetadata{ + {TableID: 1, PTableID: 2, CloneTS: 100, Level: "alter"}, + {TableID: 2, PTableID: 1, CloneTS: 200, Level: "alter"}, + }) + edges := map[uint64]HistoricalLineageEdge{ + 1: {ChildTableID: 1, ParentTableID: 2, CloneTS: 100}, + 2: {ChildTableID: 2, ParentTableID: 1, CloneTS: 200}, + } + + plan := ComputeAlterLineageCompactionPlan(dag, edges, nil) + require.Equal(t, []uint64{1, 2}, plan.TableIDs) + require.Equal(t, []string{"__mo_branch_1", "__mo_branch_2"}, plan.SnapshotNames) +} + +func TestPitrRetentionLowerBound(t *testing.T) { + now := time.Date(2026, time.July, 17, 12, 0, 0, 0, time.UTC) + + for _, tc := range []struct { + length int + unit string + want time.Time + }{ + {length: 2, unit: "h", want: now.Add(-2 * time.Hour)}, + {length: 2, unit: "d", want: now.AddDate(0, 0, -2)}, + {length: 2, unit: "mo", want: now.AddDate(0, -2, 0)}, + {length: 2, unit: "y", want: now.AddDate(-2, 0, 0)}, + } { + got, err := PitrRetentionLowerBound(now, tc.length, tc.unit) + require.NoError(t, err) + require.Equal(t, tc.want.UnixNano(), got) + } + + _, err := PitrRetentionLowerBound(now, 1, "week") + require.Error(t, err) +} + +func TestBuildAlterLineageDeleteSQL(t *testing.T) { + require.Equal(t, + "delete from mo_catalog.mo_snapshots where kind = 'branch' and sname in ('__mo_branch_2','__mo_branch_3')", + BuildAlterLineageSnapshotDeleteSQL([]string{"__mo_branch_2", "__mo_branch_3"}), + ) + require.Equal(t, + "delete from mo_catalog.mo_branch_metadata where table_id in (2,3) and (level = 'alter' or level like 'alter:%')", + BuildAlterLineageMetadataDeleteSQL([]uint64{2, 3}), + ) + require.Empty(t, BuildAlterLineageSnapshotDeleteSQL(nil)) + require.Empty(t, BuildAlterLineageMetadataDeleteSQL(nil)) +} diff --git a/pkg/frontend/databranchutils/branch_protect_snapshot.go b/pkg/frontend/databranchutils/branch_protect_snapshot.go index 38ea594eff5f0..24be28a158b18 100644 --- a/pkg/frontend/databranchutils/branch_protect_snapshot.go +++ b/pkg/frontend/databranchutils/branch_protect_snapshot.go @@ -15,9 +15,11 @@ package databranchutils import ( + "fmt" "sort" "strconv" "strings" + "time" "github.com/matrixorigin/matrixone/pkg/logutil" "go.uber.org/zap" @@ -60,6 +62,16 @@ func BranchSnapshotName(childTableID uint64) string { return BranchSnapshotSnamePrefix + strconv.FormatUint(childTableID, 10) } +// ParseBranchSnapshotName extracts the child table id from a branch-managed +// snapshot name. It rejects user snapshots and malformed internal names. +func ParseBranchSnapshotName(name string) (uint64, bool) { + if !strings.HasPrefix(name, BranchSnapshotSnamePrefix) { + return 0, false + } + id, err := strconv.ParseUint(strings.TrimPrefix(name, BranchSnapshotSnamePrefix), 10, 64) + return id, err == nil +} + // BranchReclaimDag is an in-memory picture of mo_branch_metadata suitable for // running the reclaim DAG walk. `Children` is an adjacency list keyed on // parent table id; `Info` maps every known table id to its metadata row. @@ -81,6 +93,38 @@ type BranchReclaimNode struct { Deleted bool } +// HistoricalLineageEdge identifies the physical table edge protected by an +// ALTER-owned branch snapshot. The catalog identity fields come from the +// matching kind='branch' snapshot row. +type HistoricalLineageEdge struct { + ChildTableID uint64 + ParentTableID uint64 + CloneTS int64 + AccountName string + DatabaseName string + TableName string +} + +// HistoricalSource is a normalized user snapshot or active PITR retention +// window. OldestTS is a fixed snapshot timestamp or the rolling PITR lower +// bound calculated at the owning statement's timestamp. +type HistoricalSource struct { + Level string + AccountName string + DatabaseName string + TableName string + ObjectID uint64 + OldestTS int64 +} + +// AlterLineageCompactionPlan lists the paired catalog rows that can be +// removed without disconnecting a live logical branch or active historical +// source. +type AlterLineageCompactionPlan struct { + TableIDs []uint64 + SnapshotNames []string +} + // NewBranchReclaimDag builds the reclaim DAG from a flat list of metadata // rows (shape shared with NewDAG). func NewBranchReclaimDag(rows []DataBranchMetadata) BranchReclaimDag { @@ -103,6 +147,181 @@ func NewBranchReclaimDag(rows []DataBranchMetadata) BranchReclaimDag { return dag } +// PitrRetentionLowerBound returns the oldest timestamp retained by a PITR at +// the supplied statement time. It never reads the wall clock itself, keeping +// frontend and compile-layer decisions on the same boundary. +func PitrRetentionLowerBound(now time.Time, length int, unit string) (int64, error) { + if length < 0 { + return 0, fmt.Errorf("invalid PITR length %d", length) + } + now = now.UTC() + var lower time.Time + switch unit { + case "h": + lower = now.Add(-time.Duration(length) * time.Hour) + case "d": + lower = now.AddDate(0, 0, -length) + case "mo": + lower = now.AddDate(0, -length, 0) + case "y": + lower = now.AddDate(-length, 0, 0) + default: + return 0, fmt.Errorf("unknown PITR unit %q", unit) + } + return lower.UnixNano(), nil +} + +func historicalSourceOwnsComponent( + source HistoricalSource, + nodeIDs map[uint64]struct{}, + edges []HistoricalLineageEdge, +) bool { + switch strings.ToLower(source.Level) { + case "cluster": + return true + case "account": + for _, edge := range edges { + if source.AccountName == edge.AccountName { + return true + } + } + case "database": + for _, edge := range edges { + if source.AccountName == edge.AccountName && + source.DatabaseName == edge.DatabaseName { + return true + } + } + case "table": + if source.ObjectID != 0 { + if _, ok := nodeIDs[source.ObjectID]; ok { + return true + } + } + for _, edge := range edges { + if source.AccountName == edge.AccountName && + source.DatabaseName == edge.DatabaseName && + source.TableName == edge.TableName { + return true + } + } + } + return false +} + +// ComputeAlterLineageCompactionPlan finds live ALTER-only edges that no +// longer have an owner. A live logical branch conservatively owns its entire +// connected component so sibling/ancestor LCA paths are never disconnected. +func ComputeAlterLineageCompactionPlan( + dag BranchReclaimDag, + edges map[uint64]HistoricalLineageEdge, + sources []HistoricalSource, +) AlterLineageCompactionPlan { + visited := make(map[uint64]struct{}, len(dag.Info)) + plan := AlterLineageCompactionPlan{} + + for start := range dag.Info { + if _, ok := visited[start]; ok { + continue + } + component := make([]uint64, 0, 4) + stack := []uint64{start} + logicalOwner := false + for len(stack) > 0 { + last := len(stack) - 1 + tableID := stack[last] + stack = stack[:last] + if tableID == 0 { + continue + } + if _, ok := visited[tableID]; ok { + continue + } + visited[tableID] = struct{}{} + component = append(component, tableID) + + if meta, ok := dag.Info[tableID]; ok { + if !meta.Deleted && !IsAlterLineageLevel(meta.Level) { + logicalOwner = true + } + if meta.ParentTableID != 0 { + stack = append(stack, meta.ParentTableID) + } + } + stack = append(stack, dag.Children[tableID]...) + } + + if logicalOwner { + continue + } + componentNodeIDs := make(map[uint64]struct{}, len(component)) + componentEdges := make([]HistoricalLineageEdge, 0, len(component)) + for _, tableID := range component { + componentNodeIDs[tableID] = struct{}{} + if edge, ok := edges[tableID]; ok { + componentEdges = append(componentEdges, edge) + } + } + componentSources := make([]HistoricalSource, 0, len(sources)) + for _, source := range sources { + if historicalSourceOwnsComponent(source, componentNodeIDs, componentEdges) { + componentSources = append(componentSources, source) + } + } + for _, tableID := range component { + meta, ok := dag.Info[tableID] + if !ok || meta.Deleted || !IsAlterLineageLevel(meta.Level) { + continue + } + edge, ok := edges[tableID] + if !ok || edge.ChildTableID != tableID || + edge.ParentTableID != meta.ParentTableID || edge.CloneTS != meta.CloneTS { + continue + } + covered := false + for _, source := range componentSources { + if source.OldestTS <= edge.CloneTS { + covered = true + break + } + } + if covered { + continue + } + plan.TableIDs = append(plan.TableIDs, tableID) + plan.SnapshotNames = append(plan.SnapshotNames, BranchSnapshotName(tableID)) + } + } + + sort.Slice(plan.TableIDs, func(i, j int) bool { return plan.TableIDs[i] < plan.TableIDs[j] }) + sort.Strings(plan.SnapshotNames) + return plan +} + +// BuildAlterLineageSnapshotDeleteSQL deletes only branch-managed snapshots. +func BuildAlterLineageSnapshotDeleteSQL(snames []string) string { + return BuildBranchSnapshotDeleteSQL(snames) +} + +// BuildAlterLineageMetadataDeleteSQL re-checks ALTER ownership at execution +// time so a stale compaction plan cannot delete a logical branch row. +func BuildAlterLineageMetadataDeleteSQL(tableIDs []uint64) string { + if len(tableIDs) == 0 { + return "" + } + var b strings.Builder + b.Grow(128 + len(tableIDs)*20) + b.WriteString("delete from mo_catalog.mo_branch_metadata where table_id in (") + for i, tableID := range tableIDs { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(strconv.FormatUint(tableID, 10)) + } + b.WriteString(") and (level = 'alter' or level like 'alter:%')") + return b.String() +} + // SubtreeAllDeleted returns true iff `root` and every descendant reachable // through the DAG have `Deleted == true`. A root that is not in `Info` is // treated as "deleted" (i.e. already reclaimable), which matches the diff --git a/pkg/frontend/pitr.go b/pkg/frontend/pitr.go index b2f254e6d6920..29a1b34aaf1bb 100644 --- a/pkg/frontend/pitr.go +++ b/pkg/frontend/pitr.go @@ -834,6 +834,9 @@ func doDropPitr(ctx context.Context, ses *Session, stmt *tree.DropPitr) (err err return err } } + if err = compactHistoricalAlterLineageWithBH(ctx, bh, time.Now().UTC()); err != nil { + return err + } } return err } @@ -906,6 +909,9 @@ func doAlterPitr(ctx context.Context, ses *Session, stmt *tree.AlterPitr) (err e if err != nil { return err } + if err = compactHistoricalAlterLineageWithBH(ctx, bh, time.Now().UTC()); err != nil { + return err + } } return err } diff --git a/pkg/frontend/pitr_test.go b/pkg/frontend/pitr_test.go index c7149c76c8e21..4e475a73bad48 100644 --- a/pkg/frontend/pitr_test.go +++ b/pkg/frontend/pitr_test.go @@ -3635,3 +3635,69 @@ func Test_getPitrLengthAndUnit(t *testing.T) { _, _, _, err = getPitrLengthAndUnit(ctx, bh, "table", "", "", "tbl") assert.Error(t, err) } + +func newPitrLifecycleTestSession( + t *testing.T, +) (*Session, *backgroundExecTest, context.Context) { + t.Helper() + ctrl := gomock.NewController(t) + ses := newTestSession(t, ctrl) + bh := &backgroundExecTest{} + bh.init() + registerEmptyHistoricalLineageResults(bh) + bhStub := gostub.StubFunc(&NewBackgroundExec, bh) + t.Cleanup(func() { + bhStub.Reset() + ses.Close() + ctrl.Finish() + }) + + pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) + pu.SV.SetDefaultValues() + pu.SV.KillRountinesInterval = 0 + setPu("", pu) + ctx := context.WithValue(context.Background(), config.ParameterUnitKey, pu) + rm, _ := NewRoutineManager(ctx, "") + ses.rm = rm + ses.SetTenantInfo(&TenantInfo{ + Tenant: sysAccountName, + User: rootName, + DefaultRole: moAdminRoleName, + TenantID: sysAccountID, + UserID: rootID, + DefaultRoleID: moAdminRoleID, + }) + + bh.sql2result["begin;"] = nil + bh.sql2result["commit;"] = nil + bh.sql2result["rollback;"] = nil + return ses, bh, ctx +} + +func TestDoDropPitrCompactsHistoricalAlterLineage(t *testing.T) { + ses, bh, ctx := newPitrLifecycleTestSession(t) + stmt := &tree.DropPitr{Name: "pitr01"} + + checkSQL, err := getSqlForCheckPitr(ctx, "pitr01", sysAccountID) + require.NoError(t, err) + bh.sql2result[checkSQL] = newMrsForPitrRecord([][]interface{}{{"pitr-id"}}) + bh.sql2result[getSqlForDropPitr("pitr01", sysAccountID)] = nil + otherSQL := fmt.Sprintf(getPitrFormat+" where pitr_name != '%s';", SYSMOCATALOGPITR) + bh.sql2result[otherSQL] = newMrsForPitrRecord(nil) + bh.sql2result[getSqlForDropPitr(SYSMOCATALOGPITR, sysAccountID)] = nil + + require.NoError(t, doDropPitr(ctx, ses, stmt)) + require.Contains(t, bh.executedSQLs, historicalAlterLineageMetadataSQL()) +} + +func TestDoAlterPitrCompactsHistoricalAlterLineage(t *testing.T) { + ses, bh, ctx := newPitrLifecycleTestSession(t) + stmt := &tree.AlterPitr{Name: "pitr01", PitrValue: 1, PitrUnit: "h"} + + checkSQL, err := getSqlForCheckPitr(ctx, "pitr01", sysAccountID) + require.NoError(t, err) + bh.sql2result[checkSQL] = newMrsForPitrRecord([][]interface{}{{"pitr-id"}}) + + require.NoError(t, doAlterPitr(ctx, ses, stmt)) + require.Contains(t, bh.executedSQLs, historicalAlterLineageMetadataSQL()) +} diff --git a/pkg/frontend/snapshot.go b/pkg/frontend/snapshot.go index de7f9a606560f..729797a17aa8f 100644 --- a/pkg/frontend/snapshot.go +++ b/pkg/frontend/snapshot.go @@ -564,6 +564,9 @@ func doDropSnapshot(ctx context.Context, ses *Session, stmt *tree.DropSnapShot) if err != nil { return err } + if err = compactHistoricalAlterLineageWithBH(ctx, bh, time.Now().UTC()); err != nil { + return err + } } getLogger(ses.GetService()).Debug(fmt.Sprintf("drop snapshot %s success", string(stmt.Name))) diff --git a/pkg/sql/compile/alter.go b/pkg/sql/compile/alter.go index dbe9975a9e365..43a99975b6e30 100644 --- a/pkg/sql/compile/alter.go +++ b/pkg/sql/compile/alter.go @@ -97,24 +97,43 @@ func alterDataBranchParticipationSQL(oldTableID uint64) string { ) } -func alterDataBranchHistoricalSourceSQL(accountName, databaseName, tableName string, tableID uint64) string { +func alterDataBranchHistoricalSourceScopeSQL( + accountName, databaseName, tableName string, + tableID uint64, +) string { accountName = sqlquote.EscapeString(accountName) databaseName = sqlquote.EscapeString(databaseName) tableName = sqlquote.EscapeString(tableName) return fmt.Sprintf( - `select 1 from (`+ - `select level, account_name, database_name, table_name, obj_id from %s.%s where kind = 'user' `+ - `union all `+ - `select level, account_name, database_name, table_name, obj_id from %s.%s where pitr_status = 1`+ - `) as h where h.level = 'cluster' or (`+ - `h.account_name = '%s' and (`+ - `h.level = 'account' or `+ - `(h.level = 'database' and h.database_name = '%s') or `+ - `(h.level = 'table' and (h.obj_id = %d or (h.database_name = '%s' and h.table_name = '%s')))`+ - `)) limit 1`, + `(level = 'cluster' or (`+ + `account_name = '%s' and (`+ + `level = 'account' or `+ + `(level = 'database' and database_name = '%s') or `+ + `(level = 'table' and (obj_id = %d or (database_name = '%s' and table_name = '%s')))`+ + `)))`, + accountName, databaseName, tableID, databaseName, tableName, + ) +} + +func alterDataBranchHistoricalSnapshotSourceSQL( + accountName, databaseName, tableName string, + tableID uint64, +) string { + return fmt.Sprintf( + "select 1 from %s.%s where kind = 'user' and %s limit 1 for update", catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, + alterDataBranchHistoricalSourceScopeSQL(accountName, databaseName, tableName, tableID), + ) +} + +func alterDataBranchHistoricalPitrSourceSQL( + accountName, databaseName, tableName string, + tableID uint64, +) string { + return fmt.Sprintf( + "select 1 from %s.%s where pitr_status = 1 and %s limit 1 for update", catalog.MO_CATALOG, catalog.MO_PITR, - accountName, databaseName, tableID, databaseName, tableName, + alterDataBranchHistoricalSourceScopeSQL(accountName, databaseName, tableName, tableID), ) } @@ -137,22 +156,29 @@ func (c *Compile) alterTableHasHistoricalBranchSource( oldTableID uint64, databaseName, tableName string, ) (bool, error) { - res, err := c.runSqlWithResult( - alterDataBranchHistoricalSourceSQL( + for _, sql := range []string{ + alterDataBranchHistoricalSnapshotSourceSQL( c.proc.GetSessionInfo().Account, databaseName, tableName, oldTableID, ), - int32(catalog.System_Account), - ) - if err != nil { - return false, err + alterDataBranchHistoricalPitrSourceSQL( + c.proc.GetSessionInfo().Account, databaseName, tableName, oldTableID, + ), + } { + res, err := c.runSqlWithResult(sql, int32(catalog.System_Account)) + if err != nil { + return false, err + } + hasHistory := false + res.ReadRows(func(rows int, _ []*vector.Vector) bool { + hasHistory = rows > 0 + return false + }) + res.Close() + if hasHistory { + return true, nil + } } - defer res.Close() - hasHistory := false - res.ReadRows(func(rows int, _ []*vector.Vector) bool { - hasHistory = rows > 0 - return false - }) - return hasHistory, nil + return false, nil } func (c *Compile) prepareAlterDataBranchLineage( @@ -165,6 +191,16 @@ func (c *Compile) prepareAlterDataBranchLineage( } hasLiveLineage := false if participates { + op := c.proc.GetTxnOperator() + opts := op.TxnOptions() + if err = validateAlterDataBranchLineageTxn( + opts.GetByBegin(), opts.GetAutocommit(), op.Txn().IsPessimistic(), + ); err != nil { + return alterDataBranchLineagePlan{}, err + } + if err = c.compactExpiredAlterDataBranchLineage(time.Time{}); err != nil { + return alterDataBranchLineagePlan{}, err + } dag, dagErr := c.loadAlterDataBranchDAG(false) if dagErr != nil { return alterDataBranchLineagePlan{}, dagErr @@ -266,6 +302,188 @@ func (c *Compile) loadAlterDataBranchDAG(forUpdate bool) (databranchutils.Branch return databranchutils.NewBranchReclaimDag(rows), nil } +func alterDataBranchLineageEdgeSQL() string { + return fmt.Sprintf( + "select sname, ts, account_name, database_name, table_name, obj_id from %s.%s where kind = '%s'", + catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, databranchutils.BranchSnapshotKind, + ) +} + +func alterDataBranchSnapshotSourceSQL() string { + return fmt.Sprintf( + "select ts, level, account_name, database_name, table_name, obj_id from %s.%s where kind = 'user'", + catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, + ) +} + +func alterDataBranchPitrSourceSQL() string { + return fmt.Sprintf( + "select level, account_name, database_name, table_name, obj_id, pitr_length, pitr_unit from %s.%s where pitr_status = 1", + catalog.MO_CATALOG, catalog.MO_PITR, + ) +} + +func (c *Compile) loadAlterDataBranchLineageEdges() ( + map[uint64]databranchutils.HistoricalLineageEdge, + error, +) { + res, err := c.runSqlWithResult( + alterDataBranchLineageEdgeSQL(), int32(catalog.System_Account), + ) + if err != nil { + return nil, err + } + defer res.Close() + edges := make(map[uint64]databranchutils.HistoricalLineageEdge, res.AffectedRows) + res.ReadRows(func(rowCount int, cols []*vector.Vector) bool { + if rowCount == 0 { + return true + } + names := executor.GetStringRows(cols[0]) + cloneTSs := vector.MustFixedColNoTypeCheck[int64](cols[1]) + accounts := executor.GetStringRows(cols[2]) + databases := executor.GetStringRows(cols[3]) + tables := executor.GetStringRows(cols[4]) + parentIDs := vector.MustFixedColNoTypeCheck[uint64](cols[5]) + for i, name := range names { + childID, ok := databranchutils.ParseBranchSnapshotName(name) + if !ok { + continue + } + edges[childID] = databranchutils.HistoricalLineageEdge{ + ChildTableID: childID, + ParentTableID: parentIDs[i], + CloneTS: cloneTSs[i], + AccountName: accounts[i], + DatabaseName: databases[i], + TableName: tables[i], + } + } + return true + }) + return edges, nil +} + +func appendAlterDataBranchHistoricalSources( + res executor.Result, + oldestTS func(int, []*vector.Vector) (int64, error), + columnOffset int, + sources *[]databranchutils.HistoricalSource, +) error { + var loadErr error + res.ReadRows(func(rowCount int, cols []*vector.Vector) bool { + if rowCount == 0 { + return true + } + levels := executor.GetStringRows(cols[columnOffset]) + accounts := executor.GetStringRows(cols[columnOffset+1]) + databases := executor.GetStringRows(cols[columnOffset+2]) + tables := executor.GetStringRows(cols[columnOffset+3]) + objectIDs := vector.MustFixedColNoTypeCheck[uint64](cols[columnOffset+4]) + for i := range levels { + lowerBound, err := oldestTS(i, cols) + if err != nil { + loadErr = err + return false + } + *sources = append(*sources, databranchutils.HistoricalSource{ + Level: levels[i], + AccountName: accounts[i], + DatabaseName: databases[i], + TableName: tables[i], + ObjectID: objectIDs[i], + OldestTS: lowerBound, + }) + } + return true + }) + return loadErr +} + +func (c *Compile) loadAlterDataBranchHistoricalSources( + now time.Time, +) ([]databranchutils.HistoricalSource, error) { + res, err := c.runSqlWithResult( + alterDataBranchSnapshotSourceSQL(), int32(catalog.System_Account), + ) + if err != nil { + return nil, err + } + var sources []databranchutils.HistoricalSource + err = appendAlterDataBranchHistoricalSources( + res, + func(i int, cols []*vector.Vector) (int64, error) { + return vector.MustFixedColNoTypeCheck[int64](cols[0])[i], nil + }, + 1, + &sources, + ) + res.Close() + if err != nil { + return nil, err + } + + res, err = c.runSqlWithResult( + alterDataBranchPitrSourceSQL(), int32(catalog.System_Account), + ) + if err != nil { + return nil, err + } + err = appendAlterDataBranchHistoricalSources( + res, + func(i int, cols []*vector.Vector) (int64, error) { + lengths := vector.MustFixedColNoTypeCheck[int64](cols[5]) + units := executor.GetStringRows(cols[6]) + return databranchutils.PitrRetentionLowerBound(now, int(lengths[i]), units[i]) + }, + 0, + &sources, + ) + res.Close() + if err != nil { + return nil, err + } + return sources, nil +} + +// compactExpiredAlterDataBranchLineage is ALTER's opportunistic expiry +// hook. DROP paths compact synchronously, but an active PITR can also stop +// covering an edge merely because its rolling retention window advances. +// Locking metadata first keeps the edge/snapshot pair atomic with the ALTER +// that will immediately decide whether to append a new edge. +func (c *Compile) compactExpiredAlterDataBranchLineage(now time.Time) error { + dag, err := c.loadAlterDataBranchDAG(true) + if err != nil { + return err + } + if len(dag.Info) == 0 { + return nil + } + if now.IsZero() { + now = c.proc.GetTxnOperator().SnapshotTS().ToStdTime().UTC() + } + edges, err := c.loadAlterDataBranchLineageEdges() + if err != nil { + return err + } + sources, err := c.loadAlterDataBranchHistoricalSources(now) + if err != nil { + return err + } + plan := databranchutils.ComputeAlterLineageCompactionPlan(dag, edges, sources) + if len(plan.TableIDs) == 0 { + return nil + } + if err = c.runSqlWithSystemTenant( + databranchutils.BuildAlterLineageSnapshotDeleteSQL(plan.SnapshotNames), + ); err != nil { + return err + } + return c.runSqlWithSystemTenant( + databranchutils.BuildAlterLineageMetadataDeleteSQL(plan.TableIDs), + ) +} + func alterDataBranchLineageMetadata( dag databranchutils.BranchReclaimDag, oldTableID uint64, diff --git a/pkg/sql/compile/alter_test.go b/pkg/sql/compile/alter_test.go index 77140524761b7..8430c661aa461 100644 --- a/pkg/sql/compile/alter_test.go +++ b/pkg/sql/compile/alter_test.go @@ -72,13 +72,16 @@ func TestBuildAlterDataBranchLineageSQL(t *testing.T) { } func TestAlterDataBranchHistoricalSourceSQL(t *testing.T) { - sql := alterDataBranchHistoricalSourceSQL("tenant'o", "db'x", "tbl'y", 42) - require.Contains(t, sql, "from mo_catalog.mo_snapshots where kind = 'user'") - require.Contains(t, sql, "from mo_catalog.mo_pitr where pitr_status = 1") - require.Contains(t, sql, "h.account_name = 'tenant''o'") - require.Contains(t, sql, "h.database_name = 'db''x'") - require.Contains(t, sql, "h.table_name = 'tbl''y'") - require.Contains(t, sql, "h.obj_id = 42") + for _, sql := range []string{ + alterDataBranchHistoricalSnapshotSourceSQL("tenant'o", "db'x", "tbl'y", 42), + alterDataBranchHistoricalPitrSourceSQL("tenant'o", "db'x", "tbl'y", 42), + } { + require.Contains(t, sql, "account_name = 'tenant''o'") + require.Contains(t, sql, "database_name = 'db''x'") + require.Contains(t, sql, "table_name = 'tbl''y'") + require.Contains(t, sql, "obj_id = 42") + require.Contains(t, sql, "limit 1 for update") + } } func TestAlterDataBranchLineageMetadata(t *testing.T) { @@ -527,7 +530,8 @@ func TestScopeAlterTableCopyPrecheckPrimaryKeyThenSkipDedup(t *testing.T) { require.Equal(t, alterTable.Options.TargetTableName, spyExec.insertOption.AlterCopyDedupOpt().TargetTableName) assert.Equal(t, []string{ alterDataBranchParticipationSQL(1), - alterDataBranchHistoricalSourceSQL("", "test", "dept", 1), + alterDataBranchHistoricalSnapshotSourceSQL("", "test", "dept", 1), + alterDataBranchHistoricalPitrSourceSQL("", "test", "dept", 1), alterTable.CreateTmpTableSql, alterCopyTestPkNullCheckSQL, alterCopyTestPkDuplicateCheckSQL, @@ -692,6 +696,157 @@ func newAlterCopyFixedResult[T any](t *testing.T, mp *mpool.MPool, typ types.Typ return memRes.GetResult() } +func TestCompactExpiredAlterDataBranchLineage(t *testing.T) { + now := time.Date(2026, time.July, 17, 12, 0, 0, 0, time.UTC) + cloneTS := now.Add(-48 * time.Hour).UnixNano() + const ( + metadataSQL = "select table_id, p_table_id, clone_ts, creator, level, table_deleted from mo_catalog.mo_branch_metadata for update" + edgeSQL = "select sname, ts, account_name, database_name, table_name, obj_id from mo_catalog.mo_snapshots where kind = 'branch'" + snapshotSQL = "select ts, level, account_name, database_name, table_name, obj_id from mo_catalog.mo_snapshots where kind = 'user'" + pitrSQL = "select level, account_name, database_name, table_name, obj_id, pitr_length, pitr_unit from mo_catalog.mo_pitr where pitr_status = 1" + ) + + for _, tc := range []struct { + name string + pitrLength int64 + wantDeletes bool + wantSQLSuffix []string + }{ + { + name: "expired PITR releases ALTER edge", + pitrLength: 24, + wantDeletes: true, + wantSQLSuffix: []string{ + "delete from mo_catalog.mo_snapshots where kind = 'branch' and sname in ('__mo_branch_2')", + "delete from mo_catalog.mo_branch_metadata where table_id in (2) and (level = 'alter' or level like 'alter:%')", + }, + }, + { + name: "active PITR retains ALTER edge", + pitrLength: 72, + wantDeletes: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + spyExec := &alterCopyInsertSpyExecutor{results: make(map[string]executor.Result)} + c := newAlterCopyPrecheckCompile(t, ctrl, spyExec) + mp := c.proc.Mp() + + spyExec.results[metadataSQL] = newAlterLineageMetadataResult( + t, mp, []uint64{2}, []uint64{1}, []int64{cloneTS}, + []uint64{uint64(catalog.System_Account)}, []string{databranchutils.AlterLineageLevel}, []bool{false}, + ) + spyExec.results[edgeSQL] = newAlterLineageEdgeResult( + t, mp, []string{databranchutils.BranchSnapshotName(2)}, []int64{cloneTS}, + []string{"tenant"}, []string{"db"}, []string{"tbl"}, []uint64{1}, + ) + spyExec.results[snapshotSQL] = newAlterLineageSnapshotSourceResult(t, mp, nil, nil, nil, nil, nil, nil) + spyExec.results[pitrSQL] = newAlterLineagePitrSourceResult( + t, mp, []string{"table"}, []string{"tenant"}, []string{"db"}, []string{"tbl"}, + []uint64{1}, []int64{tc.pitrLength}, []string{"h"}, + ) + + require.NoError(t, c.compactExpiredAlterDataBranchLineage(now)) + want := []string{metadataSQL, edgeSQL, snapshotSQL, pitrSQL} + if tc.wantDeletes { + want = append(want, tc.wantSQLSuffix...) + } + require.Equal(t, want, spyExec.executedSQLs) + }) + } +} + +func newAlterLineageMetadataResult( + t *testing.T, + mp *mpool.MPool, + tableIDs, parentIDs []uint64, + cloneTSs []int64, + creators []uint64, + levels []string, + deleted []bool, +) executor.Result { + memRes := executor.NewMemResult([]types.Type{ + types.T_uint64.ToType(), types.T_uint64.ToType(), types.T_int64.ToType(), + types.T_uint64.ToType(), types.T_varchar.ToType(), types.T_bool.ToType(), + }, mp) + memRes.NewBatchWithRowCount(len(tableIDs)) + require.NoError(t, executor.AppendFixedRows(memRes, 0, tableIDs)) + require.NoError(t, executor.AppendFixedRows(memRes, 1, parentIDs)) + require.NoError(t, executor.AppendFixedRows(memRes, 2, cloneTSs)) + require.NoError(t, executor.AppendFixedRows(memRes, 3, creators)) + require.NoError(t, executor.AppendStringRows(memRes, 4, levels)) + require.NoError(t, executor.AppendFixedRows(memRes, 5, deleted)) + return memRes.GetResult() +} + +func newAlterLineageEdgeResult( + t *testing.T, + mp *mpool.MPool, + names []string, + cloneTSs []int64, + accounts, databases, tables []string, + objectIDs []uint64, +) executor.Result { + memRes := executor.NewMemResult([]types.Type{ + types.T_varchar.ToType(), types.T_int64.ToType(), types.T_varchar.ToType(), + types.T_varchar.ToType(), types.T_varchar.ToType(), types.T_uint64.ToType(), + }, mp) + memRes.NewBatchWithRowCount(len(names)) + require.NoError(t, executor.AppendStringRows(memRes, 0, names)) + require.NoError(t, executor.AppendFixedRows(memRes, 1, cloneTSs)) + require.NoError(t, executor.AppendStringRows(memRes, 2, accounts)) + require.NoError(t, executor.AppendStringRows(memRes, 3, databases)) + require.NoError(t, executor.AppendStringRows(memRes, 4, tables)) + require.NoError(t, executor.AppendFixedRows(memRes, 5, objectIDs)) + return memRes.GetResult() +} + +func newAlterLineageSnapshotSourceResult( + t *testing.T, + mp *mpool.MPool, + cloneTSs []int64, + levels, accounts, databases, tables []string, + objectIDs []uint64, +) executor.Result { + memRes := executor.NewMemResult([]types.Type{ + types.T_int64.ToType(), types.T_varchar.ToType(), types.T_varchar.ToType(), + types.T_varchar.ToType(), types.T_varchar.ToType(), types.T_uint64.ToType(), + }, mp) + memRes.NewBatchWithRowCount(len(cloneTSs)) + require.NoError(t, executor.AppendFixedRows(memRes, 0, cloneTSs)) + require.NoError(t, executor.AppendStringRows(memRes, 1, levels)) + require.NoError(t, executor.AppendStringRows(memRes, 2, accounts)) + require.NoError(t, executor.AppendStringRows(memRes, 3, databases)) + require.NoError(t, executor.AppendStringRows(memRes, 4, tables)) + require.NoError(t, executor.AppendFixedRows(memRes, 5, objectIDs)) + return memRes.GetResult() +} + +func newAlterLineagePitrSourceResult( + t *testing.T, + mp *mpool.MPool, + levels, accounts, databases, tables []string, + objectIDs []uint64, + lengths []int64, + units []string, +) executor.Result { + memRes := executor.NewMemResult([]types.Type{ + types.T_varchar.ToType(), types.T_varchar.ToType(), types.T_varchar.ToType(), + types.T_varchar.ToType(), types.T_uint64.ToType(), types.T_int64.ToType(), + types.T_varchar.ToType(), + }, mp) + memRes.NewBatchWithRowCount(len(levels)) + require.NoError(t, executor.AppendStringRows(memRes, 0, levels)) + require.NoError(t, executor.AppendStringRows(memRes, 1, accounts)) + require.NoError(t, executor.AppendStringRows(memRes, 2, databases)) + require.NoError(t, executor.AppendStringRows(memRes, 3, tables)) + require.NoError(t, executor.AppendFixedRows(memRes, 4, objectIDs)) + require.NoError(t, executor.AppendFixedRows(memRes, 5, lengths)) + require.NoError(t, executor.AppendStringRows(memRes, 6, units)) + return memRes.GetResult() +} + func TestScope_AlterTableInplace(t *testing.T) { tableDef := &plan.TableDef{ TblId: 282826, diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 61a4fa7df8664..fb739841037ac 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -1985,9 +1985,9 @@ func (c *Compile) runSqlWithSystemTenant(sql string) error { // // Called synchronously by the plain `DROP TABLE` path after flipping // table_deleted=true for the affected tid (design §9.2 / §10). -func (c *Compile) reclaimBranchProtectSnapshots(deadTIDs []uint64) error { +func (c *Compile) reclaimBranchProtectSnapshots(deadTIDs []uint64) (bool, error) { if len(deadTIDs) == 0 { - return nil + return false, nil } // Fast path: the vast majority of DROP TABLE operations are on tables // that have nothing to do with data branch. Skip the `FOR UPDATE` @@ -2010,7 +2010,7 @@ func (c *Compile) reclaimBranchProtectSnapshots(deadTIDs []uint64) error { ) probeRes, err := c.runSqlWithResult(probeSQL, int32(catalog.System_Account)) if err != nil { - return err + return false, err } hasBranchRow := false probeRes.ReadRows(func(n int, _ []*vector.Vector) bool { @@ -2021,7 +2021,7 @@ func (c *Compile) reclaimBranchProtectSnapshots(deadTIDs []uint64) error { }) probeRes.Close() if !hasBranchRow { - return nil + return false, nil } loadDAG := func() (databranchutils.BranchReclaimDag, error) { @@ -2067,7 +2067,7 @@ func (c *Compile) reclaimBranchProtectSnapshots(deadTIDs []uint64) error { sql := databranchutils.BuildBranchSnapshotDeleteSQL(snames) return c.runSqlWithSystemTenant(sql) } - return databranchutils.ReclaimBranchSnapshotsCore(deadTIDs, loadDAG, execDelete) + return true, databranchutils.ReclaimBranchSnapshotsCore(deadTIDs, loadDAG, execDelete) } func (s *Scope) CreateView(c *Compile) error { @@ -3302,13 +3302,23 @@ func (s *Scope) dropTableSingle(c *Compile, qry *plan.DropTable) error { // release the corresponding `__mo_branch_*` snapshots. This must run // synchronously so drop paths have identical semantics in the frontend // and compile-layer paths (design §5.3 / §9.2). - if err = c.reclaimBranchProtectSnapshots([]uint64{tblID}); err != nil { + var branchParticipates bool + if branchParticipates, err = c.reclaimBranchProtectSnapshots([]uint64{tblID}); err != nil { logutil.Error("reclaim branch protect snapshots failed", zap.Uint64("tblID", tblID), zap.Error(err), ) return err } + if branchParticipates { + if err = c.compactExpiredAlterDataBranchLineage(time.Time{}); err != nil { + logutil.Error("compact historical ALTER lineage failed", + zap.Uint64("tblID", tblID), + zap.Error(err), + ) + return err + } + } ps := partitionservice.GetService(c.proc.GetService()) extr := rel.GetExtraInfo() diff --git a/pkg/vm/engine/test/branch_protect_snapshot_test.go b/pkg/vm/engine/test/branch_protect_snapshot_test.go index 2cc8a5717d7eb..95c8d16a47504 100644 --- a/pkg/vm/engine/test/branch_protect_snapshot_test.go +++ b/pkg/vm/engine/test/branch_protect_snapshot_test.go @@ -171,7 +171,7 @@ func (e *bpsEnv) querySnapshotsByPrefix(t *testing.T, prefix string) []string { func (e *bpsEnv) loadBranchDAG(t *testing.T) databranchutils.BranchReclaimDag { t.Helper() sql := fmt.Sprintf( - "select table_id, p_table_id, clone_ts, table_deleted from %s.%s", + "select table_id, p_table_id, clone_ts, creator, level, table_deleted from %s.%s", catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, ) res, err := e.execSQL(e.sysCtx, sql) @@ -185,13 +185,17 @@ func (e *bpsEnv) loadBranchDAG(t *testing.T) databranchutils.BranchReclaimDag { tids := vector.MustFixedColWithTypeCheck[uint64](cols[0]) pids := vector.MustFixedColWithTypeCheck[uint64](cols[1]) cts := vector.MustFixedColWithTypeCheck[int64](cols[2]) + creators := vector.MustFixedColWithTypeCheck[uint64](cols[3]) + levels := executor.GetStringRows(cols[4]) for i := 0; i < n; i++ { - deleted := !cols[3].IsNull(uint64(i)) && - vector.GetFixedAtWithTypeCheck[bool](cols[3], i) + deleted := !cols[5].IsNull(uint64(i)) && + vector.GetFixedAtWithTypeCheck[bool](cols[5], i) rows = append(rows, databranchutils.DataBranchMetadata{ TableID: tids[i], CloneTS: cts[i], PTableID: pids[i], + Creator: creators[i], + Level: levels[i], TableDeleted: deleted, }) } @@ -211,13 +215,38 @@ func (e *bpsEnv) simulateBranchCreate( cloneTS int64, parentAccount, parentDB, parentTbl string, parentTableID uint64, +) { + e.simulateLineageCreate( + t, childTID, parentTID, cloneTS, "table", + parentAccount, parentDB, parentTbl, parentTableID, + ) +} + +func (e *bpsEnv) simulateAlterLineage( + t *testing.T, + childTID, parentTID uint64, + cloneTS int64, + parentAccount, parentDB, parentTbl string, +) { + e.simulateLineageCreate( + t, childTID, parentTID, cloneTS, databranchutils.AlterLineageLevel, + parentAccount, parentDB, parentTbl, parentTID, + ) +} + +func (e *bpsEnv) simulateLineageCreate( + t *testing.T, + childTID, parentTID uint64, + cloneTS int64, + metadataLevel, parentAccount, parentDB, parentTbl string, + parentTableID uint64, ) { t.Helper() require.NoError(t, exec_sql(e.disttae, e.sysCtx, fmt.Sprintf( - "insert into %s.%s values(%d, %d, %d, %d, 'table', false)", + "insert into %s.%s values(%d, %d, %d, %d, '%s', false)", catalog.MO_CATALOG, catalog.MO_BRANCH_METADATA, - childTID, cloneTS, parentTID, 0, + childTID, cloneTS, parentTID, 0, metadataLevel, ), )) @@ -237,6 +266,64 @@ func (e *bpsEnv) simulateBranchCreate( )) } +func (e *bpsEnv) loadHistoricalLineageEdges( + t *testing.T, +) map[uint64]databranchutils.HistoricalLineageEdge { + t.Helper() + res, err := e.execSQL(e.sysCtx, fmt.Sprintf( + "select sname, ts, account_name, database_name, table_name, obj_id from %s.%s where kind = '%s'", + catalog.MO_CATALOG, catalog.MO_SNAPSHOTS, databranchutils.BranchSnapshotKind, + )) + require.NoError(t, err) + defer res.Close() + edges := make(map[uint64]databranchutils.HistoricalLineageEdge) + res.ReadRows(func(n int, cols []*vector.Vector) bool { + if n == 0 { + return true + } + names := executor.GetStringRows(cols[0]) + cloneTSs := vector.MustFixedColWithTypeCheck[int64](cols[1]) + accounts := executor.GetStringRows(cols[2]) + databases := executor.GetStringRows(cols[3]) + tables := executor.GetStringRows(cols[4]) + parentIDs := vector.MustFixedColWithTypeCheck[uint64](cols[5]) + for i, name := range names { + childID, ok := databranchutils.ParseBranchSnapshotName(name) + if !ok { + continue + } + edges[childID] = databranchutils.HistoricalLineageEdge{ + ChildTableID: childID, + ParentTableID: parentIDs[i], + CloneTS: cloneTSs[i], + AccountName: accounts[i], + DatabaseName: databases[i], + TableName: tables[i], + } + } + return true + }) + return edges +} + +func (e *bpsEnv) compactHistoricalLineage( + t *testing.T, + sources []databranchutils.HistoricalSource, +) databranchutils.AlterLineageCompactionPlan { + t.Helper() + plan := databranchutils.ComputeAlterLineageCompactionPlan( + e.loadBranchDAG(t), e.loadHistoricalLineageEdges(t), sources, + ) + if len(plan.TableIDs) == 0 { + return plan + } + require.NoError(t, exec_sql(e.disttae, e.sysCtx, + databranchutils.BuildAlterLineageSnapshotDeleteSQL(plan.SnapshotNames))) + require.NoError(t, exec_sql(e.disttae, e.sysCtx, + databranchutils.BuildAlterLineageMetadataDeleteSQL(plan.TableIDs))) + return plan +} + // markBranchDeleted flips `table_deleted=true` for a given child tid. // Matches the effect of the UPDATE issued by ddl.go before the reclaim // hook fires. @@ -553,3 +640,70 @@ func TestBranchProtectSnapshot_CreateFailedRollsBack(t *testing.T) { "no branch-snapshot row must remain for the rolled-back child") } } + +func TestBranchProtectSnapshot_HistoricalAlterLineageSourceLifecycle(t *testing.T) { + env := setupBranchProtectSnapshotEnv(t) + defer env.close(t) + + const ( + parentTID = uint64(9001) + childTID = uint64(9002) + cloneTS = int64(900_000) + ) + env.simulateAlterLineage(t, childTID, parentTID, cloneTS, "sys", "history_db", "history_tbl") + + coveringSource := []databranchutils.HistoricalSource{{ + Level: "table", + AccountName: "sys", + DatabaseName: "history_db", + TableName: "history_tbl", + ObjectID: parentTID, + OldestTS: cloneTS - 1, + }} + require.Empty(t, env.compactHistoricalLineage(t, coveringSource)) + require.Equal(t, + []string{databranchutils.BranchSnapshotName(childTID)}, + env.querySnapshotsByPrefix(t, databranchutils.BranchSnapshotSnamePrefix), + ) + require.Contains(t, env.loadBranchDAG(t).Info, childTID) + + plan := env.compactHistoricalLineage(t, nil) + require.Equal(t, []uint64{childTID}, plan.TableIDs) + require.Empty(t, env.querySnapshotsByPrefix(t, databranchutils.BranchSnapshotSnamePrefix)) + require.Empty(t, env.loadBranchDAG(t).Info) +} + +func TestBranchProtectSnapshot_HistoricalAlterLineageLiveBranchLifecycle(t *testing.T) { + env := setupBranchProtectSnapshotEnv(t) + defer env.close(t) + + const ( + rootTID = uint64(9101) + alterTID = uint64(9102) + branchTID = uint64(9103) + ) + env.simulateAlterLineage(t, alterTID, rootTID, 910_000, "sys", "history_db", "history_tbl") + env.simulateBranchCreate(t, branchTID, rootTID, 910_001, "sys", "history_db", "history_tbl", rootTID) + + require.Empty(t, env.compactHistoricalLineage(t, nil), + "a live logical branch owns the connected ALTER lineage") + require.ElementsMatch(t, + []string{ + databranchutils.BranchSnapshotName(alterTID), + databranchutils.BranchSnapshotName(branchTID), + }, + env.querySnapshotsByPrefix(t, databranchutils.BranchSnapshotSnamePrefix), + ) + + env.markBranchDeleted(t, branchTID) + require.Equal(t, + []string{databranchutils.BranchSnapshotName(alterTID)}, + env.runReclaim(t, []uint64{branchTID}), + ) + plan := env.compactHistoricalLineage(t, nil) + require.Equal(t, []uint64{alterTID}, plan.TableIDs) + require.Empty(t, env.querySnapshotsByPrefix(t, databranchutils.BranchSnapshotSnamePrefix)) + dag := env.loadBranchDAG(t) + require.NotContains(t, dag.Info, alterTID) + require.True(t, dag.Info[branchTID].Deleted) +} diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result index 54744d2ade456..04931e5fb08d6 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result @@ -83,6 +83,65 @@ alter table rename_left rename column b to bb; data branch diff rename_left against rename_right; -- @regex("schema compatibility check: base column 'b' is not present in target schema",true) internal error: schema compatibility check: base column 'b' is not present in target schema +drop snapshot if exists history_owner_s; +create table history_owner_base(a int primary key, b int); +create snapshot history_owner_s for table br_schema_drift history_owner_base; +alter table history_owner_base add column c int default 0; +set @history_owner_tid = ( +select rel_id from mo_catalog.mo_tables +where reldatabase = 'br_schema_drift' and relname = 'history_owner_base' +); +select count(*) as lineage_before_source_drop +from mo_catalog.mo_branch_metadata +where table_id = @history_owner_tid and level = 'alter'; +lineage_before_source_drop +1 +drop snapshot history_owner_s; +select count(*) as snapshots_after_source_drop +from mo_catalog.mo_snapshots +where kind = 'branch' and table_name = 'history_owner_base'; +snapshots_after_source_drop +0 +select count(*) as lineage_after_source_drop +from mo_catalog.mo_branch_metadata +where table_id = @history_owner_tid and level = 'alter'; +lineage_after_source_drop +0 +alter table history_owner_base add column d int default 0; +set @history_owner_second_tid = ( +select rel_id from mo_catalog.mo_tables +where reldatabase = 'br_schema_drift' and relname = 'history_owner_base' +); +select count(*) as lineage_after_second_alter +from mo_catalog.mo_branch_metadata +where table_id = @history_owner_second_tid and level = 'alter'; +lineage_after_second_alter +0 +drop table history_owner_base; +create table history_live_base(a int primary key, b int); +data branch create table history_live_child from history_live_base; +alter table history_live_base add column c int default 0; +set @history_live_tid = ( +select rel_id from mo_catalog.mo_tables +where reldatabase = 'br_schema_drift' and relname = 'history_live_base' +); +select count(*) as lineage_with_live_branch +from mo_catalog.mo_branch_metadata +where table_id = @history_live_tid and level = 'alter'; +lineage_with_live_branch +1 +data branch delete table history_live_child; +select count(*) as snapshots_after_branch_drop +from mo_catalog.mo_snapshots +where kind = 'branch' and table_name = 'history_live_base'; +snapshots_after_branch_drop +0 +select count(*) as lineage_after_branch_drop +from mo_catalog.mo_branch_metadata +where table_id = @history_live_tid and level = 'alter'; +lineage_after_branch_drop +0 +drop table history_live_base; drop database br_schema_drift; drop database if exists br_txn_src; drop database if exists br_txn_dst; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql index 1ca2b51b6761a..9f5aebdb9f805 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql @@ -89,9 +89,60 @@ alter table rename_left rename column b to bb; -- @regex("schema compatibility check: base column 'b' is not present in target schema",true) data branch diff rename_left against rename_right; +-- Case 3: historical ALTER lineage has explicit owners and is reclaimed +-- synchronously when the last owner disappears. +drop snapshot if exists history_owner_s; +create table history_owner_base(a int primary key, b int); +create snapshot history_owner_s for table br_schema_drift history_owner_base; +alter table history_owner_base add column c int default 0; +set @history_owner_tid = ( + select rel_id from mo_catalog.mo_tables + where reldatabase = 'br_schema_drift' and relname = 'history_owner_base' +); +select count(*) as lineage_before_source_drop + from mo_catalog.mo_branch_metadata + where table_id = @history_owner_tid and level = 'alter'; +drop snapshot history_owner_s; +select count(*) as snapshots_after_source_drop + from mo_catalog.mo_snapshots + where kind = 'branch' and table_name = 'history_owner_base'; +select count(*) as lineage_after_source_drop + from mo_catalog.mo_branch_metadata + where table_id = @history_owner_tid and level = 'alter'; +alter table history_owner_base add column d int default 0; +set @history_owner_second_tid = ( + select rel_id from mo_catalog.mo_tables + where reldatabase = 'br_schema_drift' and relname = 'history_owner_base' +); +select count(*) as lineage_after_second_alter + from mo_catalog.mo_branch_metadata + where table_id = @history_owner_second_tid and level = 'alter'; +drop table history_owner_base; + +-- A live logical branch owns the connected ALTER component. Deleting the +-- final branch releases both its protect snapshot and the ALTER-only edge. +create table history_live_base(a int primary key, b int); +data branch create table history_live_child from history_live_base; +alter table history_live_base add column c int default 0; +set @history_live_tid = ( + select rel_id from mo_catalog.mo_tables + where reldatabase = 'br_schema_drift' and relname = 'history_live_base' +); +select count(*) as lineage_with_live_branch + from mo_catalog.mo_branch_metadata + where table_id = @history_live_tid and level = 'alter'; +data branch delete table history_live_child; +select count(*) as snapshots_after_branch_drop + from mo_catalog.mo_snapshots + where kind = 'branch' and table_name = 'history_live_base'; +select count(*) as lineage_after_branch_drop + from mo_catalog.mo_branch_metadata + where table_id = @history_live_tid and level = 'alter'; +drop table history_live_base; + drop database br_schema_drift; --- Case 3: transaction behavior. +-- Case 4: transaction behavior. drop database if exists br_txn_src; drop database if exists br_txn_dst; create database br_txn_src; From 2174674cae68231a2f421ac6495df3d6bfdcf1f2 Mon Sep 17 00:00:00 2001 From: ULookup Date: Fri, 17 Jul 2026 16:57:18 +0800 Subject: [PATCH 23/31] fix: preserve altered branch lineage ownership --- .../databranchutils/branch_dag_test.go | 17 ++++++++++ .../branch_protect_snapshot.go | 16 +++++++--- .../test/branch_protect_snapshot_test.go | 32 +++++++++++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/pkg/frontend/databranchutils/branch_dag_test.go b/pkg/frontend/databranchutils/branch_dag_test.go index 1fd86f49107c7..d3729b23af4ee 100644 --- a/pkg/frontend/databranchutils/branch_dag_test.go +++ b/pkg/frontend/databranchutils/branch_dag_test.go @@ -322,6 +322,23 @@ func TestComputeAlterLineageCompactionPlanScopeAndOwners(t *testing.T) { plan = ComputeAlterLineageCompactionPlan(withDeletedLogicalSibling, edges, nil) require.Equal(t, []uint64{2}, plan.TableIDs) + // ALTER preserves the logical owner's level after the prefix. Once the + // copy-and-swap drops the old physical table, this live alter:table row is + // the only remaining representation of the logical branch and must keep + // the lineage component alive. + withInheritedLogicalOwner := NewBranchReclaimDag([]DataBranchMetadata{ + {TableID: 2, PTableID: 1, CloneTS: 100, Level: "table", TableDeleted: true}, + {TableID: 3, PTableID: 2, CloneTS: 200, Level: "alter:table"}, + }) + inheritedEdges := map[uint64]HistoricalLineageEdge{ + 3: { + ChildTableID: 3, ParentTableID: 2, CloneTS: 200, + AccountName: "acc", DatabaseName: "db", TableName: "t", + }, + } + plan = ComputeAlterLineageCompactionPlan(withInheritedLogicalOwner, inheritedEdges, nil) + require.Empty(t, plan.TableIDs) + plan = ComputeAlterLineageCompactionPlan(NewBranchReclaimDag(baseRows), nil, nil) require.Empty(t, plan.TableIDs, "missing identity must be retained conservatively") } diff --git a/pkg/frontend/databranchutils/branch_protect_snapshot.go b/pkg/frontend/databranchutils/branch_protect_snapshot.go index 24be28a158b18..46daad3028bef 100644 --- a/pkg/frontend/databranchutils/branch_protect_snapshot.go +++ b/pkg/frontend/databranchutils/branch_protect_snapshot.go @@ -15,12 +15,12 @@ package databranchutils import ( - "fmt" "sort" "strconv" "strings" "time" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/logutil" "go.uber.org/zap" ) @@ -50,6 +50,14 @@ func NextAlterLineageLevel(parentLevel string) string { return AlterLineageLevel + ":" + parentLevel } +// isLogicalBranchOwnerLevel reports whether a live metadata row represents a +// logical branch. A plain "alter" row exists only for a historical source, +// while "alter:" carries the logical ownership inherited across +// ALTER's copy-and-swap. +func isLogicalBranchOwnerLevel(level string) bool { + return !IsAlterLineageLevel(level) || strings.HasPrefix(level, AlterLineageLevel+":") +} + // BranchSnapshotSnamePrefix is the sname prefix used by branch-owned snapshot // rows. The suffix is the decimal child table id. Keep this in sync with the // design doc §4.3. @@ -152,7 +160,7 @@ func NewBranchReclaimDag(rows []DataBranchMetadata) BranchReclaimDag { // frontend and compile-layer decisions on the same boundary. func PitrRetentionLowerBound(now time.Time, length int, unit string) (int64, error) { if length < 0 { - return 0, fmt.Errorf("invalid PITR length %d", length) + return 0, moerr.NewInvalidInputNoCtxf("invalid PITR length %d", length) } now = now.UTC() var lower time.Time @@ -166,7 +174,7 @@ func PitrRetentionLowerBound(now time.Time, length int, unit string) (int64, err case "y": lower = now.AddDate(-length, 0, 0) default: - return 0, fmt.Errorf("unknown PITR unit %q", unit) + return 0, moerr.NewInvalidInputNoCtxf("unknown PITR unit %q", unit) } return lower.UnixNano(), nil } @@ -241,7 +249,7 @@ func ComputeAlterLineageCompactionPlan( component = append(component, tableID) if meta, ok := dag.Info[tableID]; ok { - if !meta.Deleted && !IsAlterLineageLevel(meta.Level) { + if !meta.Deleted && isLogicalBranchOwnerLevel(meta.Level) { logicalOwner = true } if meta.ParentTableID != 0 { diff --git a/pkg/vm/engine/test/branch_protect_snapshot_test.go b/pkg/vm/engine/test/branch_protect_snapshot_test.go index 95c8d16a47504..cf88482952c89 100644 --- a/pkg/vm/engine/test/branch_protect_snapshot_test.go +++ b/pkg/vm/engine/test/branch_protect_snapshot_test.go @@ -707,3 +707,35 @@ func TestBranchProtectSnapshot_HistoricalAlterLineageLiveBranchLifecycle(t *test require.NotContains(t, dag.Info, alterTID) require.True(t, dag.Info[branchTID].Deleted) } + +func TestBranchProtectSnapshot_AlterInheritsLiveBranchOwnership(t *testing.T) { + env := setupBranchProtectSnapshotEnv(t) + defer env.close(t) + + const ( + rootTID = uint64(9201) + oldTID = uint64(9202) + currentTID = uint64(9203) + ) + env.simulateBranchCreate(t, oldTID, rootTID, 920_000, + "sys", "history_db", "history_tbl", rootTID) + env.simulateLineageCreate(t, currentTID, oldTID, 920_001, + "alter:table", "sys", "history_db", "history_tbl", oldTID) + + // ALTER's internal DROP retires the old physical generation, but the + // replacement still owns the logical branch and both protection snapshots. + env.markBranchDeleted(t, oldTID) + require.ElementsMatch(t, + []string{ + databranchutils.BranchSnapshotName(oldTID), + databranchutils.BranchSnapshotName(currentTID), + }, + env.runReclaim(t, []uint64{oldTID}), + ) + require.Empty(t, env.compactHistoricalLineage(t, nil)) + + // Once the replacement is dropped, ordinary reclaim removes the complete + // snapshot chain; retaining alter:table above must not introduce a leak. + env.markBranchDeleted(t, currentTID) + require.Empty(t, env.runReclaim(t, []uint64{currentTID})) +} From 50001f3addda00a9a97007d2e721250555f556b9 Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Mon, 20 Jul 2026 13:38:22 +0800 Subject: [PATCH 24/31] fix(frontend): project reordered data branch batches --- pkg/frontend/data_branch_hashdiff.go | 20 ++++++++---- pkg/frontend/data_branch_hashdiff_test.go | 39 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 9b912cb2d3ea8..97382cf64952b 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -2315,12 +2315,7 @@ func buildHashmapForTable( return } - if dataBat != nil && dataBat.RowCount() > 0 && side == "base" && - len(tblStuff.def.tarOnlyIdxes) > 0 && - !dataBranchBatchHasTargetLayout(dataBat, tblStuff) { - projected := projectBaseBatchToTarget(dataBat, tblStuff, mp) - dataBat = projected // projected will be cleaned in putVectors - } + dataBat = projectBaseBatchToTargetIfNeeded(side, dataBat, tblStuff, mp) if dataBat != nil && dataBat.RowCount() > 0 { totalRows += int64(dataBat.RowCount()) @@ -2550,6 +2545,19 @@ func projectBaseBatchToTarget( ) } +func projectBaseBatchToTargetIfNeeded( + side string, + dataBat *batch.Batch, + tblStuff *tableStuff, + mp *mpool.MPool, +) *batch.Batch { + if side != "base" || dataBat == nil || dataBat.RowCount() == 0 || + dataBranchBatchHasTargetLayout(dataBat, tblStuff) { + return dataBat + } + return projectBaseBatchToTarget(dataBat, tblStuff, mp) +} + func dataBranchSourceColToTargetIdx( sourceDef, targetDef *plan2.TableDef, targetColNames []string, diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index dc87a4c3966ad..1cf84a206f54b 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -2467,6 +2467,45 @@ func TestProjectBaseBatchToTargetPreservesTrailingCommitTS(t *testing.T) { require.Equal(t, baseline, mp.CurrNB(), "projection must transfer or clean every input vector") } +func TestProjectBaseBatchToTargetIfNeededForPureColumnReorder(t *testing.T) { + ses := newValidateSession(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tblStuff := newTestBranchTableStuff(ctrl) + // MODIFY ... FIRST/AFTER can reorder columns without adding any target-only + // column. The base physical layout is [a, b], while target indexes refer to + // [b, a]. + tblStuff.def.colNames = []string{"b", "a"} + tblStuff.def.colTypes = []types.Type{types.T_int64.ToType(), types.T_int64.ToType()} + tblStuff.def.baseColToTarIdx = []int{1, 0} + require.Empty(t, tblStuff.def.tarOnlyIdxes) + + mp := ses.proc.Mp() + baseline := mp.CurrNB() + baseBat := batch.NewWithSize(3) + baseBat.SetAttributes([]string{catalog.Row_ID, "a", "b"}) + baseBat.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + baseBat.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + baseBat.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + uid, err := types.BuildUuid() + require.NoError(t, err) + blkID := objectio.NewBlockid(&uid, 0, 1) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[0], types.NewRowid(blkID, 0), false, mp)) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[1], int64(11), false, mp)) + require.NoError(t, vector.AppendFixed(baseBat.Vecs[2], int64(22), false, mp)) + baseBat.SetRowCount(1) + + projected := projectBaseBatchToTargetIfNeeded("base", baseBat, &tblStuff, mp) + require.True(t, dataBranchBatchHasTargetLayout(projected, &tblStuff)) + require.Equal(t, []string{catalog.Row_ID, "b", "a"}, projected.Attrs) + require.Equal(t, int64(22), vector.GetFixedAtNoTypeCheck[int64](projected.Vecs[1], 0)) + require.Equal(t, int64(11), vector.GetFixedAtNoTypeCheck[int64](projected.Vecs[2], 0)) + + projected.Clean(mp) + require.Equal(t, baseline, mp.CurrNB(), "reorder projection must transfer every input vector") +} + func TestDataBranchSourceColToTargetIdxRejectsHistoricalTypeDrift(t *testing.T) { sourceDef := &plan.TableDef{ Cols: []*plan.ColDef{ From 9c8679901fa6402a937c43711bc79ddfc825965d Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Mon, 20 Jul 2026 13:53:03 +0800 Subject: [PATCH 25/31] fix(frontend): preserve identity-layout branch batches --- pkg/frontend/data_branch_hashdiff.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 97382cf64952b..f885a87209593 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -2555,6 +2555,17 @@ func projectBaseBatchToTargetIfNeeded( dataBranchBatchHasTargetLayout(dataBat, tblStuff) { return dataBat } + // Legacy table metadata without a source-to-target mapping has no evidence + // of a physical layout difference. Preserve the prior path. + if len(tblStuff.def.baseColToTarIdx) == 0 { + return dataBat + } + if !dataBranchNeedsHistoricalProjection( + tblStuff.def.baseColToTarIdx, + len(tblStuff.def.colNames), + ) { + return dataBat + } return projectBaseBatchToTarget(dataBat, tblStuff, mp) } From c515a14561ef07b0f378f115e254041694804f31 Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Mon, 20 Jul 2026 16:26:23 +0800 Subject: [PATCH 26/31] fix(frontend): project target-only historical type drift --- pkg/frontend/data_branch.go | 2 + pkg/frontend/data_branch_hashdiff.go | 13 +++ pkg/frontend/data_branch_hashdiff_test.go | 84 ++++++++++++++++++- .../branch/diff/diff_schema_evolve.result | 20 +++++ .../branch/diff/diff_schema_evolve.sql | 25 ++++++ .../branch/merge/merge_schema_evolve.result | 13 +++ .../branch/merge/merge_schema_evolve.sql | 14 ++++ 7 files changed, 168 insertions(+), 3 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index d8bbcdae28b10..f581a8426be34 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -1571,6 +1571,7 @@ func constructChangeHandle( if tarRange.rel[i].GetTableID(ctx) != tables.tarRel.GetTableID(ctx) { if sourceMapping, err = dataBranchSourceColToTargetIdx( tarRange.rel[i].GetTableDef(ctx), targetDef, tables.def.colNames, + tables.def.tarOnlyIdxes, ); err != nil { return } @@ -1622,6 +1623,7 @@ func constructChangeHandle( if baseRange.rel[i].GetTableID(ctx) != tables.baseRel.GetTableID(ctx) { if sourceMapping, err = dataBranchSourceColToTargetIdx( baseRange.rel[i].GetTableDef(ctx), targetDef, tables.def.colNames, + tables.def.tarOnlyIdxes, ); err != nil { return } diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index f885a87209593..ad39b1c307cc4 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -2572,6 +2572,7 @@ func projectBaseBatchToTargetIfNeeded( func dataBranchSourceColToTargetIdx( sourceDef, targetDef *plan2.TableDef, targetColNames []string, + targetOnlyIdxes []int, ) ([]int, error) { if sourceDef == nil || targetDef == nil { return nil, moerr.NewInternalErrorNoCtx("missing schema for historical data branch projection") @@ -2612,6 +2613,10 @@ func dataBranchSourceColToTargetIdx( for _, col := range targetDef.Cols { targetCols[strings.ToLower(col.Name)] = col } + targetOnly := make(map[int]struct{}, len(targetOnlyIdxes)) + for _, idx := range targetOnlyIdxes { + targetOnly[idx] = struct{}{} + } mapping := make([]int, 0, len(sourceDef.Cols)) for _, col := range sourceDef.Cols { if col.Name == catalog.Row_ID { @@ -2625,6 +2630,14 @@ func dataBranchSourceColToTargetIdx( targetCol, ok := targetCols[strings.ToLower(col.Name)] if !ok || col.Typ.Id != targetCol.Typ.Id || !dataBranchColumnTypeAttributesEqual(col.Typ, targetCol.Typ) { + if _, isTargetOnly := targetOnly[targetIdx]; isTargetOnly { + // This column does not exist on the base endpoint and is excluded + // from comparison and MERGE apply. Its old physical representation + // cannot be moved into the endpoint-typed batch; project it as NULL + // and let endpoint hydration restore a current value when needed. + mapping[len(mapping)-1] = -1 + continue + } return nil, moerr.NewNotSupportedNoCtxf( "historical data branch column %s has a different type from the endpoint schema", col.Name, diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index 1cf84a206f54b..36f79005ddea3 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -2516,15 +2516,93 @@ func TestDataBranchSourceColToTargetIdxRejectsHistoricalTypeDrift(t *testing.T) targetDef := &plan.TableDef{ Cols: []*plan.ColDef{ {Name: "a", Typ: plan.Type{Id: int32(types.T_int32)}}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_varchar), Width: 20}}, }, Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, } - _, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a"}) + _, err := dataBranchSourceColToTargetIdx( + sourceDef, targetDef, []string{"a", "c"}, []int{1}, + ) require.Error(t, err) require.Contains(t, err.Error(), "historical data branch column a has a different type") } +func TestDataBranchSourceColToTargetIdxSkipsTargetOnlyHistoricalTypeDrift(t *testing.T) { + sourceDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_int32)}}, + }, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, + } + targetDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", Typ: plan.Type{Id: int32(types.T_varchar), Width: 20}}, + }, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, + } + + mapping, err := dataBranchSourceColToTargetIdx( + sourceDef, targetDef, []string{"a", "c"}, []int{1}, + ) + require.NoError(t, err) + require.Equal(t, []int{0, -1}, mapping) + + ses := newValidateSession(t) + mp := ses.proc.Mp() + baseline := mp.CurrNB() + tblStuff := tableStuff{} + tblStuff.def.colNames = []string{"a", "c"} + tblStuff.def.colTypes = []types.Type{ + types.T_int64.ToType(), types.New(types.T_varchar, 20, 0), + } + source := batch.NewWithSize(3) + source.SetAttributes([]string{catalog.Row_ID, "a", "c"}) + source.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + source.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + source.Vecs[2] = vector.NewVec(types.T_int32.ToType()) + require.NoError(t, vector.AppendFixed(source.Vecs[0], buildHashDiffRowID(t, 0), false, mp)) + require.NoError(t, vector.AppendFixed(source.Vecs[1], int64(1), false, mp)) + require.NoError(t, vector.AppendFixed(source.Vecs[2], int32(70), false, mp)) + source.SetRowCount(1) + + projected := projectDataBranchBatchToTarget(source, &tblStuff, mapping, mp) + require.True(t, projected.Vecs[2].IsConstNull()) + require.Equal(t, types.T_varchar, projected.Vecs[2].GetType().Oid) + projected.Clean(mp) + require.Equal(t, baseline, mp.CurrNB()) +} + +func TestDataBranchSourceColToTargetIdxPreservesCompatibleAndReorderedColumns(t *testing.T) { + sourceDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "removed", Typ: plan.Type{Id: int32(types.T_int32)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_varchar), Width: 20}}, + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "extra", Typ: plan.Type{Id: int32(types.T_int32)}}, + }, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, + } + targetDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "extra", Typ: plan.Type{Id: int32(types.T_int32)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_varchar), Width: 20}}, + {Name: "renamed", Typ: plan.Type{Id: int32(types.T_int32)}}, + }, + Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, + } + + mapping, err := dataBranchSourceColToTargetIdx( + sourceDef, targetDef, []string{"a", "extra", "b", "renamed"}, []int{1, 3}, + ) + require.NoError(t, err) + require.Equal(t, []int{-1, 2, 0, 1}, mapping, + "source-only/renamed columns are dropped while compatible target-only and reordered columns stay mapped") +} + func TestDataBranchSourceColToTargetIdxRejectsHistoricalPrimaryKeyDrift(t *testing.T) { cols := []*plan.ColDef{ {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, @@ -2539,7 +2617,7 @@ func TestDataBranchSourceColToTargetIdxRejectsHistoricalPrimaryKeyDrift(t *testi Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, } - _, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a", "b"}) + _, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a", "b"}, nil) require.Error(t, err) require.Contains(t, err.Error(), "historical data branch primary key is incompatible") } @@ -2563,7 +2641,7 @@ func TestDataBranchSourceColToTargetIdxRejectsHistoricalFakePrimaryKeyDrift(t *t } _, err := dataBranchSourceColToTargetIdx( - sourceDef, targetDef, []string{"a", "b", catalog.FakePrimaryKeyColName}, + sourceDef, targetDef, []string{"a", "b", catalog.FakePrimaryKeyColName}, nil, ) require.Error(t, err) require.Contains(t, err.Error(), "historical data branch fake primary key schema differs") diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index 2447671eb188a..713d20de50f05 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -285,4 +285,24 @@ data branch diff t1 against t0; not supported: historical data branch fake primary key schema differs from the endpoint schema drop table t1; drop table t0; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 7; +alter table t1 modify column c varchar(20); +data branch diff t1 against t0; +diff t1 against t0 flag a b c +drop table t1; +drop table t0; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 7; +update t1 set b=11, c=70 where a=1; +alter table t1 modify column c varchar(20); +data branch diff t1 against t0; +diff t1 against t0 flag a b c +t1 UPDATE 1 11 70 +drop table t1; +drop table t0; drop database test; diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 63ae77c35f15d..50749200cab55 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -396,4 +396,29 @@ data branch diff t1 against t0; drop table t1; drop table t0; +-- ===================================================== +-- Case 22: target-only historical type change with no writes is compatible +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 7; +alter table t1 modify column c varchar(20); +data branch diff t1 against t0; +drop table t1; +drop table t0; + +-- ===================================================== +-- Case 23: target-only historical values survive endpoint hydration +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 7; +update t1 set b=11, c=70 where a=1; +alter table t1 modify column c varchar(20); +data branch diff t1 against t0; +drop table t1; +drop table t0; + drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index 801e55a41727d..d0ac1400f325b 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -139,4 +139,17 @@ a b 3 3 drop table t0; drop table t1; +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 7; +update t1 set b=11, c=70 where a=1; +alter table t1 modify column c varchar(20); +data branch merge t1 into t0; +select * from t0 order by a; +a b +1 11 +2 2 +drop table t1; +drop table t0; drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index be019390d784b..eb1bedfc7fe0e 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -183,4 +183,18 @@ select * from t0 order by a; drop table t0; drop table t1; +-- ===================================================== +-- Case 9: MERGE ignores target-only historical type changes and values +-- ===================================================== +create table t0(a int primary key, b int); +insert into t0 values(1,1),(2,2); +data branch create table t1 from t0; +alter table t1 add column c int default 7; +update t1 set b=11, c=70 where a=1; +alter table t1 modify column c varchar(20); +data branch merge t1 into t0; +select * from t0 order by a; +drop table t1; +drop table t0; + drop database test; From d4ec9f8180803e65c47589a0555d6311e650cf2b Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Mon, 20 Jul 2026 18:20:10 +0800 Subject: [PATCH 27/31] fix: allow historical alter in transactions --- pkg/sql/compile/alter.go | 7 --- pkg/sql/compile/alter_test.go | 43 +++++++++++++++++++ .../branch/edge/branch_edge_cases.result | 21 +++++++++ .../branch/edge/branch_edge_cases.sql | 20 +++++++++ 4 files changed, 84 insertions(+), 7 deletions(-) diff --git a/pkg/sql/compile/alter.go b/pkg/sql/compile/alter.go index 4fd8a8f171a1c..1ba31e04a10c9 100644 --- a/pkg/sql/compile/alter.go +++ b/pkg/sql/compile/alter.go @@ -219,13 +219,6 @@ func (c *Compile) prepareAlterDataBranchLineage( } } - op := c.proc.GetTxnOperator() - opts := op.TxnOptions() - if err = validateAlterDataBranchLineageTxn( - opts.GetByBegin(), opts.GetAutocommit(), op.Txn().IsPessimistic(), - ); err != nil { - return alterDataBranchLineagePlan{}, err - } return alterDataBranchLineagePlan{ enabled: true, preserveHistoricalSource: preserveHistoricalSource, diff --git a/pkg/sql/compile/alter_test.go b/pkg/sql/compile/alter_test.go index 8430c661aa461..3fb733aa0895b 100644 --- a/pkg/sql/compile/alter_test.go +++ b/pkg/sql/compile/alter_test.go @@ -131,6 +131,49 @@ func TestValidateAlterDataBranchLineageTxn(t *testing.T) { } } +func TestPrepareAlterDataBranchLineageAllowsHistoricalSourceTxn(t *testing.T) { + const ( + oldTableID = uint64(42) + database = "test" + table = "dept" + ) + participationSQL := alterDataBranchParticipationSQL(oldTableID) + snapshotSQL := alterDataBranchHistoricalSnapshotSourceSQL("", database, table, oldTableID) + pitrSQL := alterDataBranchHistoricalPitrSourceSQL("", database, table, oldTableID) + + for _, tc := range []struct { + name string + history string + wantSQLs []string + }{ + { + name: "snapshot", + history: snapshotSQL, + wantSQLs: []string{participationSQL, snapshotSQL}, + }, + { + name: "pitr", + history: pitrSQL, + wantSQLs: []string{participationSQL, snapshotSQL, pitrSQL}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + spyExec := &alterCopyInsertSpyExecutor{results: make(map[string]executor.Result)} + c := newAlterCopyPrecheckCompile(t, ctrl, spyExec) + spyExec.results[tc.history] = newAlterCopyFixedResult( + t, c.proc.Mp(), types.T_int32.ToType(), []int32{1}, + ) + + lineagePlan, err := c.prepareAlterDataBranchLineage(oldTableID, database, table) + require.NoError(t, err) + require.True(t, lineagePlan.enabled) + require.True(t, lineagePlan.preserveHistoricalSource) + require.Equal(t, tc.wantSQLs, spyExec.executedSQLs) + }) + } +} + func TestShouldAdvanceAlterDataBranchLineageSnapshot(t *testing.T) { require.True(t, shouldAdvanceAlterDataBranchLineageSnapshot(true, true)) require.False(t, shouldAdvanceAlterDataBranchLineageSnapshot(true, false)) diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result index 04931e5fb08d6..8c12222b980ed 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result @@ -149,6 +149,27 @@ create database br_txn_src; use br_txn_src; create table base(a int primary key, b varchar(20)); insert into base values (1, 'one'), (2, 'two'); +create table alter_begin(a int primary key, b int); +insert into alter_begin values (1, 10); +create table alter_autocommit(a int primary key, b int); +insert into alter_autocommit values (1, 10); +create snapshot br_alter_begin_s for table br_txn_src alter_begin; +create snapshot br_alter_autocommit_s for table br_txn_src alter_autocommit; +begin; +alter table alter_begin add column c int default 7; +commit; +select a, b, c from alter_begin order by a; +a b c +1 10 7 +set autocommit=0; +alter table alter_autocommit add column c int default 9; +commit; +set autocommit=1; +select a, b, c from alter_autocommit order by a; +a b c +1 10 9 +drop snapshot br_alter_begin_s; +drop snapshot br_alter_autocommit_s; begin; data branch create table br_commit from base; commit; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql index 9f5aebdb9f805..7f1ccb2690ee6 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql @@ -151,6 +151,26 @@ use br_txn_src; create table base(a int primary key, b varchar(20)); insert into base values (1, 'one'), (2, 'two'); +-- Historical snapshot coverage alone must not turn an ordinary ALTER into the +-- explicit-transaction restriction used for live data-branch lineages. +create table alter_begin(a int primary key, b int); +insert into alter_begin values (1, 10); +create table alter_autocommit(a int primary key, b int); +insert into alter_autocommit values (1, 10); +create snapshot br_alter_begin_s for table br_txn_src alter_begin; +create snapshot br_alter_autocommit_s for table br_txn_src alter_autocommit; +begin; +alter table alter_begin add column c int default 7; +commit; +select a, b, c from alter_begin order by a; +set autocommit=0; +alter table alter_autocommit add column c int default 9; +commit; +set autocommit=1; +select a, b, c from alter_autocommit order by a; +drop snapshot br_alter_begin_s; +drop snapshot br_alter_autocommit_s; + begin; data branch create table br_commit from base; commit; From 8df839ed589b8f37b9e51e0392dce95ad65086ae Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Mon, 20 Jul 2026 22:00:21 +0800 Subject: [PATCH 28/31] fix(frontend): reclaim deleted alter lineage --- pkg/frontend/data_branch_snapshot_test.go | 25 +++++++++++ .../databranchutils/branch_dag_test.go | 37 ++++++++++++++++ .../branch_protect_snapshot.go | 19 ++++++--- .../branch/edge/branch_edge_cases.result | 42 +++++++++++++++++++ .../branch/edge/branch_edge_cases.sql | 32 ++++++++++++++ 5 files changed, 150 insertions(+), 5 deletions(-) diff --git a/pkg/frontend/data_branch_snapshot_test.go b/pkg/frontend/data_branch_snapshot_test.go index b5c4feea7e15c..f1ddd8f0498db 100644 --- a/pkg/frontend/data_branch_snapshot_test.go +++ b/pkg/frontend/data_branch_snapshot_test.go @@ -158,6 +158,31 @@ func TestCompactHistoricalAlterLineageWithBHRetainsPitrCoveredEdge(t *testing.T) require.Len(t, bh.executedSQLs, 4) } +func TestCompactHistoricalAlterLineageWithBHReclaimsDeletedRowsWithoutEdges(t *testing.T) { + bh := newHistoricalLineageBackgroundExec( + [][]interface{}{ + {uint64(2), uint64(1), int64(100), uint64(0), "alter", true}, + {uint64(3), uint64(2), int64(200), uint64(0), "alter", true}, + }, + nil, + nil, + nil, + ) + + err := compactHistoricalAlterLineageWithBH( + context.Background(), bh, time.Unix(0, 1_000).UTC(), + ) + require.NoError(t, err) + require.Equal(t, []string{ + historicalAlterLineageMetadataSQL(), + historicalAlterLineageEdgeSQL(), + historicalSnapshotSourceSQL(), + historicalPitrSourceSQL(), + "delete from mo_catalog.mo_snapshots where kind = 'branch' and sname in ('__mo_branch_2','__mo_branch_3')", + "delete from mo_catalog.mo_branch_metadata where table_id in (2,3) and (level = 'alter' or level like 'alter:%')", + }, bh.executedSQLs) +} + func TestCompactHistoricalAlterLineageWithBHPropagatesDeleteError(t *testing.T) { bh := newHistoricalLineageBackgroundExec( [][]interface{}{{uint64(2), uint64(1), int64(100), uint64(0), "alter", false}}, diff --git a/pkg/frontend/databranchutils/branch_dag_test.go b/pkg/frontend/databranchutils/branch_dag_test.go index d3729b23af4ee..b6fb41e122e3c 100644 --- a/pkg/frontend/databranchutils/branch_dag_test.go +++ b/pkg/frontend/databranchutils/branch_dag_test.go @@ -269,6 +269,43 @@ func TestComputeAlterLineageCompactionPlan(t *testing.T) { ) } +func TestComputeAlterLineageCompactionPlanReclaimsDeletedAlterGenerations(t *testing.T) { + rows := []DataBranchMetadata{ + {TableID: 2, PTableID: 1, CloneTS: 100, Level: "alter", TableDeleted: true}, + {TableID: 3, PTableID: 2, CloneTS: 200, Level: "alter"}, + } + edges := map[uint64]HistoricalLineageEdge{ + 2: {ChildTableID: 2, ParentTableID: 1, CloneTS: 100}, + 3: {ChildTableID: 3, ParentTableID: 2, CloneTS: 200}, + } + + require.Equal(t, + AlterLineageCompactionPlan{ + TableIDs: []uint64{2, 3}, + SnapshotNames: []string{"__mo_branch_2", "__mo_branch_3"}, + }, + ComputeAlterLineageCompactionPlan(NewBranchReclaimDag(rows), edges, nil), + ) + + // Plain DROP can reclaim both snapshots while a user snapshot still owns + // the history. After that owner disappears, the deleted metadata rows + // must remain reclaimable even though their matching edges are gone. + rows[1].TableDeleted = true + require.Equal(t, + AlterLineageCompactionPlan{ + TableIDs: []uint64{2, 3}, + SnapshotNames: []string{"__mo_branch_2", "__mo_branch_3"}, + }, + ComputeAlterLineageCompactionPlan(NewBranchReclaimDag(rows), nil, nil), + ) + + covered := []HistoricalSource{{Level: "table", ObjectID: 1, OldestTS: 50}} + require.Empty(t, + ComputeAlterLineageCompactionPlan(NewBranchReclaimDag(rows), nil, covered).TableIDs, + "the dropped current table's historical owner must retain orphaned metadata until owner deletion", + ) +} + func TestComputeAlterLineageCompactionPlanScopeAndOwners(t *testing.T) { baseRows := []DataBranchMetadata{ {TableID: 2, PTableID: 1, CloneTS: 100, Level: "alter"}, diff --git a/pkg/frontend/databranchutils/branch_protect_snapshot.go b/pkg/frontend/databranchutils/branch_protect_snapshot.go index 46daad3028bef..0cb827f4acfc4 100644 --- a/pkg/frontend/databranchutils/branch_protect_snapshot.go +++ b/pkg/frontend/databranchutils/branch_protect_snapshot.go @@ -217,7 +217,7 @@ func historicalSourceOwnsComponent( return false } -// ComputeAlterLineageCompactionPlan finds live ALTER-only edges that no +// ComputeAlterLineageCompactionPlan finds ALTER-only generations that no // longer have an owner. A live logical branch conservatively owns its entire // connected component so sibling/ancestor LCA paths are never disconnected. func ComputeAlterLineageCompactionPlan( @@ -278,17 +278,26 @@ func ComputeAlterLineageCompactionPlan( } for _, tableID := range component { meta, ok := dag.Info[tableID] - if !ok || meta.Deleted || !IsAlterLineageLevel(meta.Level) { + if !ok || !IsAlterLineageLevel(meta.Level) { continue } edge, ok := edges[tableID] - if !ok || edge.ChildTableID != tableID || - edge.ParentTableID != meta.ParentTableID || edge.CloneTS != meta.CloneTS { + if ok && (edge.ChildTableID != tableID || + edge.ParentTableID != meta.ParentTableID || edge.CloneTS != meta.CloneTS) { + continue + } + // A live row without its identity-matched snapshot is retained + // conservatively. A deleted row may legitimately have no snapshot: + // plain DROP reclaims branch snapshots before the last historical + // owner disappears. Once that owner is gone, deleting the orphaned + // metadata row (and the deterministic snapshot name, idempotently) + // closes the lifecycle instead of leaking one row per ALTER. + if !ok && !meta.Deleted { continue } covered := false for _, source := range componentSources { - if source.OldestTS <= edge.CloneTS { + if source.OldestTS <= meta.CloneTS { covered = true break } diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result index 8c12222b980ed..bf1c874839bd3 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result @@ -118,6 +118,48 @@ where table_id = @history_owner_second_tid and level = 'alter'; lineage_after_second_alter 0 drop table history_owner_base; +drop snapshot if exists history_multi_s; +create table history_multi(a int primary key, b int); +create snapshot history_multi_s for table br_schema_drift history_multi; +alter table history_multi add column c int default 0; +set @history_multi_t2 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_multi'); +alter table history_multi add column d int default 0; +set @history_multi_t3 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_multi'); +select count(*) as multi_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_multi_t2, @history_multi_t3) and level = 'alter'; +multi_lineage_before_owner_drop +2 +select count(*) as multi_snapshots_before_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_multi_t2 as char)), concat('__mo_branch_', cast(@history_multi_t3 as char))) and kind = 'branch'; +multi_snapshots_before_owner_drop +2 +drop snapshot history_multi_s; +select count(*) as multi_lineage_after_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_multi_t2, @history_multi_t3) and level = 'alter'; +multi_lineage_after_owner_drop +0 +select count(*) as multi_snapshots_after_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_multi_t2 as char)), concat('__mo_branch_', cast(@history_multi_t3 as char))) and kind = 'branch'; +multi_snapshots_after_owner_drop +0 +drop table history_multi; +drop snapshot if exists history_drop_current_s; +create table history_drop_current(a int primary key, b int); +create snapshot history_drop_current_s for table br_schema_drift history_drop_current; +alter table history_drop_current add column c int default 0; +set @history_drop_current_t2 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_drop_current'); +alter table history_drop_current add column d int default 0; +set @history_drop_current_t3 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_drop_current'); +drop table history_drop_current; +select count(*) as dropped_current_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_drop_current_t2, @history_drop_current_t3) and level = 'alter'; +dropped_current_lineage_before_owner_drop +2 +select count(*) as dropped_current_snapshots_before_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_drop_current_t2 as char)), concat('__mo_branch_', cast(@history_drop_current_t3 as char))) and kind = 'branch'; +dropped_current_snapshots_before_owner_drop +0 +drop snapshot history_drop_current_s; +select count(*) as dropped_current_lineage_after_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_drop_current_t2, @history_drop_current_t3) and level = 'alter'; +dropped_current_lineage_after_owner_drop +0 +select count(*) as dropped_current_snapshots_after_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_drop_current_t2 as char)), concat('__mo_branch_', cast(@history_drop_current_t3 as char))) and kind = 'branch'; +dropped_current_snapshots_after_owner_drop +0 create table history_live_base(a int primary key, b int); data branch create table history_live_child from history_live_base; alter table history_live_base add column c int default 0; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql index 7f1ccb2690ee6..285ccd886134d 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql @@ -119,6 +119,38 @@ select count(*) as lineage_after_second_alter where table_id = @history_owner_second_tid and level = 'alter'; drop table history_owner_base; +-- Two covered ALTERs leave a deleted intermediate generation. Dropping the +-- last historical owner must reclaim both metadata/snapshot pairs. +drop snapshot if exists history_multi_s; +create table history_multi(a int primary key, b int); +create snapshot history_multi_s for table br_schema_drift history_multi; +alter table history_multi add column c int default 0; +set @history_multi_t2 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_multi'); +alter table history_multi add column d int default 0; +set @history_multi_t3 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_multi'); +select count(*) as multi_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_multi_t2, @history_multi_t3) and level = 'alter'; +select count(*) as multi_snapshots_before_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_multi_t2 as char)), concat('__mo_branch_', cast(@history_multi_t3 as char))) and kind = 'branch'; +drop snapshot history_multi_s; +select count(*) as multi_lineage_after_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_multi_t2, @history_multi_t3) and level = 'alter'; +select count(*) as multi_snapshots_after_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_multi_t2 as char)), concat('__mo_branch_', cast(@history_multi_t3 as char))) and kind = 'branch'; +drop table history_multi; + +-- If the current table is dropped first, ordinary DROP reclaims the branch +-- snapshots. The later owner drop must still reclaim the orphaned metadata. +drop snapshot if exists history_drop_current_s; +create table history_drop_current(a int primary key, b int); +create snapshot history_drop_current_s for table br_schema_drift history_drop_current; +alter table history_drop_current add column c int default 0; +set @history_drop_current_t2 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_drop_current'); +alter table history_drop_current add column d int default 0; +set @history_drop_current_t3 = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_drop_current'); +drop table history_drop_current; +select count(*) as dropped_current_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_drop_current_t2, @history_drop_current_t3) and level = 'alter'; +select count(*) as dropped_current_snapshots_before_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_drop_current_t2 as char)), concat('__mo_branch_', cast(@history_drop_current_t3 as char))) and kind = 'branch'; +drop snapshot history_drop_current_s; +select count(*) as dropped_current_lineage_after_owner_drop from mo_catalog.mo_branch_metadata where table_id in (@history_drop_current_t2, @history_drop_current_t3) and level = 'alter'; +select count(*) as dropped_current_snapshots_after_owner_drop from mo_catalog.mo_snapshots where sname in (concat('__mo_branch_', cast(@history_drop_current_t2 as char)), concat('__mo_branch_', cast(@history_drop_current_t3 as char))) and kind = 'branch'; + -- A live logical branch owns the connected ALTER component. Deleting the -- final branch releases both its protect snapshot and the ALTER-only edge. create table history_live_base(a int primary key, b int); From 7a0fb11b0b2493134af326399c504a074862664f Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Wed, 22 Jul 2026 15:00:32 +0800 Subject: [PATCH 29/31] fix(frontend): exclude target-only columns from LCA probes --- pkg/frontend/data_branch_hashdiff.go | 43 +++++++-- pkg/frontend/data_branch_hashdiff_test.go | 94 ++++++++++++++++++- .../branch/diff/diff_schema_evolve.result | 13 +++ .../branch/diff/diff_schema_evolve.sql | 15 +++ .../branch/merge/merge_schema_evolve.result | 14 +++ .../branch/merge/merge_schema_evolve.sql | 16 ++++ 6 files changed, 183 insertions(+), 12 deletions(-) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index c5e2581a1d558..f2ee57901de0c 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -79,21 +79,30 @@ func lcaProbeResultTargetIndexes( return append([]int(nil), layout.targetIdxes...), nil } -// lcaProbeColumnLayout selects only columns that exist in both the LCA and -// target layouts. The caller reconstructs omitted target-only columns as NULL, -// because they cannot exist in an ancestor snapshot. +// lcaProbeColumnLayout selects compatible common columns from the LCA and +// target layouts. The caller reconstructs target-only columns as NULL because +// they are excluded from endpoint comparison and MERGE apply even when an +// ancestor generation still contains an older representation of the column. func lcaProbeColumnLayout( lcaDef *plan2.TableDef, targetDef *plan2.TableDef, targetColNames []string, targetColTypes []types.Type, + targetOnlyIdxes []int, ) (lcaProbeLayout, error) { + targetOnly := make(map[int]struct{}, len(targetOnlyIdxes)) + for _, idx := range targetOnlyIdxes { + targetOnly[idx] = struct{}{} + } + if len(lcaDef.Cols) == 0 { - layout := lcaProbeLayout{ - attrs: append([]string(nil), targetColNames...), - types: append([]types.Type(nil), targetColTypes...), - } - for i := range targetColNames { + layout := lcaProbeLayout{} + for i, name := range targetColNames { + if _, ok := targetOnly[i]; ok { + continue + } + layout.attrs = append(layout.attrs, name) + layout.types = append(layout.types, targetColTypes[i]) layout.targetIdxes = append(layout.targetIdxes, i) layout.enumValues = append(layout.enumValues, "") } @@ -105,6 +114,9 @@ func lcaProbeColumnLayout( layout := lcaProbeLayout{} for targetIdx, name := range targetColNames { + if _, ok := targetOnly[targetIdx]; ok { + continue + } targetCol := dataBranchColumnDefByName(targetDef, name) if targetCol == nil { return lcaProbeLayout{}, moerr.NewInternalErrorNoCtxf( @@ -115,6 +127,13 @@ func lcaProbeColumnLayout( if lcaCol == nil { continue } + if lcaCol.Name == catalog.Row_ID || + !isDataBranchLogicalTypeEquivalent(lcaCol.Typ, targetCol.Typ) { + return lcaProbeLayout{}, moerr.NewNotSupportedNoCtxf( + "LCA data branch column %s has a different type from the endpoint schema", + targetCol.Name, + ) + } layout.attrs = append(layout.attrs, lcaCol.Name) layout.types = append(layout.types, types.New(types.T(lcaCol.Typ.Id), lcaCol.Typ.Width, lcaCol.Typ.Scale)) layout.targetIdxes = append(layout.targetIdxes, targetIdx) @@ -151,6 +170,7 @@ func handleDelsOnLCA( ) lcaLayout, err := lcaProbeColumnLayout( lcaTblDef, targetTblDef, tblStuff.def.colNames, tblStuff.def.colTypes, + tblStuff.def.tarOnlyIdxes, ) if err != nil { return nil, err @@ -384,9 +404,13 @@ func handleDelsOnLCA( for j := 1; j < len(cols); j++ { targetIdx := resultTargetIdxes[j-1] + enumValues := "" + if layoutIdx := slices.Index(lcaLayout.targetIdxes, targetIdx); layoutIdx >= 0 { + enumValues = lcaLayout.enumValues[layoutIdx] + } if err = appendLCAProbeValue( dBat.Vecs[targetIdx], cols[j], i, - lcaLayout.enumValues[j-1], ses.proc.Mp(), + enumValues, ses.proc.Mp(), ); err != nil { return false } @@ -567,6 +591,7 @@ func runLCAProbeWithReaderFallback( targetTblDef := tblStuff.tarRel.GetTableDef(ctx) lcaLayout, err := lcaProbeColumnLayout( lcaTblDef, targetTblDef, tblStuff.def.colNames, tblStuff.def.colTypes, + tblStuff.def.tarOnlyIdxes, ) if err != nil { return executor.Result{}, err diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index f0be93db763c2..128a9e39f45b9 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -1082,6 +1082,7 @@ func TestLCAProbeColumnLayoutExcludesTargetOnlyColumns(t *testing.T) { targetDef, []string{"id", "name", "added"}, []types.Type{types.T_int64.ToType(), types.T_varchar.ToType(), types.T_int64.ToType()}, + []int{2}, ) require.NoError(t, err) @@ -1090,6 +1091,88 @@ func TestLCAProbeColumnLayoutExcludesTargetOnlyColumns(t *testing.T) { require.Equal(t, []types.T{types.T_int64, types.T_varchar}, []types.T{layout.types[0].Oid, layout.types[1].Oid}) } +func TestLCAProbeColumnLayoutExcludesTargetOnlyColumnsWithoutLCAMetadata(t *testing.T) { + layout, err := lcaProbeColumnLayout( + &plan.TableDef{}, + &plan.TableDef{}, + []string{"a", "c"}, + []types.Type{types.T_int64.ToType(), types.T_int32.ToType()}, + []int{1}, + ) + + require.NoError(t, err) + require.Equal(t, []string{"a"}, layout.attrs) + require.Equal(t, []int{0}, layout.targetIdxes) +} + +func TestLCAProbeColumnLayoutExcludesIncompatibleTargetOnlyColumn(t *testing.T) { + lcaDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_int32)}}, + {Name: catalog.Row_ID, ColId: 3, Seqnum: 2, Typ: plan.Type{Id: int32(types.T_Rowid)}}, + }} + targetDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_varchar), Width: 20}}, + }} + + layout, err := lcaProbeColumnLayout( + lcaDef, + targetDef, + []string{"a", "c"}, + []types.Type{types.T_int64.ToType(), types.New(types.T_varchar, 20, 0)}, + []int{1}, + ) + + require.NoError(t, err) + require.Equal(t, []string{"a"}, layout.attrs) + require.Equal(t, []int{0}, layout.targetIdxes) +} + +func TestLCAProbeColumnLayoutRejectsIncompatibleCommonColumn(t *testing.T) { + lcaDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_int32)}}, + }} + targetDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_varchar), Width: 20}}, + }} + + _, err := lcaProbeColumnLayout( + lcaDef, + targetDef, + []string{"a", "b"}, + []types.Type{types.T_int64.ToType(), types.New(types.T_varchar, 20, 0)}, + nil, + ) + + require.ErrorContains(t, err, "column b has a different type") +} + +func TestLCAProbeColumnLayoutIgnoresTargetOnlyRowIDCollision(t *testing.T) { + lcaDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: catalog.Row_ID, ColId: 3, Seqnum: 2, Typ: plan.Type{Id: int32(types.T_Rowid)}}, + }} + targetDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", ColId: 3, Seqnum: 2, Typ: plan.Type{Id: int32(types.T_int32)}}, + }} + + layout, err := lcaProbeColumnLayout( + lcaDef, + targetDef, + []string{"a", "c"}, + []types.Type{types.T_int64.ToType(), types.T_int32.ToType()}, + []int{1}, + ) + + require.NoError(t, err) + require.Equal(t, []string{"a"}, layout.attrs) + require.Equal(t, []int{0}, layout.targetIdxes) +} + func TestLCAProbeResultTargetIndexes(t *testing.T) { layout := lcaProbeLayout{targetIdxes: []int{1, 2}} @@ -1809,7 +1892,7 @@ func TestHashDiff_HasLCAUpdateIgnoresTargetOnlyColumns(t *testing.T) { bh := mock_frontend.NewMockBackgroundExec(ctrl) bh.EXPECT().Exec(gomock.Any(), gomock.Any()).Return(nil).Times(1) bh.EXPECT().GetExecResultSet().Return([]interface{}{ - buildLCAProbeResultSetWithRows([]interface{}{int64(0), int64(1), "before", nil}), + buildLCAProbeResultSetWithColumnCount(3, []interface{}{int64(0), int64(1), "before"}), }).Times(1) bh.EXPECT().ClearExecResultSet().Times(1) @@ -2051,8 +2134,12 @@ func buildLCAProbeResultSet() *MysqlResultSet { } func buildLCAProbeResultSetWithRows(rows ...[]interface{}) *MysqlResultSet { + return buildLCAProbeResultSetWithColumnCount(4, rows...) +} + +func buildLCAProbeResultSetWithColumnCount(columnCount int, rows ...[]interface{}) *MysqlResultSet { mrs := &MysqlResultSet{} - for _, col := range []struct { + cols := []struct { name string typ defines.MysqlType }{ @@ -2060,7 +2147,8 @@ func buildLCAProbeResultSetWithRows(rows ...[]interface{}) *MysqlResultSet { {name: "id", typ: defines.MYSQL_TYPE_LONGLONG}, {name: "name", typ: defines.MYSQL_TYPE_VARCHAR}, {name: "hidden", typ: defines.MYSQL_TYPE_VARCHAR}, - } { + } + for _, col := range cols[:columnCount] { mysqlCol := &MysqlColumn{} mysqlCol.SetName(col.name) mysqlCol.SetColumnType(col.typ) diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index 0c63c11ef6ae2..6ae71cd42e8cf 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -316,4 +316,17 @@ data branch diff t1 against t0; internal error: schema compatibility check: column 'b' has different identity drop table t1; drop table t0; +create table t0(a int primary key, b int, c int); +insert into t0 values(1,1,1); +data branch create table t1 from t0; +data branch create table t2 from t0; +alter table t1 modify column c varchar(20); +alter table t2 drop column c; +update t1 set b=11, c='x' where a=1; +data branch diff t1 against t2; +diff t1 against t2 flag a b c +t1 UPDATE 1 11 x +drop table t2; +drop table t1; +drop table t0; drop database test; diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 7e2474cea9947..5366421b4e7b8 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -435,4 +435,19 @@ data branch diff t1 against t0; drop table t1; drop table t0; +-- ===================================================== +-- Case 25: incompatible target-only LCA columns are not probed +-- ===================================================== +create table t0(a int primary key, b int, c int); +insert into t0 values(1,1,1); +data branch create table t1 from t0; +data branch create table t2 from t0; +alter table t1 modify column c varchar(20); +alter table t2 drop column c; +update t1 set b=11, c='x' where a=1; +data branch diff t1 against t2; +drop table t2; +drop table t1; +drop table t0; + drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index aa9af10034fe2..17cfea944be92 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -166,4 +166,18 @@ a b 1 1 drop table t1; drop table t0; +create table t0(a int primary key, b int, c int); +insert into t0 values(1,1,1); +data branch create table t1 from t0; +data branch create table t2 from t0; +alter table t1 modify column c varchar(20); +alter table t2 drop column c; +update t1 set b=11, c='x' where a=1; +data branch merge t1 into t2; +select * from t2; +a b +1 11 +drop table t2; +drop table t1; +drop table t0; drop database test; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index 2499daf8e770c..3de5bfa0190a2 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -212,4 +212,20 @@ select * from t0; drop table t1; drop table t0; +-- ===================================================== +-- Case 11: incompatible target-only LCA columns are not probed +-- ===================================================== +create table t0(a int primary key, b int, c int); +insert into t0 values(1,1,1); +data branch create table t1 from t0; +data branch create table t2 from t0; +alter table t1 modify column c varchar(20); +alter table t2 drop column c; +update t1 set b=11, c='x' where a=1; +data branch merge t1 into t2; +select * from t2; +drop table t2; +drop table t1; +drop table t0; + drop database test; From 5b57d3fa0ec18283cb15397262e5b163d04c00a6 Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Wed, 22 Jul 2026 15:59:17 +0800 Subject: [PATCH 30/31] fix(frontend): preserve derived composite key lineage --- pkg/frontend/data_branch_hashdiff.go | 69 ++++++++++++++-- pkg/frontend/data_branch_hashdiff_test.go | 79 +++++++++++++++++++ pkg/frontend/data_branch_snapshot_test.go | 55 +++++++++++++ pkg/sql/compile/alter.go | 9 ++- pkg/sql/compile/alter_test.go | 16 ++++ .../branch/diff/diff_schema_evolve.result | 6 +- .../branch/diff/diff_schema_evolve.sql | 9 ++- .../branch/merge/merge_schema_evolve.result | 8 +- .../branch/merge/merge_schema_evolve.sql | 12 +-- 9 files changed, 240 insertions(+), 23 deletions(-) diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index f2ee57901de0c..15ffc9297e04e 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -124,11 +124,23 @@ func lcaProbeColumnLayout( ) } lcaCol := dataBranchColumnDefByIdentity(lcaDef, targetCol) + candidate := dataBranchColumnDefByName(lcaDef, targetCol.Name) + if candidate != nil && + isDataBranchDerivedCompositePKColumn(lcaDef, targetDef, candidate, targetCol) { + // A rebuilt composite-key column can reuse the former RowID's + // numeric ColId/Seqnum pair. Prefer its canonical derived identity + // over that accidental positional collision. + lcaCol = candidate + } if lcaCol == nil { continue } + derivedCompositePK := isDataBranchDerivedCompositePKColumn( + lcaDef, targetDef, lcaCol, targetCol, + ) if lcaCol.Name == catalog.Row_ID || - !isDataBranchLogicalTypeEquivalent(lcaCol.Typ, targetCol.Typ) { + (!isDataBranchLogicalTypeEquivalent(lcaCol.Typ, targetCol.Typ) && + !(derivedCompositePK && lcaCol.Typ.Id == targetCol.Typ.Id)) { return lcaProbeLayout{}, moerr.NewNotSupportedNoCtxf( "LCA data branch column %s has a different type from the endpoint schema", targetCol.Name, @@ -269,15 +281,16 @@ func handleDelsOnLCA( ) sqlBuf.WriteString(fmt.Sprintf( "right join (values %s) as pks(__idx_,%s) on ", - valsBuf.String(), strings.Join(pkNames, ",")), + valsBuf.String(), joinQuotedColumnNames(pkNames)), ) for i := range pkNames { - sqlBuf.WriteString(fmt.Sprintf("lca.%s = ", pkNames[i])) + quotedPKName := quoteIdentifierForSQL(pkNames[i]) + sqlBuf.WriteString(fmt.Sprintf("lca.%s = ", quotedPKName)) if castType, ok := lcaProbeJoinCastType(colTypes[expandedPKColIdxes[i]]); ok { - sqlBuf.WriteString(fmt.Sprintf("cast(pks.%s as %s)", pkNames[i], castType)) + sqlBuf.WriteString(fmt.Sprintf("cast(pks.%s as %s)", quotedPKName, castType)) } else { - sqlBuf.WriteString(fmt.Sprintf("pks.%s", pkNames[i])) + sqlBuf.WriteString(fmt.Sprintf("pks.%s", quotedPKName)) } if i != len(pkNames)-1 { sqlBuf.WriteString(" AND ") @@ -2715,8 +2728,12 @@ func dataBranchSourceColToTargetIdx( continue } targetCol, ok := targetCols[strings.ToLower(col.Name)] - if !ok || col.Seqnum != targetCol.Seqnum || col.Typ.Id != targetCol.Typ.Id || - !dataBranchColumnTypeAttributesEqual(col.Typ, targetCol.Typ) { + derivedCompositePK := ok && + isDataBranchDerivedCompositePKColumn(sourceDef, targetDef, col, targetCol) + sameIdentity := ok && (col.Seqnum == targetCol.Seqnum || derivedCompositePK) + sameType := ok && col.Typ.Id == targetCol.Typ.Id && + (dataBranchColumnTypeAttributesEqual(col.Typ, targetCol.Typ) || derivedCompositePK) + if !sameIdentity || !sameType { if _, isTargetOnly := targetOnly[targetIdx]; isTargetOnly { // This column does not exist on the base endpoint and is excluded // from comparison and MERGE apply. Its old physical representation @@ -2725,7 +2742,7 @@ func dataBranchSourceColToTargetIdx( mapping[len(mapping)-1] = -1 continue } - if ok && col.Seqnum != targetCol.Seqnum { + if ok && !sameIdentity { return nil, moerr.NewNotSupportedNoCtxf( "historical data branch column %s has a different identity from the endpoint schema", col.Name, @@ -2740,6 +2757,42 @@ func dataBranchSourceColToTargetIdx( return mapping, nil } +// isDataBranchDerivedCompositePKColumn reports whether the two columns are the +// hidden serialization of the same logical composite primary key. COPY ALTER +// rebuilds this derived column and may assign it a new Seqnum even though the +// component-column identities, which checkDataBranchPrimaryKeyCompatibility +// has already validated, remain unchanged. +func isDataBranchDerivedCompositePKColumn( + sourceDef, targetDef *plan2.TableDef, + sourceCol, targetCol *plan2.ColDef, +) bool { + if sourceDef == nil || targetDef == nil || sourceCol == nil || targetCol == nil || + sourceDef.Pkey == nil || targetDef.Pkey == nil || + !strings.EqualFold(sourceDef.Pkey.PkeyColName, catalog.CPrimaryKeyColName) || + !strings.EqualFold(targetDef.Pkey.PkeyColName, catalog.CPrimaryKeyColName) || + !strings.EqualFold(sourceCol.Name, catalog.CPrimaryKeyColName) || + !strings.EqualFold(targetCol.Name, catalog.CPrimaryKeyColName) || + len(sourceDef.Pkey.Names) != len(targetDef.Pkey.Names) { + return false + } + for i, sourceName := range sourceDef.Pkey.Names { + targetName := targetDef.Pkey.Names[i] + if !strings.EqualFold(sourceName, targetName) { + return false + } + sourcePart := dataBranchColumnDefByName(sourceDef, sourceName) + targetPart := dataBranchColumnDefByName(targetDef, targetName) + if sourcePart == nil || targetPart == nil || + sourcePart.Seqnum != targetPart.Seqnum || + sourcePart.Typ.Id != targetPart.Typ.Id || + !dataBranchColumnTypeAttributesEqual(sourcePart.Typ, targetPart.Typ) || + sourcePart.NotNull != targetPart.NotNull { + return false + } + } + return true +} + // dataBranchNeedsHistoricalProjection reports whether change rows from an // older physical table generation have a different data-column layout from // the endpoint. A different table ID alone can also mean an ordinary branch diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index 128a9e39f45b9..d072aae8579ed 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -2683,6 +2683,85 @@ func TestDataBranchSourceColToTargetIdxRejectsHistoricalIdentityDrift(t *testing "target-only historical identity drift must be projected as NULL") } +func TestDataBranchSourceColToTargetIdxPreservesRebuiltCompositePrimaryKey(t *testing.T) { + compositePK := func(seqnum uint32) (*plan.ColDef, *plan.PrimaryKeyDef) { + col := &plan.ColDef{ + Name: catalog.CPrimaryKeyColName, Hidden: true, Seqnum: seqnum, + Typ: plan.Type{Id: int32(types.T_varchar)}, + } + return col, &plan.PrimaryKeyDef{ + PkeyColName: catalog.CPrimaryKeyColName, + Names: []string{"select", "line item"}, + CompPkeyCol: col, + } + } + sourceCPK, sourcePK := compositePK(2) + targetCPK, targetPK := compositePK(3) + sourceCPK.Typ.Width = 100 + targetCPK.Typ.Width = 200 + sourceDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "select", Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "line item", Seqnum: 1, Typ: plan.Type{Id: int32(types.T_int64)}}, + sourceCPK, + {Name: catalog.Row_ID, Hidden: true, Seqnum: 3, Typ: plan.Type{Id: int32(types.T_Rowid)}}, + }, + Pkey: sourcePK, + } + targetDef := &plan.TableDef{ + Cols: []*plan.ColDef{ + {Name: "select", Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "line item", Seqnum: 1, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "payload", Seqnum: 2, Typ: plan.Type{Id: int32(types.T_int64)}}, + targetCPK, + }, + Pkey: targetPK, + } + + mapping, err := dataBranchSourceColToTargetIdx( + sourceDef, targetDef, + []string{"select", "line item", "payload", catalog.CPrimaryKeyColName}, + []int{2}, + ) + require.NoError(t, err) + require.Equal(t, []int{0, 1, 3}, mapping) + layout, err := lcaProbeColumnLayout( + sourceDef, targetDef, + []string{"select", "line item", "payload", catalog.CPrimaryKeyColName}, + []types.Type{ + types.T_int64.ToType(), types.T_int64.ToType(), + types.T_int64.ToType(), types.T_varchar.ToType(), + }, + []int{2}, + ) + require.NoError(t, err) + require.Equal(t, []string{"select", "line item", catalog.CPrimaryKeyColName}, layout.attrs) + require.Equal(t, []int{0, 1, 3}, layout.targetIdxes) + + targetDef.Pkey.Names = []string{"select", "other"} + require.False(t, isDataBranchDerivedCompositePKColumn(sourceDef, targetDef, sourceCPK, targetCPK)) + _, err = dataBranchSourceColToTargetIdx( + sourceDef, targetDef, + []string{"select", "line item", "payload", catalog.CPrimaryKeyColName}, + []int{2}, + ) + require.ErrorContains(t, err, "primary key is incompatible") + + targetDef.Pkey.Names = []string{"select", "line item"} + targetDef.Cols[1].Seqnum = 4 + require.False(t, isDataBranchDerivedCompositePKColumn(sourceDef, targetDef, sourceCPK, targetCPK)) + _, err = dataBranchSourceColToTargetIdx( + sourceDef, targetDef, + []string{"select", "line item", "payload", catalog.CPrimaryKeyColName}, + []int{2}, + ) + require.ErrorContains(t, err, "column line item has a different identity") + + targetDef.Cols[1].Seqnum = 1 + targetDef.Cols[1].Typ.Id = int32(types.T_varchar) + require.False(t, isDataBranchDerivedCompositePKColumn(sourceDef, targetDef, sourceCPK, targetCPK)) +} + func TestDataBranchSourceColToTargetIdxPreservesCompatibleAndReorderedColumns(t *testing.T) { sourceDef := &plan.TableDef{ Cols: []*plan.ColDef{ diff --git a/pkg/frontend/data_branch_snapshot_test.go b/pkg/frontend/data_branch_snapshot_test.go index f1ddd8f0498db..220279f648518 100644 --- a/pkg/frontend/data_branch_snapshot_test.go +++ b/pkg/frontend/data_branch_snapshot_test.go @@ -27,12 +27,15 @@ import ( "time" "github.com/golang/mock/gomock" + "github.com/prashantv/gostub" "github.com/stretchr/testify/require" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/config" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/frontend/databranchutils" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" ) // --------------------------------------------------------------------------- @@ -203,6 +206,58 @@ func TestCompactHistoricalAlterLineageWithBHPropagatesDeleteError(t *testing.T) ) } +func TestDoDropSnapshotRollsBackCompactionFailure(t *testing.T) { + ctrl := gomock.NewController(t) + ses := newTestSession(t, ctrl) + defer ses.Close() + + bh := newHistoricalLineageBackgroundExec( + [][]interface{}{{uint64(2), uint64(1), int64(100), uint64(0), "alter", false}}, + [][]interface{}{{"__mo_branch_2", int64(100), "acc", "db", "t", uint64(1)}}, + nil, + nil, + ) + bhStub := gostub.StubFunc(&NewBackgroundExec, bh) + defer bhStub.Reset() + + pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) + pu.SV.SetDefaultValues() + pu.SV.KillRountinesInterval = 0 + setPu("", pu) + ctx := context.WithValue(context.Background(), config.ParameterUnitKey, pu) + rm, _ := NewRoutineManager(ctx, "") + ses.rm = rm + ses.SetTenantInfo(&TenantInfo{ + Tenant: sysAccountName, + User: rootName, + DefaultRole: moAdminRoleName, + TenantID: sysAccountID, + UserID: rootID, + DefaultRoleID: moAdminRoleID, + }) + + stmt := &tree.DropSnapShot{Name: tree.Identifier("snapshot_test")} + checkSQL, _ := getSqlForCheckSnapshot(ctx, "snapshot_test") + bh.sql2result[checkSQL] = newMrsForPasswordOfUser([][]interface{}{{0, 0}}) + kindSQL := "select kind from mo_catalog.mo_snapshots where sname = 'snapshot_test' order by snapshot_id limit 1" + bh.sql2result[kindSQL] = newMrsForPasswordOfUser([][]interface{}{{"user"}}) + userDeleteSQL := getSqlForDropSnapshot("snapshot_test") + branchDeleteSQL := "delete from mo_catalog.mo_snapshots where kind = 'branch' and sname in ('__mo_branch_2')" + wantErr := errors.New("branch snapshot delete failed") + bh.sql2err[branchDeleteSQL] = wantErr + + err := doDropSnapshot(ctx, ses, stmt) + require.ErrorIs(t, err, wantErr) + require.Contains(t, bh.executedSQLs, "begin;") + require.Contains(t, bh.executedSQLs, userDeleteSQL) + require.Contains(t, bh.executedSQLs, branchDeleteSQL) + require.Contains(t, bh.executedSQLs, "rollback;") + require.NotContains(t, bh.executedSQLs, "commit;") + require.NotContains(t, bh.executedSQLs, + "delete from mo_catalog.mo_branch_metadata where table_id in (2) and (level = 'alter' or level like 'alter:%')", + ) +} + // --------------------------------------------------------------------------- // UT-U2 — buildDagFromRows (DAG adjacency construction) // --------------------------------------------------------------------------- diff --git a/pkg/sql/compile/alter.go b/pkg/sql/compile/alter.go index 1ba31e04a10c9..c6e7b8653934c 100644 --- a/pkg/sql/compile/alter.go +++ b/pkg/sql/compile/alter.go @@ -17,6 +17,7 @@ package compile import ( "context" "fmt" + "math" "slices" "strings" "time" @@ -240,7 +241,13 @@ func shouldAdvanceAlterDataBranchLineageSnapshot(pessimistic, rcIsolation bool) func (c *Compile) advanceAlterDataBranchLineageSnapshot() (int64, error) { op := c.proc.GetTxnOperator() - requested := op.SnapshotTS().PhysicalTime + int64(time.Microsecond) + physicalTime := op.SnapshotTS().PhysicalTime + if physicalTime > math.MaxInt64-int64(time.Microsecond) { + return 0, moerr.NewInternalErrorNoCtx( + "cannot advance ALTER data-branch lineage snapshot past the timestamp limit", + ) + } + requested := physicalTime + int64(time.Microsecond) if err := op.UpdateSnapshot(c.proc.Ctx, timestamp.Timestamp{PhysicalTime: requested}); err != nil { return 0, err } diff --git a/pkg/sql/compile/alter_test.go b/pkg/sql/compile/alter_test.go index 3fb733aa0895b..32b6a2b6827a6 100644 --- a/pkg/sql/compile/alter_test.go +++ b/pkg/sql/compile/alter_test.go @@ -17,6 +17,7 @@ package compile import ( "context" "errors" + "math" "testing" "time" @@ -43,6 +44,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/pb/lock" plan2 "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/sql/plan" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/util/executor" @@ -181,6 +183,20 @@ func TestShouldAdvanceAlterDataBranchLineageSnapshot(t *testing.T) { require.False(t, shouldAdvanceAlterDataBranchLineageSnapshot(false, false)) } +func TestAdvanceAlterDataBranchLineageSnapshotRejectsOverflow(t *testing.T) { + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().SnapshotTS().Return(timestamp.Timestamp{ + PhysicalTime: math.MaxInt64 - int64(time.Microsecond) + 1, + }) + + proc := testutil.NewProcess(t) + proc.Base.TxnOperator = txnOp + c := &Compile{proc: proc} + _, err := c.advanceAlterDataBranchLineageSnapshot() + require.ErrorContains(t, err, "timestamp limit") +} + type alterCopyInsertSpyExecutor struct { insertSQL string insertErr error diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result index 6ae71cd42e8cf..d9fb436c28843 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.result @@ -82,15 +82,17 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; -create table t0(a int, b int, c int, primary key(a,b)); +create table t0(`select` int, `line item` int, c int, primary key(`select`,`line item`)); insert into t0 values(1,1,10),(2,2,20); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; +update t1 set c=11 where `select`=1 and `line item`=1; alter table t1 add column d int default 0; insert into t1 values(3,3,30,300); create snapshot sp1 for table test t1; data branch diff t1{snapshot="sp1"} against t0{snapshot="sp0"}; -diff t1 against t0 flag a b c d +diff t1 against t0 flag select line item c d +t1 UPDATE 1 1 11 0 t1 INSERT 3 3 30 300 drop snapshot sp1; drop snapshot sp0; diff --git a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql index 5366421b4e7b8..b98fa5499b270 100644 --- a/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/diff/diff_schema_evolve.sql @@ -126,14 +126,17 @@ drop table t1; -- ===================================================== -- Case 6: composite primary key + extra column --- base: [a, b, c] PK(a, b) --- target: [a, b, c, d] +-- base: [`select`, `line item`, c] composite PK with quoted/reserved names +-- target: [`select`, `line item`, c, d] -- ===================================================== -create table t0(a int, b int, c int, primary key(a,b)); +create table t0(`select` int, `line item` int, c int, primary key(`select`,`line item`)); insert into t0 values(1,1,10),(2,2,20); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; +-- The UPDATE belongs to the pre-ALTER physical generation. COPY ALTER rebuilds +-- __mo_cpkey_col with a new Seqnum; DIFF must still map that derived key. +update t1 set c=11 where `select`=1 and `line item`=1; alter table t1 add column d int default 0; insert into t1 values(3,3,30,300); create snapshot sp1 for table test t1; diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result index 17cfea944be92..ca41767760429 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.result @@ -55,17 +55,17 @@ drop snapshot sp1; drop snapshot sp0; drop table t0; drop table t1; -create table t0(a int, b int, c int, primary key(a,b)); +create table t0(`select` int, `line item` int, c int, primary key(`select`,`line item`)); insert into t0 values(1,1,10),(2,2,20); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; +update t1 set c=99 where `select`=1 and `line item`=1; alter table t1 add column d int default 0; -update t1 set c=99 where a=1 and b=1; insert into t1 values(3,3,30,300); create snapshot sp1 for table test t1; data branch merge t1 into t0 when conflict accept; -select * from t0 order by a; -a b c +select * from t0 order by `select`; +select line item c 1 1 99 2 2 20 3 3 30 diff --git a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql index 3de5bfa0190a2..fd251533b8fad 100644 --- a/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql +++ b/test/distributed/cases/git4data/branch/merge/merge_schema_evolve.sql @@ -77,23 +77,25 @@ drop table t1; -- ===================================================== -- Case 4: merge with composite PK + extra column --- base: [a, b, c] PK(a, b) --- target: [a, b, c, d] +-- base: [`select`, `line item`, c] composite PK with quoted/reserved names +-- target: [`select`, `line item`, c, d] -- ===================================================== -create table t0(a int, b int, c int, primary key(a,b)); +create table t0(`select` int, `line item` int, c int, primary key(`select`,`line item`)); insert into t0 values(1,1,10),(2,2,20); create snapshot sp0 for table test t0; data branch create table t1 from t0{snapshot="sp0"}; +-- Keep this UPDATE in the pre-ALTER generation so MERGE exercises historical +-- mapping after COPY ALTER rebuilds the hidden composite-key column. +update t1 set c=99 where `select`=1 and `line item`=1; alter table t1 add column d int default 0; -update t1 set c=99 where a=1 and b=1; insert into t1 values(3,3,30,300); create snapshot sp1 for table test t1; data branch merge t1 into t0 when conflict accept; -- t0: (1,1) c updated to 99, (3,3) inserted, d is not written -select * from t0 order by a; +select * from t0 order by `select`; drop snapshot sp1; drop snapshot sp0; From af4e2f721aa7a26d4a8f1f91ecf7ec936ff2420f Mon Sep 17 00:00:00 2001 From: VioletQwQ-0 <1659648118@qq.com> Date: Wed, 22 Jul 2026 18:20:02 +0800 Subject: [PATCH 31/31] fix(frontend): validate data branch column lineage --- pkg/frontend/data_branch.go | 167 ++++++++++++++++-- pkg/frontend/data_branch_hashdiff.go | 46 ++--- pkg/frontend/data_branch_hashdiff_test.go | 65 +++++-- pkg/frontend/data_branch_output.go | 21 +++ pkg/frontend/data_branch_test.go | 94 +++++++++- .../branch/edge/branch_edge_cases.result | 2 +- .../branch/edge/branch_edge_cases.sql | 2 +- ...ate_snapshot_for_sys_db_table_level.result | 8 +- ...create_snapshot_for_sys_db_table_level.sql | 8 +- 9 files changed, 353 insertions(+), 60 deletions(-) diff --git a/pkg/frontend/data_branch.go b/pkg/frontend/data_branch.go index 3e77ce9b15f01..148097176a8c7 100644 --- a/pkg/frontend/data_branch.go +++ b/pkg/frontend/data_branch.go @@ -1053,11 +1053,12 @@ func getTableStuff( tblStuff.def.baseColToTarIdx[i] = -1 } for i, name := range baseDataNames { - for j, tarName := range tblStuff.def.colNames { - if strings.EqualFold(name, tarName) { - tblStuff.def.baseColToTarIdx[i] = j - break - } + baseCol := dataBranchColumnDefByName(baseTblDef, name) + tarCol := dataBranchColumnDefByLogicalName(tarTblDef, baseCol) + if tarCol != nil { + tblStuff.def.baseColToTarIdx[i] = dataBranchColumnIndexByName( + tblStuff.def.colNames, tarCol.Name, + ) } } } @@ -1227,14 +1228,12 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm return } - baseColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) baseVisibleColMap := make(map[string]*plan.ColDef, len(baseDef.Cols)) for _, col := range baseDef.Cols { if col.Name == catalog.Row_ID { continue } name := strings.ToLower(col.Name) - baseColMap[name] = col if isDataBranchUserVisibleColumn(col) { baseVisibleColMap[name] = col } @@ -1253,15 +1252,8 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm } name := strings.ToLower(tarCol.Name) - baseCol, found := baseColMap[name] - if found { - if baseCol.Seqnum != tarCol.Seqnum { - err = moerr.NewInternalErrorNoCtxf( - "schema compatibility check: column '%s' has different identity", - tarCol.Name, - ) - return - } + baseCol := dataBranchColumnDefByLogicalName(baseDef, tarCol) + if baseCol != nil { if baseCol.Typ.Id == tarCol.Typ.Id { if !isDataBranchLogicalTypeEquivalent(baseCol.Typ, tarCol.Typ) { err = moerr.NewInternalErrorNoCtxf( @@ -1281,7 +1273,8 @@ func checkSchemaCompatibility(tarDef, baseDef *plan.TableDef) (commonIdxes, comm if isDataBranchUserVisibleColumn(tarCol) { commonVisibleIdxes = append(commonVisibleIdxes, dataIdx) commonColNameSet[name] = true - delete(baseVisibleColMap, name) + commonColNameSet[strings.ToLower(baseCol.Name)] = true + delete(baseVisibleColMap, strings.ToLower(baseCol.Name)) } } else { err = moerr.NewInternalErrorNoCtxf( @@ -1733,6 +1726,128 @@ func constructChangeHandle( return } +func dataBranchPathTableDefs( + ctx context.Context, + ses *Session, + bh BackgroundExec, + endpointRel engine.Relation, + endpointSP types.TS, + path []uint64, + pathTS []types.TS, +) ([]*plan.TableDef, error) { + if len(path) == 0 || len(path) != len(pathTS) { + return nil, moerr.NewInternalErrorNoCtx("data branch: invalid schema lineage path") + } + defs := make([]*plan.TableDef, len(path)) + for i, nodeID := range path { + if i == len(path)-1 { + defs[i] = endpointRel.GetTableDef(ctx) + continue + } + snapshot := dataBranchCollectRelationSnapshot(endpointSP, pathTS[i+1], false).ToTimestamp() + rel, err := getRelationById( + ctx, ses, bh, nodeID, &plan.Snapshot{ + Tenant: &plan.SnapshotTenant{TenantID: ses.GetAccountId()}, + TS: &snapshot, + }, + ) + if err != nil { + return nil, err + } + defs[i] = rel.GetTableDef(ctx) + } + return defs, nil +} + +func dataBranchColumnReachesLCA( + pathDefs []*plan.TableDef, + lineageOnly []bool, + endpointCol *plan.ColDef, +) (reachesLCA bool, lcaCol *plan.ColDef, redefined bool) { + if len(pathDefs) == 0 || len(pathDefs) != len(lineageOnly) || endpointCol == nil { + return false, nil, false + } + current := endpointCol + for i := len(pathDefs) - 2; i >= 0; i-- { + previous := dataBranchColumnDefByName(pathDefs[i], current.Name) + if previous == nil && !lineageOnly[i+1] { + // An in-place rename on an ordinary clone edge keeps OriginName, + // which links the descendant name back to its ancestor name. + previous = dataBranchColumnDefByLogicalName(pathDefs[i], current) + } + if previous == nil { + for j := i - 1; j >= 0; j-- { + if dataBranchColumnDefByLogicalName(pathDefs[j], endpointCol) != nil { + return false, nil, true + } + } + return false, nil, false + } + current = previous + } + return true, current, false +} + +func dataBranchLCASchemaContainsColumn(lcaDef *plan.TableDef, col *plan.ColDef) bool { + if lcaDef == nil || col == nil { + return false + } + if dataBranchColumnDefByName(lcaDef, col.Name) != nil { + return true + } + originName := col.GetOriginCaseName() + return !strings.EqualFold(originName, col.Name) && + dataBranchColumnDefByName(lcaDef, originName) != nil +} + +func validateDataBranchColumnLineage( + tarDefs []*plan.TableDef, + tarLineageOnly []bool, + baseDefs []*plan.TableDef, + baseLineageOnly []bool, +) error { + if len(tarDefs) == 0 || len(baseDefs) == 0 { + return moerr.NewInternalErrorNoCtx("data branch: missing schema lineage") + } + tarEndpoint := tarDefs[len(tarDefs)-1] + baseEndpoint := baseDefs[len(baseDefs)-1] + for _, tarCol := range tarEndpoint.Cols { + if !isDataBranchUserVisibleColumn(tarCol) { + continue + } + baseCol := dataBranchColumnDefByLogicalName(baseEndpoint, tarCol) + if baseCol == nil || !isDataBranchUserVisibleColumn(baseCol) { + continue + } + tarReachesLCA, tarLCACol, tarRedefined := dataBranchColumnReachesLCA( + tarDefs, tarLineageOnly, tarCol, + ) + baseReachesLCA, baseLCACol, baseRedefined := dataBranchColumnReachesLCA( + baseDefs, baseLineageOnly, baseCol, + ) + identityMismatch := tarRedefined || baseRedefined || tarReachesLCA != baseReachesLCA + if tarReachesLCA && baseReachesLCA { + identityMismatch = dataBranchColumnDefByLogicalName(baseDefs[0], tarLCACol) != baseLCACol + } + if !tarReachesLCA && !baseReachesLCA { + // Independently added same-name columns are compatible when the + // LCA did not contain that logical column. If it did, both sides + // dropped and recreated the name, so old values are not lineage- + // compatible with either endpoint column. + identityMismatch = identityMismatch || + dataBranchLCASchemaContainsColumn(tarDefs[0], tarCol) || + dataBranchLCASchemaContainsColumn(baseDefs[0], baseCol) + } + if identityMismatch { + return moerr.NewInternalErrorNoCtxf( + "schema compatibility check: column '%s' has different identity", + tarCol.Name, + ) + } + } + return nil +} + func decideCollectRange( ctx context.Context, ses *Session, @@ -1941,6 +2056,24 @@ func decideCollectRange( ) return } + tarDefs, err := dataBranchPathTableDefs( + ctx, ses, bh, tables.tarRel, tarSp, tarPath, tarPathTS, + ) + if err != nil { + return tarCollectRange, baseCollectRange, err + } + baseDefs, err := dataBranchPathTableDefs( + ctx, ses, bh, tables.baseRel, baseSp, basePath, basePathTS, + ) + if err != nil { + return tarCollectRange, baseCollectRange, err + } + if err = validateDataBranchColumnLineage( + tarDefs, dagInfo.pathFromLCAToTarLineageOnly, + baseDefs, dagInfo.pathFromLCAToBaseLineageOnly, + ); err != nil { + return tarCollectRange, baseCollectRange, err + } if tarCollectRange, err = buildSideCollectRange( ctx, ses, bh, tables, tarSp, tarCTS, diff --git a/pkg/frontend/data_branch_hashdiff.go b/pkg/frontend/data_branch_hashdiff.go index 15ffc9297e04e..449b4c3bf9156 100644 --- a/pkg/frontend/data_branch_hashdiff.go +++ b/pkg/frontend/data_branch_hashdiff.go @@ -123,7 +123,7 @@ func lcaProbeColumnLayout( "DATA BRANCH target column %q is unavailable", name, ) } - lcaCol := dataBranchColumnDefByIdentity(lcaDef, targetCol) + lcaCol := dataBranchColumnDefByLogicalName(lcaDef, targetCol) candidate := dataBranchColumnDefByName(lcaDef, targetCol.Name) if candidate != nil && isDataBranchDerivedCompositePKColumn(lcaDef, targetDef, candidate, targetCol) { @@ -2709,10 +2709,6 @@ func dataBranchSourceColToTargetIdx( } } } - targetCols := make(map[string]*plan2.ColDef, len(targetDef.Cols)) - for _, col := range targetDef.Cols { - targetCols[strings.ToLower(col.Name)] = col - } targetOnly := make(map[int]struct{}, len(targetOnlyIdxes)) for _, idx := range targetOnlyIdxes { targetOnly[idx] = struct{}{} @@ -2722,18 +2718,20 @@ func dataBranchSourceColToTargetIdx( if col.Name == catalog.Row_ID { continue } - targetIdx := dataBranchColumnIndexByName(targetColNames, col.Name) + targetCol := dataBranchColumnDefByLogicalName(targetDef, col) + targetIdx := -1 + if targetCol != nil { + targetIdx = dataBranchColumnIndexByName(targetColNames, targetCol.Name) + } mapping = append(mapping, targetIdx) if targetIdx < 0 { continue } - targetCol, ok := targetCols[strings.ToLower(col.Name)] - derivedCompositePK := ok && + derivedCompositePK := isDataBranchDerivedCompositePKColumn(sourceDef, targetDef, col, targetCol) - sameIdentity := ok && (col.Seqnum == targetCol.Seqnum || derivedCompositePK) - sameType := ok && col.Typ.Id == targetCol.Typ.Id && + sameType := col.Typ.Id == targetCol.Typ.Id && (dataBranchColumnTypeAttributesEqual(col.Typ, targetCol.Typ) || derivedCompositePK) - if !sameIdentity || !sameType { + if !sameType { if _, isTargetOnly := targetOnly[targetIdx]; isTargetOnly { // This column does not exist on the base endpoint and is excluded // from comparison and MERGE apply. Its old physical representation @@ -2742,12 +2740,6 @@ func dataBranchSourceColToTargetIdx( mapping[len(mapping)-1] = -1 continue } - if ok && !sameIdentity { - return nil, moerr.NewNotSupportedNoCtxf( - "historical data branch column %s has a different identity from the endpoint schema", - col.Name, - ) - } return nil, moerr.NewNotSupportedNoCtxf( "historical data branch column %s has a different type from the endpoint schema", col.Name, @@ -2783,7 +2775,6 @@ func isDataBranchDerivedCompositePKColumn( sourcePart := dataBranchColumnDefByName(sourceDef, sourceName) targetPart := dataBranchColumnDefByName(targetDef, targetName) if sourcePart == nil || targetPart == nil || - sourcePart.Seqnum != targetPart.Seqnum || sourcePart.Typ.Id != targetPart.Typ.Id || !dataBranchColumnTypeAttributesEqual(sourcePart.Typ, targetPart.Typ) || sourcePart.NotNull != targetPart.NotNull { @@ -2905,6 +2896,22 @@ func overlayDataBranchProbeResult( return err } +func dataBranchHistoricalProbeStuff( + ctx context.Context, + tblStuff tableStuff, + endpointRel engine.Relation, +) tableStuff { + probeStuff := tblStuff + probeStuff.lcaRel = endpointRel + if endpointRel.GetTableID(ctx) == tblStuff.tarRel.GetTableID(ctx) { + // Target-side hydration restores the current values of columns that + // were absent or incompatible in an older physical generation. + probeStuff.tarRel = endpointRel + probeStuff.def.tarOnlyIdxes = nil + } + return probeStuff +} + func hydrateHistoricalDataBranchBatch( ctx context.Context, ses *Session, @@ -2932,8 +2939,7 @@ func hydrateHistoricalDataBranchBatch( } tBat.SetRowCount(projected.RowCount()) - probeStuff := tblStuff - probeStuff.lcaRel = endpointRel + probeStuff := dataBranchHistoricalProbeStuff(ctx, tblStuff, endpointRel) probe, err := runLCAProbeWithReaderFallback( ctx, ses, tBat, probeStuff, endpointSnapshot, ) diff --git a/pkg/frontend/data_branch_hashdiff_test.go b/pkg/frontend/data_branch_hashdiff_test.go index d072aae8579ed..75543f007f6ce 100644 --- a/pkg/frontend/data_branch_hashdiff_test.go +++ b/pkg/frontend/data_branch_hashdiff_test.go @@ -1073,7 +1073,7 @@ func TestLCAProbeColumnLayoutExcludesTargetOnlyColumns(t *testing.T) { }} targetDef := &plan.TableDef{Cols: []*plan.ColDef{ {Name: "id", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, - {Name: "name", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_varchar)}}, + {Name: "name", OriginName: "old_name", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_varchar)}}, {Name: "added", ColId: 3, Seqnum: 2, Typ: plan.Type{Id: int32(types.T_int64)}}, }} @@ -1173,6 +1173,29 @@ func TestLCAProbeColumnLayoutIgnoresTargetOnlyRowIDCollision(t *testing.T) { require.Equal(t, []int{0}, layout.targetIdxes) } +func TestLCAProbeColumnLayoutIgnoresUnrelatedIdentityCollision(t *testing.T) { + lcaDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_int64)}}, + }} + targetDef := &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "a", ColId: 1, Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "c", ColId: 2, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_int64)}}, + }} + + layout, err := lcaProbeColumnLayout( + lcaDef, + targetDef, + []string{"a", "c"}, + []types.Type{types.T_int64.ToType(), types.T_int64.ToType()}, + nil, + ) + + require.NoError(t, err) + require.Equal(t, []string{"a"}, layout.attrs) + require.Equal(t, []int{0}, layout.targetIdxes) +} + func TestLCAProbeResultTargetIndexes(t *testing.T) { layout := lcaProbeLayout{targetIdxes: []int{1, 2}} @@ -1188,6 +1211,28 @@ func TestLCAProbeResultTargetIndexes(t *testing.T) { require.ErrorContains(t, err, "unexpected LCA probe result width") } +func TestDataBranchHistoricalProbeStuff(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + ctx := context.Background() + targetRel := mock_frontend.NewMockRelation(ctrl) + baseRel := mock_frontend.NewMockRelation(ctrl) + targetRel.EXPECT().GetTableID(ctx).Return(uint64(10)).AnyTimes() + baseRel.EXPECT().GetTableID(ctx).Return(uint64(20)).AnyTimes() + tblStuff := tableStuff{tarRel: targetRel, baseRel: baseRel} + tblStuff.def.tarOnlyIdxes = []int{2} + + targetProbe := dataBranchHistoricalProbeStuff(ctx, tblStuff, targetRel) + require.Same(t, targetRel, targetProbe.lcaRel) + require.Same(t, targetRel, targetProbe.tarRel) + require.Nil(t, targetProbe.def.tarOnlyIdxes) + + baseProbe := dataBranchHistoricalProbeStuff(ctx, tblStuff, baseRel) + require.Same(t, baseRel, baseProbe.lcaRel) + require.Same(t, targetRel, baseProbe.tarRel) + require.Equal(t, []int{2}, baseProbe.def.tarOnlyIdxes) +} + func newTestBranchTableDef(tableName, dataColumnName string) *plan.TableDef { return &plan.TableDef{ DbName: "db1", @@ -1367,6 +1412,7 @@ func TestHandleDelsOnLCA_SQLPaths(t *testing.T) { tblStuff := newTestBranchTableStuff(ctrl) targetTblDef := tblStuff.tarRel.GetTableDef(context.Background()) + targetTblDef.Cols[1].OriginName = "b" targetTblDef.Cols[1].Name = "bb" tblStuff.def.colNames[1] = "bb" tblStuff.lcaRel = mock_frontend.NewMockRelation(ctrl) @@ -2658,7 +2704,7 @@ func TestDataBranchSourceColToTargetIdxSkipsTargetOnlyHistoricalTypeDrift(t *tes require.Equal(t, baseline, mp.CurrNB()) } -func TestDataBranchSourceColToTargetIdxRejectsHistoricalIdentityDrift(t *testing.T) { +func TestDataBranchSourceColToTargetIdxAllowsCopyAlterIdentityReassignment(t *testing.T) { sourceDef := &plan.TableDef{ Cols: []*plan.ColDef{ {Name: "a", Seqnum: 0, Typ: plan.Type{Id: int32(types.T_int64)}}, @@ -2674,13 +2720,9 @@ func TestDataBranchSourceColToTargetIdxRejectsHistoricalIdentityDrift(t *testing Pkey: &plan.PrimaryKeyDef{PkeyColName: "a", Names: []string{"a"}}, } - _, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a", "b"}, nil) - require.ErrorContains(t, err, "column b has a different identity") - - mapping, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a", "b"}, []int{1}) + mapping, err := dataBranchSourceColToTargetIdx(sourceDef, targetDef, []string{"a", "b"}, nil) require.NoError(t, err) - require.Equal(t, []int{0, -1}, mapping, - "target-only historical identity drift must be projected as NULL") + require.Equal(t, []int{0, 1}, mapping) } func TestDataBranchSourceColToTargetIdxPreservesRebuiltCompositePrimaryKey(t *testing.T) { @@ -2749,13 +2791,14 @@ func TestDataBranchSourceColToTargetIdxPreservesRebuiltCompositePrimaryKey(t *te targetDef.Pkey.Names = []string{"select", "line item"} targetDef.Cols[1].Seqnum = 4 - require.False(t, isDataBranchDerivedCompositePKColumn(sourceDef, targetDef, sourceCPK, targetCPK)) - _, err = dataBranchSourceColToTargetIdx( + require.True(t, isDataBranchDerivedCompositePKColumn(sourceDef, targetDef, sourceCPK, targetCPK)) + mapping, err = dataBranchSourceColToTargetIdx( sourceDef, targetDef, []string{"select", "line item", "payload", catalog.CPrimaryKeyColName}, []int{2}, ) - require.ErrorContains(t, err, "column line item has a different identity") + require.NoError(t, err) + require.Equal(t, []int{0, 1, 3}, mapping) targetDef.Cols[1].Seqnum = 1 targetDef.Cols[1].Typ.Id = int32(types.T_varchar) diff --git a/pkg/frontend/data_branch_output.go b/pkg/frontend/data_branch_output.go index 8dbbd8e97f705..6f6af1e025af3 100644 --- a/pkg/frontend/data_branch_output.go +++ b/pkg/frontend/data_branch_output.go @@ -465,6 +465,27 @@ func dataBranchColumnDefByIdentity(tableDef *plan2.TableDef, sourceColDef *plan2 return nil } +func dataBranchColumnDefByLogicalName(tableDef *plan2.TableDef, sourceColDef *plan2.ColDef) *plan2.ColDef { + if tableDef == nil || sourceColDef == nil { + return nil + } + if colDef := dataBranchColumnDefByName(tableDef, sourceColDef.Name); colDef != nil { + return colDef + } + if originName := sourceColDef.GetOriginCaseName(); !strings.EqualFold(originName, sourceColDef.Name) { + if colDef := dataBranchColumnDefByName(tableDef, originName); colDef != nil { + return colDef + } + } + for _, colDef := range tableDef.Cols { + if colDef != nil && !strings.EqualFold(colDef.GetOriginCaseName(), colDef.Name) && + strings.EqualFold(colDef.GetOriginCaseName(), sourceColDef.Name) { + return colDef + } + } + return nil +} + // dataBranchColumnsByIdentity maps each source column name to the column with // the same stable identity in the destination definition. Column names are // intentionally not part of schema equivalence: a branch may rename a column diff --git a/pkg/frontend/data_branch_test.go b/pkg/frontend/data_branch_test.go index 72593da71274b..8df83d026cbb1 100644 --- a/pkg/frontend/data_branch_test.go +++ b/pkg/frontend/data_branch_test.go @@ -1208,7 +1208,7 @@ func TestCheckSchemaCompatibility_TypeMismatch(t *testing.T) { require.Contains(t, err.Error(), "has different types") } -func TestCheckSchemaCompatibility_RejectsSameNameWithDifferentIdentity(t *testing.T) { +func TestCheckSchemaCompatibility_AllowsCopyAlterIdentityReassignment(t *testing.T) { baseDef := &plan.TableDef{ Pkey: &plan.PrimaryKeyDef{Names: []string{"a"}, PkeyColName: "a"}, Cols: []*plan.ColDef{ @@ -1225,7 +1225,7 @@ func TestCheckSchemaCompatibility_RejectsSameNameWithDifferentIdentity(t *testin } _, _, _, err := checkSchemaCompatibility(targetDef, baseDef) - require.ErrorContains(t, err, "column 'b' has different identity") + require.NoError(t, err) // Physical reordering preserves Seqnum and remains compatible. targetDef.Cols[1].Seqnum = 1 @@ -1234,6 +1234,96 @@ func TestCheckSchemaCompatibility_RejectsSameNameWithDifferentIdentity(t *testin require.NoError(t, err) } +func TestValidateDataBranchColumnLineage(t *testing.T) { + tableDef := func(cols ...*plan.ColDef) *plan.TableDef { + return &plan.TableDef{Cols: cols} + } + col := func(name string, id uint64, seq uint32) *plan.ColDef { + return &plan.ColDef{ + Name: name, ColId: id, Seqnum: seq, + Typ: plan.Type{Id: int32(types.T_int64)}, + } + } + + t.Run("copy alter preserves same-name columns", func(t *testing.T) { + tarDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0), col("b", 2, 1)), + tableDef(col("a", 10, 1), col("c", 11, 0), col("b", 12, 2)), + } + baseDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0), col("b", 2, 1)), + tableDef(col("a", 20, 0), col("b", 21, 1)), + } + require.NoError(t, validateDataBranchColumnLineage( + tarDefs, []bool{false, true}, baseDefs, []bool{false, false}, + )) + }) + + t.Run("drop and add same name is discontinuous", func(t *testing.T) { + tarDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0), col("b", 2, 1)), + tableDef(col("a", 10, 0)), + tableDef(col("a", 20, 0), col("b", 21, 1)), + } + baseDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0), col("b", 2, 1)), + tableDef(col("a", 30, 0), col("b", 31, 1)), + } + err := validateDataBranchColumnLineage( + tarDefs, []bool{false, true, true}, baseDefs, []bool{false, false}, + ) + require.ErrorContains(t, err, "column 'b' has different identity") + }) + + t.Run("independent additions are compatible", func(t *testing.T) { + tarDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0)), + tableDef(col("a", 10, 0), col("c", 11, 1)), + } + baseDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0)), + tableDef(col("a", 20, 0), col("c", 21, 1)), + } + require.NoError(t, validateDataBranchColumnLineage( + tarDefs, []bool{false, true}, baseDefs, []bool{false, true}, + )) + }) + + t.Run("added column cannot be dropped and recreated", func(t *testing.T) { + tarDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0)), + tableDef(col("a", 10, 0), col("c", 11, 1)), + tableDef(col("a", 20, 0)), + tableDef(col("a", 30, 0), col("c", 31, 1)), + } + baseDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0)), + tableDef(col("a", 40, 0), col("c", 41, 1)), + } + err := validateDataBranchColumnLineage( + tarDefs, []bool{false, true, true, true}, baseDefs, []bool{false, true}, + ) + require.ErrorContains(t, err, "column 'c' has different identity") + }) + + t.Run("rename across a clone edge preserves identity", func(t *testing.T) { + tarDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0), col("b", 2, 1)), + tableDef(col("a", 1, 0), &plan.ColDef{ + Name: "bb", OriginName: "b", ColId: 2, Seqnum: 1, + Typ: plan.Type{Id: int32(types.T_int64)}, + }), + } + baseDefs := []*plan.TableDef{ + tableDef(col("a", 1, 0), col("b", 2, 1)), + tableDef(col("a", 1, 0), col("b", 2, 1)), + } + require.NoError(t, validateDataBranchColumnLineage( + tarDefs, []bool{false, false}, baseDefs, []bool{false, false}, + )) + }) +} + func TestCheckSchemaCompatibility_RejectsDifferentTypeAttributes(t *testing.T) { tarDef := &plan.TableDef{ Pkey: &plan.PrimaryKeyDef{Names: []string{"a"}, PkeyColName: "a"}, diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result index 02f573b37966c..5ec05d0c8f2bf 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.result @@ -186,7 +186,7 @@ create snapshot history_database_s for database br_schema_drift; alter table history_database add column c int default 0; set @history_database_tid = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_database'); data branch delete table history_database; -select count(*) as database_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id = @history_database_tid and level = 'alter'; +select count(*) as database_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id = @history_database_tid and level like 'alter%'; database_lineage_before_owner_drop 1 select count(*) as database_snapshots_before_owner_drop from mo_catalog.mo_snapshots where sname = concat('__mo_branch_', cast(@history_database_tid as char)) and kind = 'branch'; diff --git a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql index 5266727f3d8f0..4e3c07d6b728e 100644 --- a/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql +++ b/test/distributed/cases/git4data/branch/edge/branch_edge_cases.sql @@ -172,7 +172,7 @@ create snapshot history_database_s for database br_schema_drift; alter table history_database add column c int default 0; set @history_database_tid = (select rel_id from mo_catalog.mo_tables where reldatabase = 'br_schema_drift' and relname = 'history_database'); data branch delete table history_database; -select count(*) as database_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id = @history_database_tid and level = 'alter'; +select count(*) as database_lineage_before_owner_drop from mo_catalog.mo_branch_metadata where table_id = @history_database_tid and level like 'alter%'; select count(*) as database_snapshots_before_owner_drop from mo_catalog.mo_snapshots where sname = concat('__mo_branch_', cast(@history_database_tid as char)) and kind = 'branch'; drop snapshot history_database_s; select count(*) as database_lineage_after_owner_drop from mo_catalog.mo_branch_metadata where table_id = @history_database_tid and level = 'alter'; diff --git a/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.result b/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.result index 2c89cee4ef0e3..c2461f97fd66d 100644 --- a/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.result +++ b/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.result @@ -180,7 +180,7 @@ create snapshot `SELECT` for database sp_test02; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME select 2025-07-19 05:52:26.022949 database sys sp_test02 -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user'; sname level account_name database_name table_name select database sys sp_test02 drop snapshot `SELECT`; @@ -194,7 +194,7 @@ create snapshot AUTO_INCREMENT for database sp_test03; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME auto_increment 2025-07-19 05:52:26.069177 database sys sp_test03 -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user'; sname level account_name database_name table_name auto_increment database sys sp_test03 drop snapshot AUTO_INCREMENT; @@ -251,10 +251,10 @@ show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME spsp03 2025-07-19 05:52:26.26781 table sys db02 table01 spsp02 2025-07-19 05:52:26.260618 database sys db02 -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where level = 'database'; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user' and level = 'database'; sname level account_name database_name table_name spsp02 database sys db02 -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where level = 'table'; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user' and level = 'table'; sname level account_name database_name table_name spsp03 table sys db02 table01 drop snapshot spsp02; diff --git a/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.sql b/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.sql index b48129c13cf1f..66c2836c6b620 100644 --- a/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.sql +++ b/test/distributed/cases/snapshot/sys_create_snapshot_for_sys_db_table_level.sql @@ -119,7 +119,7 @@ drop snapshot if exists `SELECT`; create snapshot `SELECT` for database sp_test02; -- @ignore:1 show snapshots; -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user'; drop snapshot `SELECT`; drop database sp_test02; @@ -135,7 +135,7 @@ drop snapshot if exists AUTO_INCREMENT; create snapshot AUTO_INCREMENT for database sp_test03; -- @ignore:1 show snapshots; -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user'; drop snapshot AUTO_INCREMENT; drop database sp_test03; @@ -191,8 +191,8 @@ create snapshot spsp02 for database db02; create snapshot spsp03 for table db02 table01; -- @ignore:1 show snapshots; -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where level = 'database'; -select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where level = 'table'; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user' and level = 'database'; +select sname, level, account_name, database_name, table_name from mo_catalog.mo_snapshots where kind = 'user' and level = 'table'; drop snapshot spsp02; drop snapshot spsp03; drop database db02;