planner, executor: include relevant opt vars in plan cache key

pkg/bindinfo/binding_plan_generation.go

@@ -435,6 +435,8 @@ func genPlanUnderState(sctx sessionctx.Context, stmt ast.StmtNode, state *state)
 			sctx.GetSessionVars().SetAllowPreferRangeScan(state.varValues[i].(bool))
 		case vardef.TiDBOptEnableNoDecorrelateInSelect:
 			sctx.GetSessionVars().EnableNoDecorrelateInSelect = state.varValues[i].(bool)
+		case vardef.TiDBOptAlwaysKeepJoinKey:
+			sctx.GetSessionVars().AlwaysKeepJoinKey = state.varValues[i].(bool)
 		case vardef.TiDBOptEnableSemiJoinRewrite:
 			sctx.GetSessionVars().EnableSemiJoinRewrite = state.varValues[i].(bool)
 		case vardef.TiDBOptSelectivityFactor:

pkg/bindinfo/testdata/binding_auto_suite_out.json

@@ -209,15 +209,15 @@
         "Fixes": "[52869]"
       },
       {
-        "Vars": "[tidb_opt_hash_join_cost_factor tidb_opt_index_join_cost_factor tidb_opt_index_reader_cost_factor tidb_opt_index_scan_cost_factor tidb_opt_merge_join_cost_factor tidb_opt_prefer_range_scan]",
+        "Vars": "[tidb_opt_always_keep_join_key tidb_opt_hash_join_cost_factor tidb_opt_index_join_cost_factor tidb_opt_index_reader_cost_factor tidb_opt_index_scan_cost_factor tidb_opt_merge_join_cost_factor tidb_opt_prefer_range_scan]",
         "Fixes": "[44855 45132 52869]"
       },
       {
-        "Vars": "[tidb_opt_hash_join_cost_factor tidb_opt_index_join_cost_factor tidb_opt_index_reader_cost_factor tidb_opt_index_scan_cost_factor tidb_opt_merge_join_cost_factor tidb_opt_prefer_range_scan]",
+        "Vars": "[tidb_opt_always_keep_join_key tidb_opt_hash_join_cost_factor tidb_opt_index_join_cost_factor tidb_opt_index_reader_cost_factor tidb_opt_index_scan_cost_factor tidb_opt_merge_join_cost_factor tidb_opt_prefer_range_scan]",
         "Fixes": "[44855 45132 52869]"
       },
       {
-        "Vars": "[tidb_opt_hash_join_cost_factor tidb_opt_prefer_range_scan tidb_opt_table_full_scan_cost_factor tidb_opt_table_reader_cost_factor]",
+        "Vars": "[tidb_opt_always_keep_join_key tidb_opt_hash_join_cost_factor tidb_opt_prefer_range_scan tidb_opt_table_full_scan_cost_factor tidb_opt_table_reader_cost_factor]",
         "Fixes": "[52869]"
       }
     ]

pkg/executor/test/plancache/plan_cache_test.go

@@ -896,9 +896,9 @@ func testPreparePlanCache4Function(t *testing.T, tk *testkit.TestKit) {
 
 func testPreparePlanCache4DifferentSystemVars(t *testing.T, tk *testkit.TestKit) {
 	t.Helper()
-	tk.MustExec("set @old_sql_select_limit := @@sql_select_limit, @old_tidb_enable_index_merge := @@tidb_enable_index_merge, @old_tidb_enable_collect_execution_info := @@tidb_enable_collect_execution_info, @old_tidb_enable_parallel_apply := @@tidb_enable_parallel_apply")
+	tk.MustExec("set @old_sql_select_limit := @@sql_select_limit, @old_tidb_enable_index_merge := @@tidb_enable_index_merge, @old_tidb_enable_collect_execution_info := @@tidb_enable_collect_execution_info, @old_tidb_enable_parallel_apply := @@tidb_enable_parallel_apply, @old_tidb_enable_plan_cache_for_subquery := @@tidb_enable_plan_cache_for_subquery, @old_tidb_opt_enable_semi_join_rewrite := @@tidb_opt_enable_semi_join_rewrite, @old_tidb_opt_always_keep_join_key := @@tidb_opt_always_keep_join_key")
 	defer func() {
-		tk.MustExec("set @@sql_select_limit = @old_sql_select_limit, @@tidb_enable_index_merge = @old_tidb_enable_index_merge, @@tidb_enable_collect_execution_info = @old_tidb_enable_collect_execution_info, @@tidb_enable_parallel_apply = @old_tidb_enable_parallel_apply")
+		tk.MustExec("set @@sql_select_limit = @old_sql_select_limit, @@tidb_enable_index_merge = @old_tidb_enable_index_merge, @@tidb_enable_collect_execution_info = @old_tidb_enable_collect_execution_info, @@tidb_enable_parallel_apply = @old_tidb_enable_parallel_apply, @@tidb_enable_plan_cache_for_subquery = @old_tidb_enable_plan_cache_for_subquery, @@tidb_opt_enable_semi_join_rewrite = @old_tidb_opt_enable_semi_join_rewrite, @@tidb_opt_always_keep_join_key = @old_tidb_opt_always_keep_join_key")
 	}()
 
 	tk.MustExec(`set tidb_enable_prepared_plan_cache=1`)
@@ -951,6 +951,42 @@ func testPreparePlanCache4DifferentSystemVars(t *testing.T, tk *testkit.TestKit)
 	require.NotContains(t, fmt.Sprint(applyRow), "Concurrency")
 	tk.MustExec("execute stmt;")
 	tk.MustQuery("select @@last_plan_from_cache;").Check(testkit.Rows("0"))
+
+	tk.MustExec("set @@tidb_enable_collect_execution_info = 1")
+	tk.MustExec("set @@tidb_enable_plan_cache_for_subquery = 1")
+	tk.MustExec("set @@tidb_opt_enable_semi_join_rewrite = 1")
+	tk.MustExec("drop table if exists test_exists_a, test_exists_b")
+	tk.MustExec("create table test_exists_a (a int primary key)")
+	tk.MustExec("create table test_exists_b (a int primary key)")
+	tk.MustExec("insert into test_exists_a values (1), (2)")
+	tk.MustExec("insert into test_exists_b values (1)")
+	query := "select * from test_exists_a where exists (select 1 from test_exists_b where test_exists_b.a = test_exists_a.a)"
+	rewrittenPlan := tk.MustQuery("explain format='brief' " + query)
+	require.Contains(t, fmt.Sprint(rewrittenPlan.Rows()), "inner join")
+	tk.MustExec("prepare stmt from '" + query + "'")
+	tk.MustQuery("execute stmt").Check(testkit.Rows("1"))
+	tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0"))
+
+	tk.MustExec("set @@tidb_opt_enable_semi_join_rewrite = 0")
+	tk.MustQuery("execute stmt").Check(testkit.Rows("1"))
+	tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0"))
+	tk.MustQuery("execute stmt").Check(testkit.Rows("1"))
+	tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1"))
+	semiJoinPlan := tk.MustQuery("explain format='brief' " + query)
+	require.Contains(t, fmt.Sprint(semiJoinPlan.Rows()), "semi join")
+
+	tk.MustExec("set @@tidb_opt_always_keep_join_key = 1")
+	tk.MustExec("drop table if exists join_key_a, join_key_b")
+	tk.MustExec("create table join_key_a (a int)")
+	tk.MustExec("create table join_key_b (a int)")
+	tk.MustExec("prepare stmt from 'select 1 from join_key_a left join join_key_b on join_key_a.a = join_key_b.a where join_key_a.a = 1'")
+	tk.MustQuery("execute stmt").Check(testkit.Rows())
+	tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0"))
+	tk.MustExec("set @@tidb_opt_always_keep_join_key = 0")
+	tk.MustQuery("execute stmt").Check(testkit.Rows())
+	tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0"))
+	tk.MustQuery("execute stmt").Check(testkit.Rows())
+	tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1"))
 }
 
 func testPreparePC4Binding(t *testing.T, tk *testkit.TestKit) {

pkg/planner/cascades/old/transformation_rules.go

@@ -903,10 +903,10 @@ func (*pushDownJoin) predicatePushDown(
 		nullSensitive := join.JoinType == base.AntiLeftOuterSemiJoin || join.JoinType == base.LeftOuterSemiJoin
 		if join.JoinType == base.RightOuterJoin {
 			joinConds, remainCond = expression.PropConstForOuterJoin(join.SCtx().GetExprCtx(), joinConds, remainCond, rightSchema, leftSchema,
-				join.SCtx().GetSessionVars().AlwaysKeepJoinKey, nullSensitive, nil)
+				join.SCtx().GetSessionVars().GetAlwaysKeepJoinKey(), nullSensitive, nil)
 		} else {
 			joinConds, remainCond = expression.PropConstForOuterJoin(join.SCtx().GetExprCtx(), joinConds, remainCond, leftSchema, rightSchema,
-				join.SCtx().GetSessionVars().AlwaysKeepJoinKey, nullSensitive, nil)
+				join.SCtx().GetSessionVars().GetAlwaysKeepJoinKey(), nullSensitive, nil)
 		}
 		eq, left, right, other := join.ExtractOnCondition(joinConds, leftSchema, rightSchema, false, false)
 		join.AppendJoinConds(eq, left, right, other)

pkg/planner/core/logical_plan_builder.go

@@ -741,6 +741,8 @@ func (b *PlanBuilder) buildJoin(ctx context.Context, joinNode *ast.Join) (base.L
 		return b.buildResultSetNode(ctx, joinNode.Left, false)
 	}
 
+	b.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptAlwaysKeepJoinKey)
+
 	// Detect whether the right subtree contains any LATERAL table source.
 	// This is used only to decide whether to push outerSchemas before building
 	// the right side, so that nested LATERAL sources can resolve outer columns.

pkg/planner/core/operator/logicalop/logical_join.go

@@ -1971,7 +1971,7 @@ func (p *LogicalJoin) outerJoinPropConst(predicates []expression.Expression, vai
 	outerTableSchema := outerTable.Schema()
 	innerTableSchema := innerTable.Schema()
 	joinConds, predicates = expression.PropConstForOuterJoin(exprCtx, joinConds, predicates, outerTableSchema,
-		innerTableSchema, p.SCtx().GetSessionVars().AlwaysKeepJoinKey, nullSensitive, vaildExprFunc)
+		innerTableSchema, p.SCtx().GetSessionVars().GetAlwaysKeepJoinKey(), nullSensitive, vaildExprFunc)
 	p.AttachOnConds(joinConds)
 	return predicates
 }

pkg/planner/core/plan_cache_utils.go

@@ -177,12 +177,16 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context,
 		}
 	}
 
+	vars.ResetRelevantOptVarsAndFixes(true)
+	defer vars.ResetRelevantOptVarsAndFixes(false)
+
 	var p base.Plan
 	destBuilder, _ := NewPlanBuilder().Init(sctx.GetPlanCtx(), ret.InfoSchema, hint.NewQBHintHandler(nil))
 	p, err = destBuilder.Build(ctx, nodeW)
 	if err != nil {
 		return nil, nil, 0, err
 	}
+	relevantOptVarNames, relevantOptFixIDs := collectRelevantOptVarsAndFixes(vars)
 
 	if cacheable && destBuilder.optFlag&rule.FlagPartitionProcessor > 0 {
 		// dynamic prune mode is not used, could be that global statistics not yet available!
@@ -229,6 +233,8 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context,
 		SchemaVersion:       ret.InfoSchema.SchemaMetaVersion(),
 		RelateVersion:       relateVersion,
 		Params:              extractor.markers,
+		RelevantOptVarNames: relevantOptVarNames,
+		RelevantOptFixIDs:   relevantOptFixIDs,
 	}
 
 	stmtProcessor := &planCacheStmtProcessor{ctx: ctx, is: is, stmt: preparedObj}
@@ -404,6 +410,11 @@ func newPlanCacheKeyWithMatchedBinding(
 	hashLen += 8 * len(vars.StmtCtx.TblInfo2UnionScan)
 	// txn status
 	hashLen += 6
+	if len(stmt.RelevantOptVarNames) > 0 || len(stmt.RelevantOptFixIDs) > 0 {
+		hashLen++
+		hashLen += len(stmt.RelevantOptVarNames) * 128
+		hashLen += relevantOptFixesHashLen(vars, stmt.RelevantOptFixIDs)
+	}
 
 	hash := make([]byte, 0, hashLen)
 	// hashInitCap is not used, just for test purposes
@@ -510,6 +521,7 @@ func newPlanCacheKeyWithMatchedBinding(
 	hash = append(hash, bool2Byte(config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Load()))
 	hash = append(hash, bool2Byte(vars.StmtCtx.ForShareLockEnabledByNoop))
 	hash = append(hash, bool2Byte(vars.SharedLockPromotion))
+	hash = appendRelevantOptVarsAndFixes(hash, vars, stmt)
 
 	if intest.InTest {
 		if cap(hash) != hashInitCap {
@@ -527,6 +539,114 @@ func bool2Byte(flag bool) byte {
 	return '0'
 }
 
+func collectRelevantOptVarsAndFixes(vars *variable.SessionVars) (varNames []string, fixIDs []uint64) {
+	for varName := range vars.RelevantOptVars {
+		varNames = append(varNames, varName)
+	}
+	slices.Sort(varNames)
+
+	for fixID := range vars.RelevantOptFixes {
+		fixIDs = append(fixIDs, fixID)
+	}
+	slices.Sort(fixIDs)
+	return
+}
+
+func relevantOptFixesHashLen(vars *variable.SessionVars, fixIDs []uint64) int {
+	hashLen := 0
+	for _, fixID := range fixIDs {
+		// codec.EncodeUint(fixID) + '=' + fixValue + ';'
+		hashLen += 8 + 2
+		if fixVal, ok := vars.OptimizerFixControl[fixID]; ok {
+			hashLen += len(fixVal)
+		}
+	}
+	return hashLen
+}
+
+func appendRelevantOptVarsAndFixes(hash []byte, vars *variable.SessionVars, stmt *PlanCacheStmt) []byte {
+	if stmt == nil || (len(stmt.RelevantOptVarNames) == 0 && len(stmt.RelevantOptFixIDs) == 0) {
+		return hash
+	}
+	hash = append(hash, '|')
+	for _, varName := range stmt.RelevantOptVarNames {
+		hash = append(hash, varName...)
+		hash = append(hash, '=')
+		hash = appendRelevantOptVarValue(hash, vars, varName)
+		hash = append(hash, ';')
+	}
+	for _, fixID := range stmt.RelevantOptFixIDs {
+		hash = codec.EncodeUint(hash, fixID)
+		hash = append(hash, '=')
+		if fixVal, ok := vars.OptimizerFixControl[fixID]; ok {
+			hash = append(hash, fixVal...)
+		}
+		hash = append(hash, ';')
+	}
+	return hash
+}
+
+func appendRelevantOptVarValue(hash []byte, vars *variable.SessionVars, varName string) []byte {
+	switch varName {
+	case vardef.TiDBOptIndexScanCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.IndexScanCostFactor))
+	case vardef.TiDBOptIndexReaderCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.IndexReaderCostFactor))
+	case vardef.TiDBOptTableReaderCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.TableReaderCostFactor))
+	case vardef.TiDBOptTableFullScanCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.TableFullScanCostFactor))
+	case vardef.TiDBOptTableRangeScanCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.TableRangeScanCostFactor))
+	case vardef.TiDBOptTableRowIDScanCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.TableRowIDScanCostFactor))
+	case vardef.TiDBOptTableTiFlashScanCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.TableTiFlashScanCostFactor))
+	case vardef.TiDBOptIndexLookupCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.IndexLookupCostFactor))
+	case vardef.TiDBOptIndexMergeCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.IndexMergeCostFactor))
+	case vardef.TiDBOptSortCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.SortCostFactor))
+	case vardef.TiDBOptTopNCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.TopNCostFactor))
+	case vardef.TiDBOptLimitCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.LimitCostFactor))
+	case vardef.TiDBOptStreamAggCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.StreamAggCostFactor))
+	case vardef.TiDBOptHashAggCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.HashAggCostFactor))
+	case vardef.TiDBOptMergeJoinCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.MergeJoinCostFactor))
+	case vardef.TiDBOptHashJoinCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.HashJoinCostFactor))
+	case vardef.TiDBOptIndexJoinCostFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.IndexJoinCostFactor))
+	case vardef.TiDBOptOrderingIdxSelRatio:
+		return codec.EncodeUint(hash, math.Float64bits(vars.OptOrderingIdxSelRatio))
+	case vardef.TiDBOptRiskEqSkewRatio:
+		return codec.EncodeUint(hash, math.Float64bits(vars.RiskEqSkewRatio))
+	case vardef.TiDBOptRiskRangeSkewRatio:
+		return codec.EncodeUint(hash, math.Float64bits(vars.RiskRangeSkewRatio))
+	case vardef.TiDBOptRiskGroupNDVSkewRatio:
+		return codec.EncodeUint(hash, math.Float64bits(vars.RiskGroupNDVSkewRatio))
+	case vardef.TiDBOptCartesianJoinOrderThreshold:
+		return codec.EncodeUint(hash, math.Float64bits(vars.CartesianJoinOrderThreshold))
+	case vardef.TiDBOptSelectivityFactor:
+		return codec.EncodeUint(hash, math.Float64bits(vars.SelectivityFactor))
+	case vardef.TiDBOptPreferRangeScan:
+		return append(hash, bool2Byte(vars.GetAllowPreferRangeScan()))
+	case vardef.TiDBOptEnableNoDecorrelateInSelect:
+		return append(hash, bool2Byte(vars.EnableNoDecorrelateInSelect))
+	case vardef.TiDBOptEnableSemiJoinRewrite:
+		return append(hash, bool2Byte(vars.EnableSemiJoinRewrite))
+	case vardef.TiDBOptAlwaysKeepJoinKey:
+		return append(hash, bool2Byte(vars.AlwaysKeepJoinKey))
+	default:
+		return hash
+	}
+}
+
 // PlanCacheValue stores the cached Statement and StmtNode.
 type PlanCacheValue struct {
 	// Meta Info, all are READ-ONLY once initialized.
@@ -768,6 +888,11 @@ type PlanCacheStmt struct {
 	// BindingInfo caches normalization results for binding matching across executions.
 	BindingInfo bindinfo.BindingMatchInfo
 
+	// RelevantOptVarNames stores optimizer variables observed during prepared plan building.
+	RelevantOptVarNames []string
+	// RelevantOptFixIDs stores optimizer fix-control IDs observed during prepared plan building.
+	RelevantOptFixIDs []uint64
+
 	// the different between NormalizedSQL, NormalizedSQL4PC and StmtText:
 	//  for the query `select * from t where a>1 and b<?`, then
 	//  NormalizedSQL: select * from `t` where `a` > ? and `b` < ? --> constants are normalized to '?',

pkg/planner/core/rule/rule_predicate_simplification.go

@@ -211,7 +211,7 @@ func applyPredicateSimplificationHelper(sctx base.PlanContext, predicates []expr
 	// Thus, we utilize a switch to govern this particular logic.
 	if propagateConstant {
 		if forJoin {
-			simplifiedPredicate = expression.PropagateConstantForJoin(exprCtx, sctx.GetSessionVars().AlwaysKeepJoinKey,
+			simplifiedPredicate = expression.PropagateConstantForJoin(exprCtx, sctx.GetSessionVars().GetAlwaysKeepJoinKey(),
 				schema1, schema2, vaildConstantPropagationExpressionFunc, simplifiedPredicate...)
 		} else {
 			simplifiedPredicate = expression.PropagateConstant(exprCtx, vaildConstantPropagationExpressionFunc, simplifiedPredicate...)

pkg/sessionctx/variable/session.go

@@ -2591,6 +2591,12 @@ func (s *SessionVars) SetAllowPreferRangeScan(val bool) {
 	s.preferRangeScan = val
 }
 
+// GetAlwaysKeepJoinKey gets AlwaysKeepJoinKey and records it for plan cache keys.
+func (s *SessionVars) GetAlwaysKeepJoinKey() bool {
+	s.RecordRelevantOptVar(vardef.TiDBOptAlwaysKeepJoinKey)
+	return s.AlwaysKeepJoinKey
+}
+
 // GetEnableCascadesPlanner get EnableCascadesPlanner from sql hints and SessionVars.EnableCascadesPlanner.
 func (s *SessionVars) GetEnableCascadesPlanner() bool {
 	if s.StmtCtx.HasEnableCascadesPlannerHint {