diff --git a/pkg/cmd/generate-logictest/templates.go b/pkg/cmd/generate-logictest/templates.go index 83bb27f3833d..91f0782d99e6 100644 --- a/pkg/cmd/generate-logictest/templates.go +++ b/pkg/cmd/generate-logictest/templates.go @@ -42,7 +42,11 @@ func runExecBuildLogicTest(t *testing.T, file string) { ForceProductionValues: true,{{end}} // Disable the direct scans in order to keep the output of EXPLAIN (VEC) // deterministic. - DisableDirectColumnarScans: true, + DisableDirectColumnarScans: true,{{ if not .ForceProductionValues }} + // ForceProductionValues is not set for this config, so we must + // explicitly disable optimizer perturbations to keep EXPLAIN output + // deterministic when COCKROACH_LOGIC_TEST_OPTIMIZER_METAMORPHIC is set. + DisableOptimizerPerturbations: true,{{end}} } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(execBuildLogicTestDir, file)) } @@ -65,7 +69,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } @@ -188,7 +193,8 @@ func TestLogic_tmp(t *testing.T) { {{- if .ExecBuildLogicTest }} glob = filepath.Join(execBuildLogicTestDir, "_*") serverArgs := logictest.TestServerArgs{ - DisableWorkmemRandomization: true, + DisableWorkmemRandomization: true, + DisableOptimizerPerturbations: true, } logictest.RunLogicTests(t, serverArgs, configIdx, glob) {{- end }} diff --git a/pkg/sql/logictest/logic.go b/pkg/sql/logictest/logic.go index 76f7996f03c2..0fc0b5e08c6a 100644 --- a/pkg/sql/logictest/logic.go +++ b/pkg/sql/logictest/logic.go @@ -600,6 +600,24 @@ var ( ) sqlfmtLen = flag.Int("line-length", tree.DefaultPrettyCfg().LineWidth, "target line length when using -rewrite-sql") + + // metamorphicDisableOptRuleProbability and metamorphicOptimizerCostPerturbation + // mirror the testing_optimizer_cost_perturbation and + // testing_optimizer_disable_rule_probability session settings used by costfuzz and + // unoptimized-query-oracle. They are only applied when + // COCKROACH_LOGIC_TEST_OPTIMIZER_METAMORPHIC=true so that logictest files with + // fixed query plans are not perturbed during normal CI runs. + metamorphicDisableOptRuleProbability = metamorphic.ConstantWithTestChoice( + "logictest-disable-opt-rule-probability", float64(0), float64(0.5), float64(1.0)) + // metamorphicOptimizerCostPerturbation mirrors the range used by costfuzz + // (see testing_optimizer_cost_perturbation). 0.1 exercises mild plan + // variation while 1.0 stress-tests extreme perturbations. + metamorphicOptimizerCostPerturbation = metamorphic.ConstantWithTestChoice( + "logictest-optimizer-cost-perturbation", float64(0), float64(0.1), float64(1.0)) + + logicTestOptimizerMetamorphicEnabled = envutil.EnvOrDefaultBool( + "COCKROACH_LOGIC_TEST_OPTIMIZER_METAMORPHIC", false) + disableOptRuleProbability = flag.Float64( "disable-opt-rule-probability", 0, "disable transformation rules in the cost-based optimizer with the given probability.") @@ -1638,12 +1656,15 @@ func (t *logicTest) newCluster( return st } setSQLTestingKnobs := func(knobs *base.TestingKnobs) { + disableProb, costPerturb := optimizerTestingKnobValues( + serverArgs, *disableOptRuleProbability, *optimizerCostPerturbation, + ) knobs.SQLEvalContext = &eval.TestingKnobs{ AssertBinaryExprReturnTypes: true, AssertUnaryExprReturnTypes: true, AssertFuncExprReturnTypes: true, - DisableOptimizerRuleProbability: *disableOptRuleProbability, - OptimizerCostPerturbation: *optimizerCostPerturbation, + DisableOptimizerRuleProbability: disableProb, + OptimizerCostPerturbation: costPerturb, ForceProductionValues: serverArgs.ForceProductionValues, UnsafeOverride: func() *bool { v := t.allowUnsafe.Load() @@ -4850,6 +4871,35 @@ type TestServerArgs struct { BatchBytesLimitLowerBound int64 // If set, sql.distsql.direct_columnar_scans.enabled is set to false. DisableDirectColumnarScans bool + // If set, testing optimizer perturbation knobs are forced to 0. This is + // needed for tests with fixed query plans in expected output (e.g. EXPLAIN + // tests and SQLite logic tests). + DisableOptimizerPerturbations bool +} + +func optimizerTestingKnobValues( + serverArgs TestServerArgs, disableFlag, costFlag float64, +) (disableProb, costPerturb float64) { + disableProb = disableFlag + costPerturb = costFlag + if serverArgs.ForceProductionValues || serverArgs.DisableOptimizerPerturbations { + return 0, 0 + } + if logicTestOptimizerMetamorphicEnabled { + // Only apply the metamorphic constant when no explicit non-zero value + // was passed via the command-line flag. This lets callers override the + // metamorphic behaviour (e.g. -disable-opt-rule-probability=0.3) while + // still getting metamorphic defaults in unattended runs. Note: the + // metamorphic constant itself may be 0 (one of its valid choices), in + // which case the assignment is a no-op. + if disableProb == 0 { + disableProb = metamorphicDisableOptRuleProbability + } + if costPerturb == 0 { + costPerturb = metamorphicOptimizerCostPerturbation + } + } + return disableProb, costPerturb } // RunLogicTests runs logic tests for all files matching the given glob. diff --git a/pkg/sql/opt/exec/execbuilder/tests/3node-tenant-multiregion/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/3node-tenant-multiregion/generated_test.go index ad0eb3d29718..cf675ed915bf 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/3node-tenant-multiregion/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/3node-tenant-multiregion/generated_test.go @@ -78,7 +78,8 @@ func TestLogic_tmp(t *testing.T) { var glob string glob = filepath.Join(execBuildLogicTestDir, "_*") serverArgs := logictest.TestServerArgs{ - DisableWorkmemRandomization: true, + DisableWorkmemRandomization: true, + DisableOptimizerPerturbations: true, } logictest.RunLogicTests(t, serverArgs, configIdx, glob) } diff --git a/pkg/sql/opt/exec/execbuilder/tests/3node-tenant/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/3node-tenant/generated_test.go index 20ae1a3e7505..dc00eb72405b 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/3node-tenant/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/3node-tenant/generated_test.go @@ -78,7 +78,8 @@ func TestLogic_tmp(t *testing.T) { var glob string glob = filepath.Join(execBuildLogicTestDir, "_*") serverArgs := logictest.TestServerArgs{ - DisableWorkmemRandomization: true, + DisableWorkmemRandomization: true, + DisableOptimizerPerturbations: true, } logictest.RunLogicTests(t, serverArgs, configIdx, glob) } diff --git a/pkg/sql/opt/exec/execbuilder/tests/5node/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/5node/generated_test.go index de036a067280..77b15be04702 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/5node/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/5node/generated_test.go @@ -78,7 +78,8 @@ func TestLogic_tmp(t *testing.T) { var glob string glob = filepath.Join(execBuildLogicTestDir, "_*") serverArgs := logictest.TestServerArgs{ - DisableWorkmemRandomization: true, + DisableWorkmemRandomization: true, + DisableOptimizerPerturbations: true, } logictest.RunLogicTests(t, serverArgs, configIdx, glob) } diff --git a/pkg/sql/opt/exec/execbuilder/tests/local-read-committed/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/local-read-committed/generated_test.go index c278a47b2d1b..1db4e940ba17 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/local-read-committed/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/local-read-committed/generated_test.go @@ -78,7 +78,8 @@ func TestLogic_tmp(t *testing.T) { var glob string glob = filepath.Join(execBuildLogicTestDir, "_*") serverArgs := logictest.TestServerArgs{ - DisableWorkmemRandomization: true, + DisableWorkmemRandomization: true, + DisableOptimizerPerturbations: true, } logictest.RunLogicTests(t, serverArgs, configIdx, glob) } diff --git a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go index 933860411bac..3639703cc1a2 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go @@ -78,7 +78,8 @@ func TestLogic_tmp(t *testing.T) { var glob string glob = filepath.Join(execBuildLogicTestDir, "_*") serverArgs := logictest.TestServerArgs{ - DisableWorkmemRandomization: true, + DisableWorkmemRandomization: true, + DisableOptimizerPerturbations: true, } logictest.RunLogicTests(t, serverArgs, configIdx, glob) } diff --git a/pkg/sql/opt/exec/execbuilder/tests/multiregion-9node-3region-3azs/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/multiregion-9node-3region-3azs/generated_test.go index 4d9944fb6957..436f6f060fdf 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/multiregion-9node-3region-3azs/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/multiregion-9node-3region-3azs/generated_test.go @@ -78,7 +78,8 @@ func TestLogic_tmp(t *testing.T) { var glob string glob = filepath.Join(execBuildLogicTestDir, "_*") serverArgs := logictest.TestServerArgs{ - DisableWorkmemRandomization: true, + DisableWorkmemRandomization: true, + DisableOptimizerPerturbations: true, } logictest.RunLogicTests(t, serverArgs, configIdx, glob) } diff --git a/pkg/sql/sqlitelogictest/tests/3node-tenant/generated_test.go b/pkg/sql/sqlitelogictest/tests/3node-tenant/generated_test.go index a87ac46c6fef..12a6781c4ce0 100644 --- a/pkg/sql/sqlitelogictest/tests/3node-tenant/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/3node-tenant/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/fakedist-disk/generated_test.go b/pkg/sql/sqlitelogictest/tests/fakedist-disk/generated_test.go index 44147c400405..96c3ef8e2351 100644 --- a/pkg/sql/sqlitelogictest/tests/fakedist-disk/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/fakedist-disk/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/fakedist-vec-off/generated_test.go b/pkg/sql/sqlitelogictest/tests/fakedist-vec-off/generated_test.go index 2136561524c2..ee651e6c33de 100644 --- a/pkg/sql/sqlitelogictest/tests/fakedist-vec-off/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/fakedist-vec-off/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/fakedist/generated_test.go b/pkg/sql/sqlitelogictest/tests/fakedist/generated_test.go index c93ba59abdf9..9e5fd76e2e28 100644 --- a/pkg/sql/sqlitelogictest/tests/fakedist/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/fakedist/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-legacy-schema-changer/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-legacy-schema-changer/generated_test.go index 3872b9ee3e88..31d691e93b19 100644 --- a/pkg/sql/sqlitelogictest/tests/local-legacy-schema-changer/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-legacy-schema-changer/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-mixed-25.4/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-mixed-25.4/generated_test.go index 49c68e67c11d..a847962c02c6 100644 --- a/pkg/sql/sqlitelogictest/tests/local-mixed-25.4/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-mixed-25.4/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-mixed-26.1/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-mixed-26.1/generated_test.go index 7234a3aaf208..b13e1a02591c 100644 --- a/pkg/sql/sqlitelogictest/tests/local-mixed-26.1/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-mixed-26.1/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-mixed-26.2/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-mixed-26.2/generated_test.go index 532e38bc91a1..34654bc2d45f 100644 --- a/pkg/sql/sqlitelogictest/tests/local-mixed-26.2/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-mixed-26.2/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-prepared/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-prepared/generated_test.go index f3a2c7ba6d94..6e3d50967c7a 100644 --- a/pkg/sql/sqlitelogictest/tests/local-prepared/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-prepared/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-read-committed/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-read-committed/generated_test.go index 623f02279458..1cf9b1ebfe6f 100644 --- a/pkg/sql/sqlitelogictest/tests/local-read-committed/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-read-committed/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-repeatable-read/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-repeatable-read/generated_test.go index ff182221d77a..e88f665dc3bc 100644 --- a/pkg/sql/sqlitelogictest/tests/local-repeatable-read/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-repeatable-read/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local-vec-off/generated_test.go b/pkg/sql/sqlitelogictest/tests/local-vec-off/generated_test.go index 924229ae1414..4ba62133f11a 100644 --- a/pkg/sql/sqlitelogictest/tests/local-vec-off/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local-vec-off/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) } diff --git a/pkg/sql/sqlitelogictest/tests/local/generated_test.go b/pkg/sql/sqlitelogictest/tests/local/generated_test.go index 9868b2643ae5..d8b1d431bf06 100644 --- a/pkg/sql/sqlitelogictest/tests/local/generated_test.go +++ b/pkg/sql/sqlitelogictest/tests/local/generated_test.go @@ -75,7 +75,8 @@ func runSqliteLogicTest(t *testing.T, file string) { DisableUseMVCCRangeTombstonesForPointDeletes: true, // Some sqlite tests with very low bytes limit value are too slow, so // ensure 3 KiB lower bound. - BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + BatchBytesLimitLowerBound: 3 << 10, // 3 KiB + DisableOptimizerPerturbations: true, } logictest.RunLogicTest(t, serverArgs, configIdx, filepath.Join(sqliteLogicTestDir, file)) }