Skip to content

Commit e9e41e4

Browse files
trunk-io[bot]trunk-io[bot]
authored
opt: add optimizer_span_limit and use in GenerateConstrainedScans (#167180)
opt: add optimizer_span_limit and use in GenerateConstrainedScans
2 parents 2b23072 + 9887a57 commit e9e41e4

22 files changed

Lines changed: 590 additions & 9 deletions

File tree

‎docs/generated/sql/session_vars.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ SHOW application_name;
161161
<tr><td><code>optimizer_prove_implication_with_virtual_computed_columns</code></td><td>Controls whether the optimizer should use virtual computed columns to prove partial index implication.</td><td><code>on</code></td><td>No</td><td>-</td></tr>
162162
<tr><td><code>optimizer_push_limit_into_project_filtered_scan</code></td><td>Controls whether the optimizer should push limit expressions into projects of filtered scans.</td><td><code>on</code></td><td>No</td><td>-</td></tr>
163163
<tr><td><code>optimizer_push_offset_into_index_join</code></td><td>Controls whether the optimizer should push offset expressions into index joins.</td><td><code>on</code></td><td>No</td><td>-</td></tr>
164+
<tr><td><code>optimizer_span_limit</code></td><td>Sets the maximum number of constraint spans allowed in a scan during query optimization. 0 means no limit.</td><td><code>131072</code></td><td>No</td><td>-</td></tr>
164165
<tr><td><code>optimizer_use_conditional_hoist_fix</code></td><td>Prevents the optimizer from hoisting volatile expressions that are conditionally executed by CASE, COALESCE, or IFERR expressions.</td><td><code>on</code></td><td>No</td><td>-</td></tr>
165166
<tr><td><code>optimizer_use_delete_range_fast_path</code></td><td>Controls whether the optimizer uses the fast path for DELETE operations using range deletions.</td><td><code>on</code></td><td>No</td><td>-</td></tr>
166167
<tr><td><code>optimizer_use_exists_filter_hoist_rule</code></td><td>Controls whether the optimizer hoists filters out of EXISTS subqueries.</td><td><code>on</code></td><td>No</td><td>-</td></tr>

‎pkg/sql/logictest/testdata/logic_test/information_schema‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3919,6 +3919,7 @@ optimizer_prefer_bounded_cardinality on
39193919
optimizer_prove_implication_with_virtual_computed_columns on
39203920
optimizer_push_limit_into_project_filtered_scan on
39213921
optimizer_push_offset_into_index_join on
3922+
optimizer_span_limit 131072
39223923
optimizer_use_conditional_hoist_fix on
39233924
optimizer_use_delete_range_fast_path on
39243925
optimizer_use_exists_filter_hoist_rule on

‎pkg/sql/logictest/testdata/logic_test/pg_catalog‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3363,6 +3363,7 @@ optimizer_prefer_bounded_cardinality on
33633363
optimizer_prove_implication_with_virtual_computed_columns on NULL NULL NULL string
33643364
optimizer_push_limit_into_project_filtered_scan on NULL NULL NULL string
33653365
optimizer_push_offset_into_index_join on NULL NULL NULL string
3366+
optimizer_span_limit 131072 NULL NULL NULL string
33663367
optimizer_use_conditional_hoist_fix on NULL NULL NULL string
33673368
optimizer_use_delete_range_fast_path on NULL NULL NULL string
33683369
optimizer_use_exists_filter_hoist_rule on NULL NULL NULL string
@@ -3622,6 +3623,7 @@ optimizer_prefer_bounded_cardinality on
36223623
optimizer_prove_implication_with_virtual_computed_columns on NULL user NULL on on
36233624
optimizer_push_limit_into_project_filtered_scan on NULL user NULL on on
36243625
optimizer_push_offset_into_index_join on NULL user NULL on on
3626+
optimizer_span_limit 131072 NULL user NULL 131072 131072
36253627
optimizer_use_conditional_hoist_fix on NULL user NULL on on
36263628
optimizer_use_delete_range_fast_path on NULL user NULL on on
36273629
optimizer_use_exists_filter_hoist_rule on NULL user NULL on on
@@ -3872,6 +3874,7 @@ optimizer_prefer_bounded_cardinality NULL NULL
38723874
optimizer_prove_implication_with_virtual_computed_columns NULL NULL NULL NULL NULL
38733875
optimizer_push_limit_into_project_filtered_scan NULL NULL NULL NULL NULL
38743876
optimizer_push_offset_into_index_join NULL NULL NULL NULL NULL
3877+
optimizer_span_limit NULL NULL NULL NULL NULL
38753878
optimizer_use_conditional_hoist_fix NULL NULL NULL NULL NULL
38763879
optimizer_use_delete_range_fast_path NULL NULL NULL NULL NULL
38773880
optimizer_use_exists_filter_hoist_rule NULL NULL NULL NULL NULL

‎pkg/sql/logictest/testdata/logic_test/select_index‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
statement ok
32
CREATE TABLE t (
43
a INT PRIMARY KEY,
@@ -721,3 +720,58 @@ query I
721720
SELECT DISTINCT _int FROM t88110 WHERE NOT _bool;
722721
----
723722
NULL
723+
724+
subtest optimizer_span_limit
725+
726+
# Correctness tests for the optimizer_span_limit session setting.
727+
728+
statement ok
729+
CREATE TABLE t167620 (a INT, b INT, c INT, INDEX (a, b, c))
730+
731+
statement ok
732+
INSERT INTO t167620 VALUES
733+
(1, 2, 7), (1, 4, 9), (3, 6, 11),
734+
(5, 2, 9), (3, 4, 7), (1, 6, 11),
735+
(1, 2, 99), (1, 3, 7), (2, 2, 7)
736+
737+
# Verify that results are the same regardless of span limit.
738+
query III rowsort
739+
SELECT * FROM t167620@t167620_a_b_c_idx WHERE a IN (1, 3, 5) AND b IN (2, 4, 6) AND c IN (7, 9, 11)
740+
----
741+
1 2 7
742+
1 4 9
743+
1 6 11
744+
3 4 7
745+
3 6 11
746+
5 2 9
747+
748+
statement ok
749+
SET optimizer_span_limit = 10
750+
751+
query III rowsort
752+
SELECT * FROM t167620@t167620_a_b_c_idx WHERE a IN (1, 3, 5) AND b IN (2, 4, 6) AND c IN (7, 9, 11)
753+
----
754+
1 2 7
755+
1 4 9
756+
1 6 11
757+
3 4 7
758+
3 6 11
759+
5 2 9
760+
761+
statement ok
762+
SET optimizer_span_limit = 8
763+
764+
query III rowsort
765+
SELECT * FROM t167620@t167620_a_b_c_idx WHERE a IN (1, 3, 5) AND b IN (2, 4, 6) AND c IN (7, 9, 11)
766+
----
767+
1 2 7
768+
1 4 9
769+
1 6 11
770+
3 4 7
771+
3 6 11
772+
5 2 9
773+
774+
statement ok
775+
RESET optimizer_span_limit
776+
777+
subtest end

‎pkg/sql/logictest/testdata/logic_test/show_source‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ optimizer_prefer_bounded_cardinality on
168168
optimizer_prove_implication_with_virtual_computed_columns on Controls whether the optimizer should use virtual computed columns to prove partial index implication.
169169
optimizer_push_limit_into_project_filtered_scan on Controls whether the optimizer should push limit expressions into projects of filtered scans.
170170
optimizer_push_offset_into_index_join on Controls whether the optimizer should push offset expressions into index joins.
171+
optimizer_span_limit 131072 Sets the maximum number of constraint spans allowed in a scan during query optimization. 0 means no limit.
171172
optimizer_use_conditional_hoist_fix on Prevents the optimizer from hoisting volatile expressions that are conditionally executed by CASE, COALESCE, or IFERR expressions.
172173
optimizer_use_delete_range_fast_path on Controls whether the optimizer uses the fast path for DELETE operations using range deletions.
173174
optimizer_use_exists_filter_hoist_rule on Controls whether the optimizer hoists filters out of EXISTS subqueries.

‎pkg/sql/opt/exec/execbuilder/testdata/select_index‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1925,3 +1925,100 @@ distribution: local
19251925
missing stats
19261926
table: t5@t5_a_idx
19271927
spans: [/'A.B.C' - /'A.B.C-')
1928+
1929+
subtest optimizer_span_limit
1930+
1931+
# Tests for the optimizer_span_limit session setting, which limits the number of
1932+
# index constraint spans generated during optimization. When the cross-product
1933+
# of spans across index columns would exceed the limit, the optimizer stops
1934+
# extending spans and adds remaining filters instead.
1935+
1936+
statement ok
1937+
CREATE TABLE t167620 (a INT, b INT, c INT, INDEX (a, b, c));
1938+
INSERT INTO t167620 VALUES
1939+
(1, 2, 7), (1, 4, 9), (3, 6, 11),
1940+
(5, 2, 9), (3, 4, 7), (1, 6, 11),
1941+
(1, 2, 99), (1, 3, 7), (2, 2, 7)
1942+
1943+
# No limit (default): 27 spans for the full 3x3x3 cross-product.
1944+
query T
1945+
EXPLAIN (OPT) SELECT * FROM t167620@t167620_a_b_c_idx WHERE a IN (1, 3, 5) AND b IN (2, 4, 6) AND c IN (7, 9, 11)
1946+
----
1947+
scan t167620@t167620_a_b_c_idx
1948+
├── constraint: /1/2/3/4
1949+
│ ├── [/1/2/7 - /1/2/7]
1950+
│ ├── [/1/2/9 - /1/2/9]
1951+
│ ├── [/1/2/11 - /1/2/11]
1952+
│ ├── [/1/4/7 - /1/4/7]
1953+
│ ├── [/1/4/9 - /1/4/9]
1954+
│ ├── [/1/4/11 - /1/4/11]
1955+
│ ├── [/1/6/7 - /1/6/7]
1956+
│ ├── [/1/6/9 - /1/6/9]
1957+
│ ├── [/1/6/11 - /1/6/11]
1958+
│ ├── [/3/2/7 - /3/2/7]
1959+
│ ├── [/3/2/9 - /3/2/9]
1960+
│ ├── [/3/2/11 - /3/2/11]
1961+
│ ├── [/3/4/7 - /3/4/7]
1962+
│ ├── [/3/4/9 - /3/4/9]
1963+
│ ├── [/3/4/11 - /3/4/11]
1964+
│ ├── [/3/6/7 - /3/6/7]
1965+
│ ├── [/3/6/9 - /3/6/9]
1966+
│ ├── [/3/6/11 - /3/6/11]
1967+
│ ├── [/5/2/7 - /5/2/7]
1968+
│ ├── [/5/2/9 - /5/2/9]
1969+
│ ├── [/5/2/11 - /5/2/11]
1970+
│ ├── [/5/4/7 - /5/4/7]
1971+
│ ├── [/5/4/9 - /5/4/9]
1972+
│ ├── [/5/4/11 - /5/4/11]
1973+
│ ├── [/5/6/7 - /5/6/7]
1974+
│ ├── [/5/6/9 - /5/6/9]
1975+
│ └── [/5/6/11 - /5/6/11]
1976+
└── flags: force-index=t167620_a_b_c_idx
1977+
1978+
# With limit=10, the final 9*3 cross-product is blocked. We get 9 (a, b) spans
1979+
# with a remaining filter on c.
1980+
statement ok
1981+
SET optimizer_span_limit = 10
1982+
1983+
query T
1984+
EXPLAIN (OPT) SELECT * FROM t167620@t167620_a_b_c_idx WHERE a IN (1, 3, 5) AND b IN (2, 4, 6) AND c IN (7, 9, 11)
1985+
----
1986+
select
1987+
├── scan t167620@t167620_a_b_c_idx
1988+
│ ├── constraint: /1/2/3/4
1989+
│ │ ├── [/1/2 - /1/2]
1990+
│ │ ├── [/1/4 - /1/4]
1991+
│ │ ├── [/1/6 - /1/6]
1992+
│ │ ├── [/3/2 - /3/2]
1993+
│ │ ├── [/3/4 - /3/4]
1994+
│ │ ├── [/3/6 - /3/6]
1995+
│ │ ├── [/5/2 - /5/2]
1996+
│ │ ├── [/5/4 - /5/4]
1997+
│ │ └── [/5/6 - /5/6]
1998+
│ └── flags: force-index=t167620_a_b_c_idx
1999+
└── filters
2000+
└── c IN (7, 9, 11)
2001+
2002+
# With limit=8, the 3*3 cross-product for b is also blocked. We get 3 (a) spans
2003+
# with remaining filters on b and c.
2004+
statement ok
2005+
SET optimizer_span_limit = 8
2006+
2007+
query T
2008+
EXPLAIN (OPT) SELECT * FROM t167620@t167620_a_b_c_idx WHERE a IN (1, 3, 5) AND b IN (2, 4, 6) AND c IN (7, 9, 11)
2009+
----
2010+
select
2011+
├── scan t167620@t167620_a_b_c_idx
2012+
│ ├── constraint: /1/2/3/4
2013+
│ │ ├── [/1 - /1]
2014+
│ │ ├── [/3 - /3]
2015+
│ │ └── [/5 - /5]
2016+
│ └── flags: force-index=t167620_a_b_c_idx
2017+
└── filters
2018+
├── b IN (2, 4, 6)
2019+
└── c IN (7, 9, 11)
2020+
2021+
statement ok
2022+
RESET optimizer_span_limit
2023+
2024+
subtest end

‎pkg/sql/opt/idxconstraint/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ go_library(
1616
"//pkg/sql/sem/tree",
1717
"//pkg/sql/types",
1818
"//pkg/util",
19+
"//pkg/util/log",
1920
"//pkg/util/ltree",
2021
"@com_github_cockroachdb_errors//:errors",
2122
],

‎pkg/sql/opt/idxconstraint/index_constraints.go‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
2121
"github.com/cockroachdb/cockroach/pkg/sql/types"
2222
"github.com/cockroachdb/cockroach/pkg/util"
23+
"github.com/cockroachdb/cockroach/pkg/util/log"
2324
"github.com/cockroachdb/cockroach/pkg/util/ltree"
2425
"github.com/cockroachdb/errors"
2526
)
@@ -145,6 +146,16 @@ func (c *indexConstraintCtx) makeSpansForSingleColumn(
145146
) (tight bool) {
146147
if op == opt.InOp && memo.CanExtractConstTuple(val) {
147148
tupVal := val.(*memo.TupleExpr)
149+
// If the IN list has more elements than the span limit, return
150+
// unconstrained.
151+
if c.spanLimit > 0 && len(tupVal.Elems) > c.spanLimit {
152+
log.VEventf(c.ctx, 2,
153+
"not building spans for IN list: has %d elements, exceeds optimizer_span_limit of %d",
154+
len(tupVal.Elems), c.spanLimit,
155+
)
156+
c.unconstrained(offset, out)
157+
return false
158+
}
148159
keyCtx := &c.keyCtx[offset]
149160
var spans constraint.Spans
150161
spans.Alloc(len(tupVal.Elems))
@@ -608,6 +619,16 @@ func (c *indexConstraintCtx) makeSpansForTupleIn(
608619
return false
609620
}
610621

622+
// If the tuple has more elements than the span limit, return unconstrained.
623+
if c.spanLimit > 0 && len(rhs.Elems) > c.spanLimit {
624+
log.VEventf(c.ctx, 2,
625+
"not building spans for tuple: has %d elements, exceeds optimizer_span_limit of %d",
626+
len(rhs.Elems), c.spanLimit,
627+
)
628+
c.unconstrained(offset, out)
629+
return false
630+
}
631+
611632
// Create a span for each (tuple) value inside the right-hand side tuple.
612633
keyCtx := &c.keyCtx[offset]
613634
var spans constraint.Spans
@@ -891,17 +912,32 @@ func (c *indexConstraintCtx) makeSpansForAnd(
891912
break
892913
}
893914

915+
var tightFilters util.FastIntMap
894916
tight := c.makeSpansForExpr(offset+delta, filters[0].Condition, &ofsC)
895917
if tight {
896-
tightDeltaMap.Set(0, delta)
918+
tightFilters.Set(0, delta)
897919
}
898920
for j := 1; j < len(filters); j++ {
899921
tight := c.makeSpansForExpr(offset+delta, filters[j].Condition, &exprConstraint)
900922
if tight {
901-
tightDeltaMap.Set(j, delta)
923+
tightFilters.Set(j, delta)
902924
}
903925
ofsC.IntersectWith(c.ctx, c.evalCtx, &exprConstraint)
904926
}
927+
// If combining could produce more spans than the limit (e.g. via
928+
// cross-product of exact-match prefix spans with suffix spans), skip the
929+
// suffix extension.
930+
if c.spanLimit > 0 && out.Spans.Count()*ofsC.Spans.Count() > c.spanLimit {
931+
log.VEventf(c.ctx, 2,
932+
"limiting index span tightness: combining %d spans with %d suffix spans could exceed optimizer_span_limit of %d",
933+
out.Spans.Count(), ofsC.Spans.Count(), c.spanLimit,
934+
)
935+
break
936+
}
937+
// Record tightness after we've confirmed we'll actually combine.
938+
tightFilters.ForEach(func(j, delta int) {
939+
tightDeltaMap.Set(j, delta)
940+
})
905941
out.Combine(c.ctx, c.evalCtx, &ofsC, c.checkCancellation)
906942
numIterations++
907943
// In case we can't exit this loop, allow the cancel checker to cancel
@@ -1155,6 +1191,11 @@ type Instance struct {
11551191
// they need not generate remaining filters. This is e.g. used for check
11561192
// constraints that can help generate better spans but don't actually need to be
11571193
// enforced.
1194+
//
1195+
// spanLimit limits the number of spans that will be generated during constraint
1196+
// building. When a span-generating operation would produce more spans than this
1197+
// limit, the constraint builder returns a looser result instead (in some cases
1198+
// fully unconstrained). A value of 0 means no limit.
11581199
func (ic *Instance) Init(
11591200
ctx context.Context,
11601201
requiredFilters memo.FiltersExpr,
@@ -1167,6 +1208,7 @@ func (ic *Instance) Init(
11671208
evalCtx *eval.Context,
11681209
factory *norm.Factory,
11691210
ps partition.PrefixSorter,
1211+
spanLimit int,
11701212
checkCancellation func(),
11711213
) {
11721214
// This initialization pattern ensures that fields are not unwittingly
@@ -1183,7 +1225,7 @@ func (ic *Instance) Init(
11831225
ic.allFilters = requiredFilters[:len(requiredFilters):len(requiredFilters)]
11841226
ic.allFilters = append(ic.allFilters, optionalFilters...)
11851227
}
1186-
ic.indexConstraintCtx.init(ctx, columns, notNullCols, computedCols, colsInComputedColsExpressions, evalCtx, factory, checkCancellation)
1228+
ic.indexConstraintCtx.init(ctx, columns, notNullCols, computedCols, colsInComputedColsExpressions, evalCtx, factory, spanLimit, checkCancellation)
11871229
ic.tight = ic.makeSpansForExpr(0 /* offset */, &ic.allFilters, &ic.constraint)
11881230

11891231
// Note: If consolidate is true, we only consolidate spans at the
@@ -1295,6 +1337,10 @@ type indexConstraintCtx struct {
12951337
ctx context.Context
12961338
evalCtx *eval.Context
12971339

1340+
// spanLimit limits the number of spans that will be generated during
1341+
// constraint building. A value of 0 means no limit.
1342+
spanLimit int
1343+
12981344
// We pre-initialize the KeyContext for each suffix of the index columns.
12991345
keyCtx []constraint.KeyContext
13001346

@@ -1311,6 +1357,7 @@ func (c *indexConstraintCtx) init(
13111357
colsInComputedColsExpressions opt.ColSet,
13121358
evalCtx *eval.Context,
13131359
factory *norm.Factory,
1360+
spanLimit int,
13141361
checkCancellation func(),
13151362
) {
13161363
var keyCols, computedColSet opt.ColSet
@@ -1332,6 +1379,7 @@ func (c *indexConstraintCtx) init(
13321379
colsInComputedColsExpressions: colsInComputedColsExpressions,
13331380
ctx: ctx,
13341381
evalCtx: evalCtx,
1382+
spanLimit: spanLimit,
13351383
factory: factory,
13361384
keyCtx: make([]constraint.KeyContext, len(columns)),
13371385
checkCancellation: checkCancellation,
@@ -1422,6 +1470,8 @@ func IndexPrefixCols(
14221470
// spans have the same start and end keys for all prefix columns. This is
14231471
// required for building spans for scanning multi-column inverted/vector indexes
14241472
// (see span.Builder.SpansFromInvertedSpans).
1473+
//
1474+
// TODO(michae2): Accept and use optimizer_span_limit.
14251475
func ConstrainIndexPrefixCols(
14261476
ctx context.Context,
14271477
evalCtx *eval.Context,
@@ -1461,7 +1511,9 @@ func ConstrainIndexPrefixCols(
14611511
columns, notNullCols, tabMeta.ComputedCols,
14621512
tabMeta.ColsInComputedColsExpressions,
14631513
false, /* consolidate */
1464-
evalCtx, factory, ps, checkCancellation,
1514+
evalCtx, factory, ps,
1515+
0, /* spanLimit */
1516+
checkCancellation,
14651517
)
14661518
var c constraint.Constraint
14671519
ic.UnconsolidatedConstraint(&c)

0 commit comments

Comments
 (0)