Skip to content

Commit 97563a7

Browse files
jycorJames Cor
andauthored
Decompose Filters with RecordExprs for index costing (#2860)
* transforming equality filters for record types for indexes * decompose filters when index is present * fix bug * aaa * off by 1 * fix * feedback * bump --------- Co-authored-by: James Cor <james@dolthub.com>
1 parent d2c0af2 commit 97563a7

5 files changed

Lines changed: 350 additions & 6 deletions

File tree

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@ require (
66
github.com/PuerkitoBio/goquery v1.8.1
77
github.com/cockroachdb/apd/v3 v3.2.3
88
github.com/cockroachdb/errors v1.7.5
9-
github.com/dolthub/dolt/go v0.40.5-0.20260615194251-a16fe3baf64b
9+
github.com/dolthub/dolt/go v0.40.5-0.20260624000019-4f8184183c4e
1010
github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69
1111
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
12-
github.com/dolthub/go-mysql-server v0.20.1-0.20260623211023-73b94872e61c
12+
github.com/dolthub/go-mysql-server v0.20.1-0.20260623224402-eee792a22ff6
1313
github.com/dolthub/pg_query_go/v6 v6.0.0-20251215122834-fb20be4254d1
1414
github.com/dolthub/sqllogictest/go v0.0.0-20240618184124-ca47f9354216
1515
github.com/dolthub/vitess v0.0.0-20260617012411-2f308f6cdc23

go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -246,8 +246,8 @@ github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 h1:I
246246
github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12/go.mod h1:rN7X8BHwkjPcfMQQ2QTAq/xM3leUSGLfb+1Js7Y6TVo=
247247
github.com/dolthub/dolt-mcp v0.3.4 h1:AyG5cw+fNWXDHXujtQnqUPZrpWtPg6FN6yYtjv1pP44=
248248
github.com/dolthub/dolt-mcp v0.3.4/go.mod h1:bCZ7KHvDYs+M0e+ySgmGiNvLhcwsN7bbf5YCyillLrk=
249-
github.com/dolthub/dolt/go v0.40.5-0.20260615194251-a16fe3baf64b h1:E9LrCMGUv7ULg/MXcq5Iq2l1huIuWLaPxF/JErDOiz4=
250-
github.com/dolthub/dolt/go v0.40.5-0.20260615194251-a16fe3baf64b/go.mod h1:B7Tmj1F2Jqz1mttm7Q3p4bNGPAwkULe0LvleelBFCIU=
249+
github.com/dolthub/dolt/go v0.40.5-0.20260624000019-4f8184183c4e h1:EydVoxtynei1yZ6vCWKFKU9XuIY3awlG5+K4qSdGUm8=
250+
github.com/dolthub/dolt/go v0.40.5-0.20260624000019-4f8184183c4e/go.mod h1:Ag5UOmZsjACE4bx1AmahSCf0HiOO4avxvyOQebVGKXM=
251251
github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69 h1:JShhbqMw26nKx3pqqu/cFxOpzBkN+4elVhzuUfgDw2k=
252252
github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69/go.mod h1:SSLraQS/jGLYFgff3vuZ+JbVUct6vyEeMzjLBqWqoyM=
253253
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww=
@@ -256,8 +256,8 @@ github.com/dolthub/fslock v0.0.5 h1:QoXhBhgY1oumHE26qyE7tgmXUT8qjJwxsIzo54O/B/k=
256256
github.com/dolthub/fslock v0.0.5/go.mod h1:sdofYYqE0D79zNZyB4/kmlnsQOVap1C2yByjGKSirEM=
257257
github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83 h1:FEMjCGEroDnY/BXyAffVZxUpXhP2GpoUJyyq5KaLn8c=
258258
github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83/go.mod h1:F3cnm+vMRK1HaU6+rNqQrOCyR03HHhR1GWG2gnPOqaE=
259-
github.com/dolthub/go-mysql-server v0.20.1-0.20260623211023-73b94872e61c h1:gGlf80pixECmSO9Rw0pgHKRFG0hVAj6yypkXO13tmAw=
260-
github.com/dolthub/go-mysql-server v0.20.1-0.20260623211023-73b94872e61c/go.mod h1:nd4+RZ0rp2P4JnJ16UwJPEBPN7kfYiaYQscnGyh1NYA=
259+
github.com/dolthub/go-mysql-server v0.20.1-0.20260623224402-eee792a22ff6 h1:vtPT9yQVaZoJ17llG4U1RDMIUYlgiJio/WJ7zWTXNbE=
260+
github.com/dolthub/go-mysql-server v0.20.1-0.20260623224402-eee792a22ff6/go.mod h1:nd4+RZ0rp2P4JnJ16UwJPEBPN7kfYiaYQscnGyh1NYA=
261261
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
262262
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
263263
github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 h1:oIW9HwuWrhxv+4HZxA+QQSKHLqWFyXZ2FmNjUYwkdiM=

server/analyzer/init.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const (
3838
ruleId_GenerateForeignKeyName // generateForeignKeyName
3939
ruleId_ReplaceIndexedTables // replaceIndexedTables
4040
ruleId_ReplaceNode // replaceNode
41+
ruleId_TransformRecordFilter // transformRecordFilter
4142
ruleId_ReplaceSerial // replaceSerial
4243
ruleId_InsertContextRootFinalizer // insertContextRootFinalizer
4344
ruleId_ResolveType // resolveType
@@ -96,6 +97,10 @@ func Init() {
9697
// Remove all other validation rules that do not apply to Postgres
9798
analyzer.DefaultValidationRules = removeAnalyzerRules(analyzer.DefaultValidationRules, analyzer.ValidateOperandsId)
9899

100+
analyzer.DefaultRules = append(analyzer.DefaultRules,
101+
analyzer.Rule{Id: ruleId_TransformRecordFilter, Apply: TransformRecordFilter},
102+
)
103+
99104
analyzer.OnceAfterDefault = append(analyzer.OnceAfterDefault,
100105
analyzer.Rule{Id: ruleId_ReplaceSerial, Apply: ReplaceSerial},
101106
analyzer.Rule{Id: ruleId_ReplaceArithmeticExpressions, Apply: ReplaceArithmeticExpressions},
Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
// Copyright 2026 Dolthub, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package analyzer
16+
17+
import (
18+
"github.com/dolthub/doltgresql/server/expression"
19+
"github.com/dolthub/doltgresql/server/functions/framework"
20+
"github.com/dolthub/doltgresql/server/types"
21+
22+
"github.com/dolthub/go-mysql-server/sql"
23+
"github.com/dolthub/go-mysql-server/sql/analyzer"
24+
gmsexpr "github.com/dolthub/go-mysql-server/sql/expression"
25+
"github.com/dolthub/go-mysql-server/sql/plan"
26+
"github.com/dolthub/go-mysql-server/sql/transform"
27+
)
28+
29+
// TransformRecordFilter finds Filter nodes over Indexable tables with composite indexes over Record columns and
30+
// decomposes the Record comparison into an equivalent filter expression over each column joined by ANDs and ORs.
31+
func TransformRecordFilter(
32+
ctx *sql.Context,
33+
_ *analyzer.Analyzer,
34+
node sql.Node,
35+
_ *plan.Scope,
36+
_ analyzer.RuleSelector,
37+
_ *sql.QueryFlags,
38+
) (sql.Node, transform.TreeIdentity, error) {
39+
return transform.Node(ctx, node, func(ctx *sql.Context, n sql.Node) (sql.Node, transform.TreeIdentity, error) {
40+
filter, ok := n.(*plan.Filter)
41+
if !ok {
42+
return n, transform.SameTree, nil
43+
}
44+
45+
var tblNode sql.TableNode
46+
switch child := filter.Child.(type) {
47+
case *plan.ResolvedTable:
48+
tblNode = child
49+
case *plan.TableAlias:
50+
tblNode, _ = child.Child.(sql.TableNode)
51+
default:
52+
return n, transform.SameTree, nil
53+
}
54+
// TODO: should only convert expressions when there's applicable index
55+
if _, ok = tblNode.UnderlyingTable().(sql.IndexAddressableTable); !ok {
56+
return n, transform.SameTree, nil
57+
}
58+
59+
newExpr, same, err := decomposeRecordFilter(ctx, filter.Expression)
60+
if err != nil {
61+
return nil, transform.SameTree, err
62+
}
63+
if same {
64+
return n, transform.SameTree, nil
65+
}
66+
return plan.NewFilter(ctx, newExpr, filter.Child), transform.NewTree, nil
67+
})
68+
}
69+
70+
// decomposeRecordFilter is the helper function to TransformRecordFilter that decomposes the Record comparison.
71+
func decomposeRecordFilter(ctx *sql.Context, expr sql.Expression) (sql.Expression, transform.TreeIdentity, error) {
72+
return transform.Expr(ctx, expr,
73+
func(ctx *sql.Context, e sql.Expression) (sql.Expression, transform.TreeIdentity, error) {
74+
binExpr, ok := e.(*expression.BinaryOperator)
75+
if !ok {
76+
return e, transform.SameTree, nil
77+
}
78+
// TODO: possible for Literal to be on left
79+
recExpr, ok := binExpr.Left().(*expression.RecordExpr)
80+
if !ok {
81+
return e, transform.SameTree, nil
82+
}
83+
recExprs := recExpr.Expressions()
84+
85+
// TODO: possible for RecordExpr to be on right
86+
litExpr, ok := binExpr.Right().(*gmsexpr.Literal)
87+
if !ok {
88+
return e, transform.SameTree, nil
89+
}
90+
recVals, ok := litExpr.Val.([]types.RecordValue)
91+
if !ok {
92+
return e, transform.SameTree, nil
93+
}
94+
95+
var newExpr sql.Expression
96+
var err error
97+
switch binExpr.Operator() {
98+
case framework.Operator_BinaryEqual:
99+
newExpr, err = decomposeRecordFilterEquals(ctx, recExprs, recVals)
100+
case framework.Operator_BinaryLessThan:
101+
newExpr, err = decomposeRecordFilterLessThan(ctx, recExprs, recVals)
102+
case framework.Operator_BinaryGreaterThan:
103+
newExpr, err = decomposeRecordFilterGreaterThan(ctx, recExprs, recVals)
104+
case framework.Operator_BinaryLessOrEqual:
105+
newExpr, err = decomposeRecordFilterLessThanEquals(ctx, recExprs, recVals)
106+
case framework.Operator_BinaryGreaterOrEqual:
107+
newExpr, err = decomposeRecordFilterGreaterThanEquals(ctx, recExprs, recVals)
108+
default:
109+
return e, transform.SameTree, nil
110+
}
111+
if err != nil {
112+
return nil, transform.SameTree, err
113+
}
114+
return newExpr, transform.NewTree, nil
115+
})
116+
}
117+
118+
// decomposeRecordFilterEquals splits up (i, j, ...) = (x, y, ...) expression into (i = x) AND (j = y) AND ...
119+
func decomposeRecordFilterEquals(
120+
ctx *sql.Context,
121+
recExprs []sql.Expression,
122+
recVals []types.RecordValue,
123+
) (sql.Expression, error) {
124+
n := len(recExprs)
125+
exprs := make([]sql.Expression, n)
126+
for i := 0; i < n; i++ {
127+
newLit := gmsexpr.NewLiteral(recVals[i].Value, recVals[i].Type)
128+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryEqual).WithChildren(ctx, recExprs[i], newLit)
129+
if err != nil {
130+
return nil, err
131+
}
132+
exprs[i] = expr
133+
}
134+
return gmsexpr.JoinAnd(exprs...), nil
135+
}
136+
137+
// decomposeRecordFilterLessThan splits up (i, j, ..., k) < (x, y, ..., z) into ((i = x) OR (j = y) ... OR (k < z)) AND ...
138+
func decomposeRecordFilterLessThan(
139+
ctx *sql.Context,
140+
recExprs []sql.Expression,
141+
recVals []types.RecordValue,
142+
) (sql.Expression, error) {
143+
n := len(recExprs)
144+
orExprs := make([]sql.Expression, n)
145+
for i := 0; i < n; i++ {
146+
andExprs := make([]sql.Expression, n-i)
147+
for j := 0; j < n-i-1; j++ {
148+
newLit := gmsexpr.NewLiteral(recVals[j].Value, recVals[j].Type)
149+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryEqual).WithChildren(
150+
ctx,
151+
recExprs[j],
152+
newLit,
153+
)
154+
if err != nil {
155+
return nil, err
156+
}
157+
andExprs[j] = expr
158+
}
159+
newLit := gmsexpr.NewLiteral(recVals[n-i-1].Value, recVals[n-i-1].Type)
160+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryLessThan).WithChildren(
161+
ctx,
162+
recExprs[i],
163+
newLit,
164+
)
165+
if err != nil {
166+
return nil, err
167+
}
168+
andExprs[n-i-1] = expr
169+
orExprs[i] = gmsexpr.JoinAnd(andExprs...)
170+
}
171+
return gmsexpr.JoinOr(orExprs...), nil
172+
}
173+
174+
// decomposeRecordFilterLessThan splits up (i, j, ..., k) > (x, y, ..., z) into ((i = x) OR (j = y) ... OR (k > z)) AND ...
175+
func decomposeRecordFilterGreaterThan(
176+
ctx *sql.Context,
177+
recExprs []sql.Expression,
178+
recVals []types.RecordValue,
179+
) (sql.Expression, error) {
180+
n := len(recExprs)
181+
orExprs := make([]sql.Expression, n)
182+
for i := 0; i < n; i++ {
183+
andExprs := make([]sql.Expression, n-i)
184+
for j := 0; j < n-i-1; j++ {
185+
newLit := gmsexpr.NewLiteral(recVals[j].Value, recVals[j].Type)
186+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryEqual).WithChildren(
187+
ctx,
188+
recExprs[j],
189+
newLit,
190+
)
191+
if err != nil {
192+
return nil, err
193+
}
194+
andExprs[j] = expr
195+
}
196+
newLit := gmsexpr.NewLiteral(recVals[n-i-1].Value, recVals[n-i-1].Type)
197+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryGreaterThan).WithChildren(
198+
ctx,
199+
recExprs[n-i-1],
200+
newLit,
201+
)
202+
if err != nil {
203+
return nil, err
204+
}
205+
andExprs[n-i-1] = expr
206+
orExprs[i] = gmsexpr.JoinAnd(andExprs...)
207+
}
208+
return gmsexpr.JoinOr(orExprs...), nil
209+
}
210+
211+
// decomposeRecordFilterLessThanEquals splits up (i, j, ..., k) <= (x, y, ..., z) into ((i = x) OR (j = y) ... OR (k <= z)) AND ...
212+
func decomposeRecordFilterLessThanEquals(
213+
ctx *sql.Context,
214+
recExprs []sql.Expression,
215+
recVals []types.RecordValue,
216+
) (sql.Expression, error) {
217+
n := len(recExprs)
218+
orExprs := make([]sql.Expression, n)
219+
andExprs := make([]sql.Expression, n)
220+
for i := 0; i < n-1; i++ {
221+
newLit := gmsexpr.NewLiteral(recVals[i].Value, recVals[i].Type)
222+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryEqual).WithChildren(
223+
ctx,
224+
recExprs[i],
225+
newLit,
226+
)
227+
if err != nil {
228+
return nil, err
229+
}
230+
andExprs[i] = expr
231+
}
232+
newLit := gmsexpr.NewLiteral(recVals[n-1].Value, recVals[n-1].Type)
233+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryLessOrEqual).WithChildren(
234+
ctx,
235+
recExprs[n-1],
236+
newLit,
237+
)
238+
if err != nil {
239+
return nil, err
240+
}
241+
andExprs[n-1] = expr
242+
orExprs[0] = gmsexpr.JoinAnd(andExprs...)
243+
244+
for i := 1; i < n; i++ {
245+
andExprs = make([]sql.Expression, n-i)
246+
for j := 0; j < n-i-1; j++ {
247+
newLit = gmsexpr.NewLiteral(recVals[j].Value, recVals[j].Type)
248+
expr, err = expression.NewBinaryOperator(framework.Operator_BinaryEqual).WithChildren(
249+
ctx,
250+
recExprs[j],
251+
newLit,
252+
)
253+
if err != nil {
254+
return nil, err
255+
}
256+
andExprs[j] = expr
257+
}
258+
newLit = gmsexpr.NewLiteral(recVals[n-i-1].Value, recVals[n-i-1].Type)
259+
expr, err = expression.NewBinaryOperator(framework.Operator_BinaryLessThan).WithChildren(
260+
ctx,
261+
recExprs[n-i-1],
262+
newLit,
263+
)
264+
if err != nil {
265+
return nil, err
266+
}
267+
andExprs[n-i-1] = expr
268+
orExprs[i] = gmsexpr.JoinAnd(andExprs...)
269+
}
270+
return gmsexpr.JoinOr(orExprs...), nil
271+
}
272+
273+
// decomposeRecordFilterGreaterThanEquals splits up (i, j, ..., k) >= (x, y, ..., z) into ((i = x) OR (j = y) ... OR (k >= z)) AND ...
274+
func decomposeRecordFilterGreaterThanEquals(
275+
ctx *sql.Context,
276+
recExprs []sql.Expression,
277+
recVals []types.RecordValue,
278+
) (sql.Expression, error) {
279+
n := len(recExprs)
280+
orExprs := make([]sql.Expression, n)
281+
andExprs := make([]sql.Expression, n)
282+
for i := 0; i < n-1; i++ {
283+
newLit := gmsexpr.NewLiteral(recVals[i].Value, recVals[i].Type)
284+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryEqual).WithChildren(
285+
ctx,
286+
recExprs[i],
287+
newLit,
288+
)
289+
if err != nil {
290+
return nil, err
291+
}
292+
andExprs[i] = expr
293+
}
294+
295+
newLit := gmsexpr.NewLiteral(recVals[n-1].Value, recVals[n-1].Type)
296+
expr, err := expression.NewBinaryOperator(framework.Operator_BinaryGreaterOrEqual).WithChildren(
297+
ctx,
298+
recExprs[n-1],
299+
newLit,
300+
)
301+
if err != nil {
302+
return nil, err
303+
}
304+
andExprs[n-1] = expr
305+
orExprs[0] = gmsexpr.JoinAnd(andExprs...)
306+
307+
for i := 1; i < n; i++ {
308+
andExprs = make([]sql.Expression, n-i)
309+
for j := 0; j < n-i-1; j++ {
310+
newLit = gmsexpr.NewLiteral(recVals[j].Value, recVals[j].Type)
311+
expr, err = expression.NewBinaryOperator(framework.Operator_BinaryEqual).WithChildren(
312+
ctx,
313+
recExprs[j],
314+
newLit,
315+
)
316+
if err != nil {
317+
return nil, err
318+
}
319+
andExprs[j] = expr
320+
}
321+
newLit = gmsexpr.NewLiteral(recVals[n-i-1].Value, recVals[n-i-1].Type)
322+
expr, err = expression.NewBinaryOperator(framework.Operator_BinaryGreaterThan).WithChildren(
323+
ctx,
324+
recExprs[n-i-1],
325+
newLit,
326+
)
327+
if err != nil {
328+
return nil, err
329+
}
330+
andExprs[n-i-1] = expr
331+
orExprs[i] = gmsexpr.JoinAnd(andExprs...)
332+
}
333+
return gmsexpr.JoinOr(orExprs...), nil
334+
}

server/expression/record.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,8 @@ func (t *RecordExpr) WithResolvedChildren(ctx context.Context, children []any) (
115115
}
116116
return t.WithChildren(ctx.(*sql.Context), newExpressions...)
117117
}
118+
119+
// Expressions implements the expression.TupleLike interface.
120+
func (t *RecordExpr) Expressions() []sql.Expression {
121+
return t.exprs
122+
}

0 commit comments

Comments
 (0)