diff --git a/pkg/container/batch/batch_test.go b/pkg/container/batch/batch_test.go index 7b7a2970dc2ea..47fe7b616fbfb 100644 --- a/pkg/container/batch/batch_test.go +++ b/pkg/container/batch/batch_test.go @@ -18,8 +18,6 @@ import ( "bytes" "testing" - "github.com/matrixorigin/matrixone/pkg/sql/colexec/aggexec" - "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -155,7 +153,6 @@ func newBatch(ts []types.Type, rows int) *Batch { } bat.ExtraBuf = []byte("extra buf") - aggexec.RegisterGroupConcatAgg(0, ",") bat.Attrs = []string{"1"} return bat } diff --git a/pkg/sql/colexec/aggexec/aggState.go b/pkg/sql/colexec/aggexec/aggState.go index 86c6165e7629e..abe9d703903d8 100644 --- a/pkg/sql/colexec/aggexec/aggState.go +++ b/pkg/sql/colexec/aggexec/aggState.go @@ -46,6 +46,10 @@ type MarshalerUnmarshaler interface { UnmarshalFromReader(io.Reader) error } +type freeableMarshalerUnmarshaler interface { + Free() +} + type aggInfo struct { aggId int64 isDistinct bool @@ -648,6 +652,12 @@ func (ag *aggState) free(mp *mpool.MPool) { vec.Free(mp) } ag.vecs = nil + for _, mob := range ag.mobs { + if freeable, ok := mob.(freeableMarshalerUnmarshaler); ok { + freeable.Free() + } + } + ag.mobs = nil } type aggExec struct { @@ -822,9 +832,15 @@ func checkAggStateMagic(reader io.Reader) { } } -func (ae *aggExec) UnmarshalFromReader(reader io.Reader, mp *mpool.MPool) error { +func (ae *aggExec) UnmarshalFromReader(reader io.Reader, mp *mpool.MPool) (retErr error) { checkAggStateMagic(reader) defer checkAggStateMagic(reader) + defer func() { + if retErr != nil { + ae.Free() + ae.state = nil + } + }() // read number of chunks cnt, err := types.ReadInt32(reader) diff --git a/pkg/sql/colexec/aggexec/aggexec_test.go b/pkg/sql/colexec/aggexec/aggexec_test.go index f4c36dbd95015..9b37e93d5162e 100644 --- a/pkg/sql/colexec/aggexec/aggexec_test.go +++ b/pkg/sql/colexec/aggexec/aggexec_test.go @@ -30,11 +30,9 @@ func TestMakeAggSpecialAgg(t *testing.T) { require.Equal(t, int64(0), mp.CurrNB()) }() - const testAggID = -9901 param := types.T_int64.ToType() - RegisterAvgTwCache(testAggID) - exec, err := MakeAgg(mp, testAggID, false, param) + exec, err := MakeAgg(mp, AggIdOfAvgTwCache, false, param) require.NoError(t, err) require.NoError(t, exec.GroupGrow(2)) @@ -73,6 +71,59 @@ func TestVectorsUnmarshalFromReader(t *testing.T) { exec2.Free() } +func TestAggExecUnmarshalReplacesExistingState(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + newExec := func() *countColumnExec { + return newCountColumnExec( + mp, + AggIdOfCountColumn, + false, + []types.Type{types.T_int64.ToType()}, + ).(*countColumnExec) + } + + t.Run("empty", func(t *testing.T) { + target := newExec() + defer target.Free() + require.NoError(t, target.GroupGrow(1)) + + empty := newExec() + var buf bytes.Buffer + require.NoError(t, empty.SaveIntermediateResult(0, nil, &buf)) + empty.Free() + require.NoError(t, target.UnmarshalFromReader(bytes.NewReader(buf.Bytes()), mp)) + require.Zero(t, target.GetNumGroups()) + }) + + t.Run("multiple_chunks", func(t *testing.T) { + target := newExec() + defer target.Free() + require.NoError(t, target.GroupGrow(2)) + + source := newExec() + defer source.Free() + require.NoError(t, source.GroupGrow(AggBatchSize+1)) + flags := [][]uint8{make([]uint8, AggBatchSize), {1}} + for i := range flags[0] { + flags[0][i] = 1 + } + var buf bytes.Buffer + require.NoError(t, source.SaveIntermediateResult(AggBatchSize+1, flags, &buf)) + require.NoError(t, target.UnmarshalFromReader(bytes.NewReader(buf.Bytes()), mp)) + require.Equal(t, AggBatchSize+1, target.GetNumGroups()) + + results, err := target.Flush() + require.NoError(t, err) + require.Len(t, results, 2) + for _, result := range results { + result.Free(mp) + } + }) +} + func TestIntermediateResultCompactsAndReusesSingleChunk(t *testing.T) { mp := mpool.MustNewZero() defer func() { diff --git a/pkg/sql/colexec/aggexec/approx_percentile.go b/pkg/sql/colexec/aggexec/approx_percentile.go new file mode 100644 index 0000000000000..92c57b2756bc4 --- /dev/null +++ b/pkg/sql/colexec/aggexec/approx_percentile.go @@ -0,0 +1,1046 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aggexec + +import ( + "bytes" + "io" + "math" + "math/big" + "sort" + "strconv" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" +) + +const ( + approxPercentileSketchCapacity = 200 + approxPercentileMaxLevels = 64 + approxPercentileSketchVersion = byte(1) + approxPercentileDecimalWidth = int32(38) +) + +func ApproxPercentileReturnType(args []types.Type) types.Type { + if !args[0].IsDecimal() { + return types.T_float64.ToType() + } + scale := args[0].Scale + if args[0].Width < approxPercentileDecimalWidth { + scale++ + } + return types.New(types.T_decimal128, approxPercentileDecimalWidth, scale) +} + +type quantileValue interface { + numeric | types.Decimal64 | types.Decimal128 +} + +// quantileSketch is a bounded, mergeable KLL-style sketch. Each level stores +// values with weight 2^level and is compacted once it reaches 2*k entries. +// The fixed level limit bounds every group's retained state independently of +// its input row count. Level headers are Go-managed, while every value buffer +// is allocated from mp so group spill accounting sees the retained samples. +type quantileSketch[T quantileValue] struct { + levels [][]T + parity []bool + count uint64 + min T + max T + hasValue bool + compare func(T, T) int + mp *mpool.MPool +} + +func newQuantileSketch[T quantileValue](mp *mpool.MPool, compare func(T, T) int) *quantileSketch[T] { + return &quantileSketch[T]{compare: compare, mp: mp} +} + +func (s *quantileSketch[T]) Add(value T) error { + if s.count == math.MaxUint64 { + return moerr.NewInvalidInputNoCtx("approx_percentile: row count overflow") + } + if err := s.appendValue(0, value); err != nil { + return err + } + if !s.hasValue { + s.min, s.max, s.hasValue = value, value, true + } else { + if s.compare(value, s.min) < 0 { + s.min = value + } + if s.compare(value, s.max) > 0 { + s.max = value + } + } + s.count++ + return s.compactFrom(0) +} + +func (s *quantileSketch[T]) Merge(other *quantileSketch[T]) error { + if other == nil || other.count == 0 { + return nil + } + if math.MaxUint64-s.count < other.count { + return moerr.NewInvalidInputNoCtx("approx_percentile: row count overflow") + } + if !s.hasValue { + s.min, s.max, s.hasValue = other.min, other.max, other.hasValue + } else if other.hasValue { + if s.compare(other.min, s.min) < 0 { + s.min = other.min + } + if s.compare(other.max, s.max) > 0 { + s.max = other.max + } + } + for level, values := range other.levels { + s.ensureLevel(level) + if err := s.appendValues(level, values); err != nil { + return err + } + if level < len(other.parity) && other.parity[level] { + s.parity[level] = !s.parity[level] + } + } + if err := s.compactFrom(0); err != nil { + return err + } + s.count += other.count + return nil +} + +func (s *quantileSketch[T]) ensureLevel(level int) { + for len(s.levels) <= level { + s.levels = append(s.levels, nil) + s.parity = append(s.parity, false) + } +} + +func (s *quantileSketch[T]) appendValue(level int, value T) error { + s.ensureLevel(level) + values := s.levels[level] + if len(values) < cap(values) { + values = values[:len(values)+1] + values[len(values)-1] = value + s.levels[level] = values + return nil + } + return s.replaceLevel(level, len(values)+1, func(dst []T) { + copy(dst, values) + dst[len(values)] = value + }) +} + +func (s *quantileSketch[T]) appendValues(level int, added []T) error { + if len(added) == 0 { + return nil + } + s.ensureLevel(level) + values := s.levels[level] + needed := len(values) + len(added) + if needed <= cap(values) { + values = values[:needed] + copy(values[needed-len(added):], added) + s.levels[level] = values + return nil + } + return s.replaceLevel(level, needed, func(dst []T) { + copy(dst, values) + copy(dst[len(values):], added) + }) +} + +func (s *quantileSketch[T]) replaceLevel(level, length int, fill func([]T)) error { + old := s.levels[level] + capacity := max(1, cap(old)*2) + for capacity < length { + capacity *= 2 + } + values, err := mpool.MakeSlice[T](capacity, s.mp, true) + if err != nil { + return err + } + values = values[:length] + fill(values) + s.freeLevel(old) + s.levels[level] = values + return nil +} + +func (s *quantileSketch[T]) freeLevel(values []T) { + if cap(values) > 0 { + mpool.FreeSlice(s.mp, values[:1]) + } +} + +func (s *quantileSketch[T]) compactFrom(start int) error { + for level := start; level < len(s.levels); level++ { + for len(s.levels[level]) >= 2*approxPercentileSketchCapacity { + if level+1 >= approxPercentileMaxLevels { + return moerr.NewInvalidInputNoCtx("approx_percentile: sketch level overflow") + } + values := s.levels[level] + sort.Slice(values, func(i, j int) bool { + return s.compare(values[i], values[j]) < 0 + }) + + pickSecond := s.parity[level] + startAt, endAt := 0, len(values) + retained := 0 + var retainedValue T + if len(values)&1 == 1 { + if pickSecond { + retainedValue = values[0] + startAt = 1 + } else { + endAt-- + retainedValue = values[endAt] + } + retained = 1 + } + pick := 0 + if pickSecond { + pick = 1 + } + promoted := (endAt - startAt) / 2 + s.ensureLevel(level + 1) + next := s.levels[level+1] + needed := len(next) + promoted + if needed <= cap(next) { + next = next[:needed] + for i, dst := startAt, needed-promoted; i < endAt; i, dst = i+2, dst+1 { + next[dst] = values[i+pick] + } + s.levels[level+1] = next + } else if err := s.replaceLevel(level+1, needed, func(dst []T) { + copy(dst, next) + for i, out := startAt, len(next); i < endAt; i, out = i+2, out+1 { + dst[out] = values[i+pick] + } + }); err != nil { + return err + } + if retained == 1 { + values[0] = retainedValue + } + s.levels[level] = values[:retained] + s.parity[level] = !s.parity[level] + } + } + return nil +} + +type weightedQuantileValue[T quantileValue] struct { + value T + weight uint64 +} + +func (s *quantileSketch[T]) valueAtRank(rank uint64, sorted []weightedQuantileValue[T]) T { + var cumulative uint64 + for _, item := range sorted { + if rank < cumulative+item.weight { + return item.value + } + cumulative += item.weight + } + return sorted[len(sorted)-1].value +} + +func (s *quantileSketch[T]) Quantile(p *big.Rat) (lo, hi T, frac *big.Rat, err error) { + if s.count == 0 { + return lo, hi, nil, moerr.NewInternalErrorNoCtx("approx_percentile: empty sketch") + } + if p.Sign() == 0 { + return s.min, s.min, new(big.Rat), nil + } + if p.Cmp(big.NewRat(1, 1)) == 0 { + return s.max, s.max, new(big.Rat), nil + } + weighted, err := mpool.MakeSlice[weightedQuantileValue[T]](s.retained(), s.mp, true) + if err != nil { + return lo, hi, nil, err + } + defer mpool.FreeSlice(s.mp, weighted) + weighted = weighted[:0] + for level, values := range s.levels { + weight := uint64(1) << level + for _, value := range values { + weighted = append(weighted, weightedQuantileValue[T]{value: value, weight: weight}) + } + } + sort.Slice(weighted, func(i, j int) bool { + return s.compare(weighted[i].value, weighted[j].value) < 0 + }) + loRank, hiRank, frac := percentileRanks(s.count, p) + return s.valueAtRank(loRank, weighted), s.valueAtRank(hiRank, weighted), frac, nil +} + +func (s *quantileSketch[T]) retained() int { + n := 0 + for _, values := range s.levels { + n += len(values) + } + return n +} + +func (s *quantileSketch[T]) Size() int64 { + var zero T + valueSize := len(types.EncodeFixed(zero)) + capacity := 0 + for _, values := range s.levels { + capacity += cap(values) + } + return int64((capacity+2)*valueSize + cap(s.levels)*24 + cap(s.parity)) +} + +func (s *quantileSketch[T]) Free() { + for _, values := range s.levels { + s.freeLevel(values) + } + s.levels = nil + s.parity = nil + s.count = 0 + s.hasValue = false +} + +func (s *quantileSketch[T]) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + buf.WriteByte(approxPercentileSketchVersion) + if err := types.WriteUint64(&buf, s.count); err != nil { + return nil, err + } + if s.hasValue { + buf.WriteByte(1) + if _, err := buf.Write(types.EncodeFixed(s.min)); err != nil { + return nil, err + } + if _, err := buf.Write(types.EncodeFixed(s.max)); err != nil { + return nil, err + } + } else { + buf.WriteByte(0) + } + if len(s.levels) > approxPercentileMaxLevels { + return nil, moerr.NewInternalErrorNoCtx("approx_percentile: too many sketch levels") + } + if err := types.WriteUint16(&buf, uint16(len(s.levels))); err != nil { + return nil, err + } + for level, values := range s.levels { + if s.parity[level] { + buf.WriteByte(1) + } else { + buf.WriteByte(0) + } + if len(values) >= 2*approxPercentileSketchCapacity { + return nil, moerr.NewInternalErrorNoCtx("approx_percentile: uncompacted sketch") + } + if err := types.WriteUint16(&buf, uint16(len(values))); err != nil { + return nil, err + } + for _, value := range values { + if _, err := buf.Write(types.EncodeFixed(value)); err != nil { + return nil, err + } + } + } + return buf.Bytes(), nil +} + +func (s *quantileSketch[T]) UnmarshalBinary(data []byte) error { + reader := bytes.NewReader(data) + restored, err := s.decode(reader) + if err != nil { + return err + } + if reader.Len() != 0 { + restored.Free() + return moerr.NewInvalidInputNoCtx("approx_percentile: inconsistent sketch state") + } + s.restore(restored) + return nil +} + +func (s *quantileSketch[T]) decode(reader io.Reader) (_ *quantileSketch[T], retErr error) { + restored := newQuantileSketch(s.mp, s.compare) + defer func() { + if retErr != nil { + restored.Free() + } + }() + readByte := func() (byte, error) { + var encoded [1]byte + _, err := io.ReadFull(reader, encoded[:]) + return encoded[0], err + } + version, err := readByte() + if err != nil { + return nil, err + } + if version != approxPercentileSketchVersion { + return nil, moerr.NewInvalidInputNoCtxf("approx_percentile: unsupported sketch version %d", version) + } + count, err := types.ReadUint64(reader) + if err != nil { + return nil, err + } + var zero T + valueSize := len(types.EncodeFixed(zero)) + hasValue, err := readByte() + if err != nil || hasValue > 1 { + return nil, moerr.NewInvalidInputNoCtx("approx_percentile: invalid sketch extrema flag") + } + encoded := make([]byte, valueSize) + if hasValue == 1 { + if _, err := io.ReadFull(reader, encoded); err != nil { + return nil, err + } + restored.min = types.DecodeFixed[T](encoded) + if _, err := io.ReadFull(reader, encoded); err != nil { + return nil, err + } + restored.max = types.DecodeFixed[T](encoded) + restored.hasValue = true + } + levelCount, err := types.ReadUint16(reader) + if err != nil { + return nil, err + } + if levelCount > approxPercentileMaxLevels { + return nil, moerr.NewInvalidInputNoCtx("approx_percentile: invalid sketch level count") + } + restored.levels = make([][]T, int(levelCount)) + restored.parity = make([]bool, int(levelCount)) + var represented uint64 + for level := range int(levelCount) { + parity, err := readByte() + if err != nil || parity > 1 { + return nil, moerr.NewInvalidInputNoCtx("approx_percentile: invalid sketch parity") + } + restored.parity[level] = parity == 1 + length, err := types.ReadUint16(reader) + if err != nil { + return nil, err + } + if length >= 2*approxPercentileSketchCapacity { + return nil, moerr.NewInvalidInputNoCtx("approx_percentile: invalid sketch level size") + } + if length > 0 { + values, err := mpool.MakeSlice[T](int(length), restored.mp, true) + if err != nil { + return nil, err + } + restored.levels[level] = values + for i := range values { + if _, err := io.ReadFull(reader, encoded); err != nil { + return nil, err + } + values[i] = types.DecodeFixed[T](encoded) + } + } + weight := uint64(1) << level + if uint64(length) > math.MaxUint64/weight || math.MaxUint64-represented < uint64(length)*weight { + return nil, moerr.NewInvalidInputNoCtx("approx_percentile: invalid sketch weight") + } + represented += uint64(length) * weight + } + if represented != count || restored.hasValue != (count > 0) || + restored.hasValue && restored.compare(restored.min, restored.max) > 0 { + return nil, moerr.NewInvalidInputNoCtx("approx_percentile: inconsistent sketch state") + } + restored.count = count + return restored, nil +} + +func (s *quantileSketch[T]) UnmarshalFromReader(reader io.Reader) error { + restored, err := s.decode(reader) + if err != nil { + return err + } + s.restore(restored) + return nil +} + +func (s *quantileSketch[T]) restore(restored *quantileSketch[T]) { + s.Free() + *s = *restored + restored.levels = nil +} + +func percentileRanks(count uint64, p *big.Rat) (lo, hi uint64, frac *big.Rat) { + if count <= 1 { + return 0, 0, new(big.Rat) + } + rank := new(big.Rat).Mul(p, new(big.Rat).SetInt(new(big.Int).SetUint64(count-1))) + loInt := new(big.Int).Quo(rank.Num(), rank.Denom()) + lo = loInt.Uint64() + hi = lo + if lo < count-1 { + hi++ + } + rem := new(big.Int).Mod(new(big.Int).Set(rank.Num()), rank.Denom()) + return lo, hi, new(big.Rat).SetFrac(rem, new(big.Int).Set(rank.Denom())) +} + +func parsePercentileConfig(partialResult any) (*big.Rat, float64, error) { + b, ok := partialResult.([]byte) + if !ok { + return nil, 0, moerr.NewInternalErrorNoCtx("approx_percentile: expected []byte config") + } + text := string(b) + p, err := strconv.ParseFloat(text, 64) + if err != nil { + return nil, 0, err + } + if math.IsNaN(p) || math.IsInf(p, 0) || p < 0 || p > 1 { + return nil, 0, moerr.NewInvalidInputNoCtxf( + "approx_percentile: percentile must be in [0,1] and finite, got %v", p) + } + rat, ok := new(big.Rat).SetString(text) + if !ok || rat.Sign() < 0 || rat.Cmp(big.NewRat(1, 1)) > 0 { + return nil, 0, moerr.NewInvalidInputNoCtxf("approx_percentile: invalid percentile %q", text) + } + return rat, p, nil +} + +func orderedCompare[T numeric](a, b T) int { + af, bf := float64(a), float64(b) + if math.IsNaN(af) { + if math.IsNaN(bf) { + return 0 + } + return -1 + } + if math.IsNaN(bf) { + return 1 + } + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +type approxPercentileExecBase[T quantileValue] struct { + aggExec + percentile *big.Rat + percentileFloat float64 + compare func(T, T) int +} + +func newApproxPercentileExecBase[T quantileValue](mp *mpool.MPool, info singleAggInfo, compare func(T, T) int) approxPercentileExecBase[T] { + exec := approxPercentileExecBase[T]{compare: compare} + exec.mp = mp + exec.aggInfo = aggInfo{ + aggId: info.aggID, + isDistinct: info.distinct, + argTypes: []types.Type{info.argType}, + retType: info.retType, + emptyNull: true, + saveArg: false, + makeMarshalerUnmarshaler: func(mp *mpool.MPool) (MarshalerUnmarshaler, error) { + return newQuantileSketch(mp, compare), nil + }, + } + return exec +} + +func (exec *approxPercentileExecBase[T]) GroupGrow(more int) error { + start := exec.GetNumGroups() + if err := exec.aggExec.GroupGrow(more); err != nil { + return err + } + for group := start; group < start+more; group++ { + if _, err := exec.ensureSketch(uint64(group)); err != nil { + return err + } + } + return nil +} + +func (exec *approxPercentileExecBase[T]) ensureSketch(group uint64) (*quantileSketch[T], error) { + x, y := exec.getXY(group) + if exec.state[x].mobs[y] == nil { + mob, err := exec.makeMarshalerUnmarshaler(exec.mp) + if err != nil { + return nil, err + } + exec.state[x].mobs[y] = mob + } + return exec.state[x].mobs[y].(*quantileSketch[T]), nil +} + +func (exec *approxPercentileExecBase[T]) Fill(groupIndex int, row int, vectors []*vector.Vector) error { + if vectors[0].IsNull(uint64(row)) { + return nil + } + if vectors[0].IsConst() { + row = 0 + } + sketch, err := exec.ensureSketch(uint64(groupIndex)) + if err != nil { + return err + } + return sketch.Add(vector.MustFixedColWithTypeCheck[T](vectors[0])[row]) +} + +func (exec *approxPercentileExecBase[T]) BulkFill(groupIndex int, vectors []*vector.Vector) error { + if vectors[0].IsConstNull() { + return nil + } + sketch, err := exec.ensureSketch(uint64(groupIndex)) + if err != nil { + return err + } + values := vector.MustFixedColWithTypeCheck[T](vectors[0]) + if vectors[0].IsConst() { + for range vectors[0].Length() { + if err := sketch.Add(values[0]); err != nil { + return err + } + } + return nil + } + for row, value := range values { + if vectors[0].IsNull(uint64(row)) { + continue + } + if err := sketch.Add(value); err != nil { + return err + } + } + return nil +} + +func (exec *approxPercentileExecBase[T]) BatchFill(offset int, groups []uint64, vectors []*vector.Vector) error { + if vectors[0].IsConstNull() { + return nil + } + values := vector.MustFixedColWithTypeCheck[T](vectors[0]) + for i, group := range groups { + if group == GroupNotMatched { + continue + } + row := offset + i + if vectors[0].IsConst() { + row = 0 + } + if vectors[0].IsNull(uint64(row)) { + continue + } + sketch, err := exec.ensureSketch(group - 1) + if err != nil { + return err + } + if err := sketch.Add(values[row]); err != nil { + return err + } + } + return nil +} + +func (exec *approxPercentileExecBase[T]) merge(other *approxPercentileExecBase[T], groupIdx1, groupIdx2 int) error { + if exec.percentile != nil && other.percentile != nil && exec.percentile.Cmp(other.percentile) != 0 { + return moerr.NewInvalidInputNoCtx("approx_percentile: cannot merge different percentile configurations") + } + x2, y2 := other.getXY(uint64(groupIdx2)) + if other.state[x2].mobs[y2] == nil { + return nil + } + target, err := exec.ensureSketch(uint64(groupIdx1)) + if err != nil { + return err + } + return target.Merge(other.state[x2].mobs[y2].(*quantileSketch[T])) +} + +func (exec *approxPercentileExecBase[T]) batchMerge(other *approxPercentileExecBase[T], offset int, groups []uint64) error { + for i, group := range groups { + if group == GroupNotMatched { + continue + } + if err := exec.merge(other, int(group-1), offset+i); err != nil { + return err + } + } + return nil +} + +func (exec *approxPercentileExecBase[T]) SetExtraInformation(partialResult any, groupIndex int) error { + percentile, percentileFloat, err := parsePercentileConfig(partialResult) + if err != nil { + return err + } + exec.percentile = percentile + exec.percentileFloat = percentileFloat + return nil +} + +func (exec *approxPercentileExecBase[T]) Size() int64 { + var size int64 + for _, state := range exec.state { + size += int64(cap(state.mobs)) * 8 + for _, mob := range state.mobs { + if mob != nil { + size += mob.(*quantileSketch[T]).Size() + } + } + } + return size +} + +func (exec *approxPercentileExecBase[T]) Free() { + exec.aggExec.Free() + exec.state = nil +} + +type approxPercentileNumericExec[T numeric] struct { + approxPercentileExecBase[T] +} + +func (exec *approxPercentileNumericExec[T]) Merge(next AggFuncExec, groupIdx1, groupIdx2 int) error { + return exec.merge(&next.(*approxPercentileNumericExec[T]).approxPercentileExecBase, groupIdx1, groupIdx2) +} + +func (exec *approxPercentileNumericExec[T]) BatchMerge(next AggFuncExec, offset int, groups []uint64) error { + return exec.batchMerge(&next.(*approxPercentileNumericExec[T]).approxPercentileExecBase, offset, groups) +} + +func (exec *approxPercentileNumericExec[T]) Flush() (_ []*vector.Vector, retErr error) { + if exec.percentile == nil { + return nil, moerr.NewInternalErrorNoCtx("approx_percentile: percentile configuration is not set") + } + results := make([]*vector.Vector, len(exec.state)) + defer func() { + if retErr != nil { + for _, result := range results { + if result != nil { + result.Free(exec.mp) + } + } + } + }() + for x, state := range exec.state { + result := vector.NewOffHeapVecWithType(exec.retType) + results[x] = result + if err := result.PreExtend(int(state.length), exec.mp); err != nil { + return nil, err + } + result.SetLength(int(state.length)) + values := vector.MustFixedColNoTypeCheck[float64](result) + for y := 0; y < int(state.length); y++ { + if state.mobs[y] == nil || state.mobs[y].(*quantileSketch[T]).count == 0 { + result.SetNull(uint64(y)) + continue + } + lo, hi, frac, err := state.mobs[y].(*quantileSketch[T]).Quantile(exec.percentile) + if err != nil { + return nil, err + } + values[y] = interpolateNumeric(lo, hi, frac) + } + } + return results, nil +} + +type approxPercentileDecimalExec[T types.Decimal64 | types.Decimal128] struct { + approxPercentileExecBase[T] +} + +func (exec *approxPercentileDecimalExec[T]) Merge(next AggFuncExec, groupIdx1, groupIdx2 int) error { + return exec.merge(&next.(*approxPercentileDecimalExec[T]).approxPercentileExecBase, groupIdx1, groupIdx2) +} + +func (exec *approxPercentileDecimalExec[T]) BatchMerge(next AggFuncExec, offset int, groups []uint64) error { + return exec.batchMerge(&next.(*approxPercentileDecimalExec[T]).approxPercentileExecBase, offset, groups) +} + +func (exec *approxPercentileDecimalExec[T]) Flush() (_ []*vector.Vector, retErr error) { + if exec.percentile == nil { + return nil, moerr.NewInternalErrorNoCtx("approx_percentile: percentile configuration is not set") + } + results := make([]*vector.Vector, len(exec.state)) + defer func() { + if retErr != nil { + for _, result := range results { + if result != nil { + result.Free(exec.mp) + } + } + } + }() + for x, state := range exec.state { + result := vector.NewOffHeapVecWithType(exec.retType) + results[x] = result + if err := result.PreExtend(int(state.length), exec.mp); err != nil { + return nil, err + } + result.SetLength(int(state.length)) + values := vector.MustFixedColNoTypeCheck[types.Decimal128](result) + for y := 0; y < int(state.length); y++ { + if state.mobs[y] == nil || state.mobs[y].(*quantileSketch[T]).count == 0 { + result.SetNull(uint64(y)) + continue + } + lo, hi, frac, err := state.mobs[y].(*quantileSketch[T]).Quantile(exec.percentile) + if err != nil { + return nil, err + } + values[y], err = interpolateDecimal( + toDecimal128(lo), toDecimal128(hi), frac, exec.retType.Scale-exec.argTypes[0].Scale, + ) + if err != nil { + return nil, err + } + } + } + return results, nil +} + +func toDecimal128[T types.Decimal64 | types.Decimal128](value T) types.Decimal128 { + switch value := any(value).(type) { + case types.Decimal64: + return FromD64ToD128(value) + case types.Decimal128: + return value + default: + panic("unreachable") + } +} + +func decimal128ToBigInt(value types.Decimal128) *big.Int { + result := new(big.Int).SetUint64(value.B64_127) + result.Lsh(result, 64) + result.Or(result, new(big.Int).SetUint64(value.B0_63)) + if value.Sign() { + result.Sub(result, new(big.Int).Lsh(big.NewInt(1), 128)) + } + return result +} + +func decimal128FromBigInt(value *big.Int) (types.Decimal128, error) { + limit := new(big.Int).Lsh(big.NewInt(1), 127) + if value.Cmp(new(big.Int).Neg(new(big.Int).Set(limit))) < 0 || value.Cmp(new(big.Int).Sub(new(big.Int).Set(limit), big.NewInt(1))) > 0 { + return types.Decimal128{}, moerr.NewInvalidInputNoCtx("approx_percentile: decimal interpolation overflow") + } + unsigned := new(big.Int).Set(value) + if unsigned.Sign() < 0 { + unsigned.Add(unsigned, new(big.Int).Lsh(big.NewInt(1), 128)) + } + lowMask := new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 64), big.NewInt(1)) + low := new(big.Int).And(new(big.Int).Set(unsigned), lowMask).Uint64() + high := new(big.Int).Rsh(unsigned, 64).Uint64() + return types.Decimal128{B0_63: low, B64_127: high}, nil +} + +// interpolateDecimal converts from the input scale to the declared result +// scale. All arithmetic stays integral/rational, so values above 2^53 do not +// pass through float64. +func interpolateDecimal(lo, hi types.Decimal128, frac *big.Rat, scaleDelta int32) (types.Decimal128, error) { + if scaleDelta < 0 || scaleDelta > 1 { + return types.Decimal128{}, moerr.NewInternalErrorNoCtx("approx_percentile: invalid decimal result scale") + } + den := new(big.Int).Set(frac.Denom()) + rem := new(big.Int).Set(frac.Num()) + loInt := decimal128ToBigInt(lo) + diff := new(big.Int).Sub(decimal128ToBigInt(hi), loInt) + numerator := new(big.Int).Mul(loInt, den) + numerator.Add(numerator, new(big.Int).Mul(diff, rem)) + if scaleDelta == 1 { + numerator.Mul(numerator, big.NewInt(10)) + } + quotient, remainder := new(big.Int), new(big.Int) + quotient.QuoRem(numerator, den, remainder) + if new(big.Int).Lsh(new(big.Int).Abs(remainder), 1).Cmp(den) >= 0 { + if numerator.Sign() < 0 { + quotient.Sub(quotient, big.NewInt(1)) + } else { + quotient.Add(quotient, big.NewInt(1)) + } + } + return decimal128FromBigInt(quotient) +} + +func newApproxPercentileExec(mp *mpool.MPool, info singleAggInfo) (AggFuncExec, error) { + if info.distinct { + return nil, moerr.NewNotSupportedNoCtx("approx_percentile in distinct mode") + } + switch info.argType.Oid { + case types.T_bit: + return &approxPercentileNumericExec[uint64]{newApproxPercentileExecBase[uint64](mp, info, orderedCompare[uint64])}, nil + case types.T_int8: + return &approxPercentileNumericExec[int8]{newApproxPercentileExecBase[int8](mp, info, orderedCompare[int8])}, nil + case types.T_int16: + return &approxPercentileNumericExec[int16]{newApproxPercentileExecBase[int16](mp, info, orderedCompare[int16])}, nil + case types.T_int32: + return &approxPercentileNumericExec[int32]{newApproxPercentileExecBase[int32](mp, info, orderedCompare[int32])}, nil + case types.T_int64: + return &approxPercentileNumericExec[int64]{newApproxPercentileExecBase[int64](mp, info, orderedCompare[int64])}, nil + case types.T_uint8: + return &approxPercentileNumericExec[uint8]{newApproxPercentileExecBase[uint8](mp, info, orderedCompare[uint8])}, nil + case types.T_uint16: + return &approxPercentileNumericExec[uint16]{newApproxPercentileExecBase[uint16](mp, info, orderedCompare[uint16])}, nil + case types.T_uint32: + return &approxPercentileNumericExec[uint32]{newApproxPercentileExecBase[uint32](mp, info, orderedCompare[uint32])}, nil + case types.T_uint64: + return &approxPercentileNumericExec[uint64]{newApproxPercentileExecBase[uint64](mp, info, orderedCompare[uint64])}, nil + case types.T_float32: + return &approxPercentileNumericExec[float32]{newApproxPercentileExecBase[float32](mp, info, orderedCompare[float32])}, nil + case types.T_float64: + return &approxPercentileNumericExec[float64]{newApproxPercentileExecBase[float64](mp, info, orderedCompare[float64])}, nil + case types.T_decimal64: + compare := func(a, b types.Decimal64) int { return a.Compare(b) } + return &approxPercentileDecimalExec[types.Decimal64]{newApproxPercentileExecBase[types.Decimal64](mp, info, compare)}, nil + case types.T_decimal128: + compare := func(a, b types.Decimal128) int { return a.Compare(b) } + return &approxPercentileDecimalExec[types.Decimal128]{newApproxPercentileExecBase[types.Decimal128](mp, info, compare)}, nil + default: + return nil, moerr.NewInternalErrorNoCtx("unsupported type for approx_percentile()") + } +} + +// Exact helpers retained for direct callers and small-data regression tests. +func PercentileNumeric[T numeric](vs *Vectors[T], p float64) (float64, error) { + return percentileNumericVals(collectMedianValues(vs), p), nil +} + +func interpolateFloat64(lo, hi, fraction float64) float64 { + if fraction == 0 { + return lo + } + if fraction == 1 { + return hi + } + if lo == hi { + return lo + } + // Preserve IEEE-754 propagation when distinct non-finite values actually + // need interpolation. + if math.IsNaN(lo) || math.IsNaN(hi) || math.IsInf(lo, 0) || math.IsInf(hi, 0) { + return lo + (hi-lo)*fraction + } + // For opposite signs, hi-lo can overflow even though the interpolated value + // is finite. Both weighted terms are bounded by their finite endpoints, and + // their opposite signs keep the final addition from overflowing. + if math.Signbit(lo) != math.Signbit(hi) { + return math.FMA(lo, 1-fraction, hi*fraction) + } + return lo + (hi-lo)*fraction +} + +func numericIntegerToBigInt[T numeric](value T) (*big.Int, bool) { + switch value := any(value).(type) { + case int8: + return big.NewInt(int64(value)), true + case int16: + return big.NewInt(int64(value)), true + case int32: + return big.NewInt(int64(value)), true + case int64: + return big.NewInt(value), true + case uint8: + return new(big.Int).SetUint64(uint64(value)), true + case uint16: + return new(big.Int).SetUint64(uint64(value)), true + case uint32: + return new(big.Int).SetUint64(uint64(value)), true + case uint64: + return new(big.Int).SetUint64(value), true + default: + return nil, false + } +} + +func interpolateNumeric[T numeric](lo, hi T, fraction *big.Rat) float64 { + loInt, integer := numericIntegerToBigInt(lo) + if !integer { + fractionFloat64, _ := fraction.Float64() + return interpolateFloat64(float64(lo), float64(hi), fractionFloat64) + } + if fraction.Sign() == 0 { + return float64(lo) + } + + hiInt, _ := numericIntegerToBigInt(hi) + denominator := new(big.Int).Set(fraction.Denom()) + numerator := new(big.Int).Mul(loInt, denominator) + difference := new(big.Int).Sub(hiInt, loInt) + numerator.Add(numerator, new(big.Int).Mul(difference, fraction.Num())) + result, _ := new(big.Rat).SetFrac(numerator, denominator).Float64() + return result +} + +func percentileNumericVals[T numeric](values []T, p float64) float64 { + if len(values) == 0 || p < 0 || p > 1 { + return math.NaN() + } + rat, _, err := parsePercentileConfig([]byte(strconv.FormatFloat(p, 'g', -1, 64))) + if err != nil { + return math.NaN() + } + loRank, hiRank, frac := percentileRanks(uint64(len(values)), rat) + lo := selectKthNumeric(values, int(loRank)) + hi := selectKthNumeric(values, int(hiRank)) + return interpolateNumeric(lo, hi, frac) +} + +func PercentileDecimal64(vs *Vectors[types.Decimal64], p float64, argScale int32) (types.Decimal128, error) { + return percentileDecimal64Vals(collectMedianValues(vs), p, argScale) +} + +func percentileDecimal64Vals(values []types.Decimal64, p float64, argScale int32) (types.Decimal128, error) { + if len(values) == 0 || p < 0 || p > 1 { + return types.Decimal128{}, nil + } + rat, _, err := parsePercentileConfig([]byte(strconv.FormatFloat(p, 'g', -1, 64))) + if err != nil { + return types.Decimal128{}, err + } + loRank, hiRank, frac := percentileRanks(uint64(len(values)), rat) + compare := func(a, b types.Decimal64) int { return a.Compare(b) } + lo := FromD64ToD128(selectKthFunc(values, int(loRank), compare)) + hi := FromD64ToD128(selectKthFunc(values, int(hiRank), compare)) + return interpolateDecimal(lo, hi, frac, 1) +} + +func PercentileDecimal128(vs *Vectors[types.Decimal128], p float64, argScale int32) (types.Decimal128, error) { + argWidth := approxPercentileDecimalWidth + if len(vs.vecs) > 0 { + argWidth = vs.vecs[0].GetType().Width + } + return percentileDecimal128Vals(collectMedianValues(vs), p, argWidth, argScale) +} + +func percentileDecimal128Vals(values []types.Decimal128, p float64, argWidth, argScale int32) (types.Decimal128, error) { + if len(values) == 0 || p < 0 || p > 1 { + return types.Decimal128{}, nil + } + rat, _, err := parsePercentileConfig([]byte(strconv.FormatFloat(p, 'g', -1, 64))) + if err != nil { + return types.Decimal128{}, err + } + loRank, hiRank, frac := percentileRanks(uint64(len(values)), rat) + compare := func(a, b types.Decimal128) int { return a.Compare(b) } + lo := selectKthFunc(values, int(loRank), compare) + hi := selectKthFunc(values, int(hiRank), compare) + argType := types.New(types.T_decimal128, argWidth, argScale) + resultType := ApproxPercentileReturnType([]types.Type{argType}) + return interpolateDecimal(lo, hi, frac, resultType.Scale-argScale) +} diff --git a/pkg/sql/colexec/aggexec/approx_percentile_test.go b/pkg/sql/colexec/aggexec/approx_percentile_test.go new file mode 100644 index 0000000000000..3f17e10c8035c --- /dev/null +++ b/pkg/sql/colexec/aggexec/approx_percentile_test.go @@ -0,0 +1,1353 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aggexec + +import ( + "bytes" + "math" + "math/big" + "strconv" + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/require" +) + +// --- percentile computation algorithm tests --- + +func TestPercentileNumericVals_Basic(t *testing.T) { + // N=10 values 1..10 + vals := []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + + // p=0.0 returns first element + require.Equal(t, float64(1), percentileNumericVals(vals, 0.0)) + // p=0.5 returns 5.5 (median) + require.Equal(t, 5.5, percentileNumericVals(vals, 0.5)) + // p=0.95 returns 9.55 + require.InDelta(t, 9.55, percentileNumericVals(vals, 0.95), 1e-10) + // p=0.99 returns 9.91 + require.InDelta(t, 9.91, percentileNumericVals(vals, 0.99), 1e-10) + // p=1.0 returns last element + require.Equal(t, float64(10), percentileNumericVals(vals, 1.0)) +} + +func TestPercentileNumericVals_EvenN(t *testing.T) { + vals := []float64{1.0, 2.0, 4.0, 5.0} + + // p=0.0 -> 1.0 (min) + require.Equal(t, float64(1), percentileNumericVals(vals, 0.0)) + // p=0.5 -> 3.0 (interpolation: lo=1, hi=2, idx=1.5, vLo=2, vHi=4) + require.InDelta(t, 3.0, percentileNumericVals(vals, 0.5), 1e-10) + // p=0.25 -> idx=0.75, lo=0(1.0), hi=1(2.0) => 1.0 + 0.75*(2.0-1.0) = 1.75 + require.InDelta(t, 1.75, percentileNumericVals(vals, 0.25), 1e-10) + // p=0.75 -> idx=2.25, lo=2(4.0), hi=3(5.0) => 4.0 + 0.25*(5.0-4.0) = 4.25 + require.InDelta(t, 4.25, percentileNumericVals(vals, 0.75), 1e-10) + // p=1.0 -> 5.0 (max) + require.Equal(t, float64(5), percentileNumericVals(vals, 1.0)) +} + +func TestPercentileNumericVals_SingleValue(t *testing.T) { + vals := []int64{42} + + require.Equal(t, float64(42), percentileNumericVals(vals, 0.0)) + require.Equal(t, float64(42), percentileNumericVals(vals, 0.5)) + require.Equal(t, float64(42), percentileNumericVals(vals, 1.0)) +} + +func TestPercentileNumericVals_TwoValues(t *testing.T) { + vals := []int64{10, 20} + + require.Equal(t, float64(10), percentileNumericVals(vals, 0.0)) + require.Equal(t, float64(15), percentileNumericVals(vals, 0.5)) + require.Equal(t, float64(20), percentileNumericVals(vals, 1.0)) +} + +func TestPercentileNumericVals_EdgeCases(t *testing.T) { + tests := []struct { + name string + vals []int64 + p float64 + want float64 + }{ + // Empty - returns NaN + {name: "empty_p05", vals: nil, p: 0.5, want: math.NaN()}, + {name: "empty_p00", vals: []int64{}, p: 0.0, want: math.NaN()}, + // p < 0 - returns NaN + {name: "negative_p", vals: []int64{1, 2, 3}, p: -0.1, want: math.NaN()}, + // p > 1 - returns NaN + {name: "above_one_p", vals: []int64{1, 2, 3}, p: 1.1, want: math.NaN()}, + // N=3, p=0.0 -> min + {name: "three_min", vals: []int64{3, 1, 2}, p: 0.0, want: 1}, + // N=3, p=0.5 -> median + {name: "three_median", vals: []int64{3, 1, 2}, p: 0.5, want: 2}, + // N=3, p=1.0 -> max + {name: "three_max", vals: []int64{3, 1, 2}, p: 1.0, want: 3}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := percentileNumericVals(tt.vals, tt.p) + if math.IsNaN(tt.want) { + require.True(t, math.IsNaN(result)) + } else { + require.Equal(t, tt.want, result) + } + }) + } +} + +func TestPercentileNumericVals_Int64Overflow(t *testing.T) { + vals := []int64{math.MaxInt64, math.MaxInt64} + require.Equal(t, float64(math.MaxInt64), percentileNumericVals(vals, 0.5)) +} + +func TestPercentileNumericVals_Int64Precision(t *testing.T) { + require.Equal(t, -0.5, percentileNumericVals([]int64{-9007199254740993, 9007199254740992}, 0.5)) + require.Equal(t, -0.5, percentileNumericVals([]int64{math.MinInt64, math.MaxInt64}, 0.5)) + require.Equal(t, float64(9007199254740994), percentileNumericVals([]uint64{9007199254740993, 9007199254740994}, 0.5)) +} + +func TestPercentileNumericVals_Float64ExtremeInterpolation(t *testing.T) { + vals := []float64{-math.MaxFloat64, math.MaxFloat64} + require.Equal(t, 0.0, percentileNumericVals(vals, 0.5)) + require.Equal(t, -math.MaxFloat64, percentileNumericVals(vals, 0)) + require.Equal(t, math.MaxFloat64, percentileNumericVals(vals, 1)) +} + +func TestInterpolateFloat64(t *testing.T) { + tests := []struct { + name string + lo, hi, frac float64 + want float64 + }{ + {name: "opposite extreme midpoint", lo: -math.MaxFloat64, hi: math.MaxFloat64, frac: 0.5, want: 0}, + {name: "opposite extreme quarter", lo: -math.MaxFloat64, hi: math.MaxFloat64, frac: 0.25, want: -math.MaxFloat64 / 2}, + {name: "positive same sign", lo: math.MaxFloat64 / 2, hi: math.MaxFloat64, frac: 0.5, want: math.MaxFloat64 * 0.75}, + {name: "negative same sign", lo: -math.MaxFloat64, hi: -math.MaxFloat64 / 2, frac: 0.5, want: -math.MaxFloat64 * 0.75}, + {name: "subnormal midpoint", lo: -math.SmallestNonzeroFloat64, hi: math.SmallestNonzeroFloat64, frac: 0.5, want: 0}, + {name: "left endpoint", lo: -math.MaxFloat64, hi: math.MaxFloat64, frac: 0, want: -math.MaxFloat64}, + {name: "right endpoint", lo: -math.MaxFloat64, hi: math.MaxFloat64, frac: 1, want: math.MaxFloat64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := interpolateFloat64(test.lo, test.hi, test.frac) + if test.want == 0 || test.frac == 0 || test.frac == 1 { + require.Equal(t, test.want, got) + } else { + require.InEpsilon(t, test.want, got, 1e-15) + } + }) + } + + // Keep IEEE-754 propagation for non-finite interpolations, but exact ranks + // must still return their selected endpoint. + require.True(t, math.IsNaN(interpolateFloat64(math.NaN(), 1, 0.5))) + require.True(t, math.IsNaN(interpolateFloat64(math.Inf(-1), math.Inf(1), 0.5))) + for _, test := range []struct { + name string + lo, hi, frac float64 + positive bool + }{ + {name: "positive infinity left endpoint", lo: math.Inf(1), hi: math.Inf(1), frac: 0, positive: true}, + {name: "positive infinity equal endpoints", lo: math.Inf(1), hi: math.Inf(1), frac: 0.5, positive: true}, + {name: "positive infinity right endpoint", lo: math.Inf(1), hi: math.Inf(1), frac: 1, positive: true}, + {name: "negative infinity left endpoint", lo: math.Inf(-1), hi: math.Inf(-1), frac: 0}, + {name: "negative infinity equal endpoints", lo: math.Inf(-1), hi: math.Inf(-1), frac: 0.5}, + {name: "negative infinity right endpoint", lo: math.Inf(-1), hi: math.Inf(-1), frac: 1}, + } { + t.Run(test.name, func(t *testing.T) { + wantInf := -1 + if test.positive { + wantInf = 1 + } + require.True(t, math.IsInf(interpolateFloat64(test.lo, test.hi, test.frac), wantInf)) + }) + } +} + +func TestPercentileDecimal64Vals(t *testing.T) { + vals := mustDecimal64s(t, "1.00", "2.00", "3.00", "4.00", "5.00", "6.00", "7.00", "8.00", "9.00", "10.00") + + tests := []struct { + name string + p float64 + want string + }{ + {name: "p00", p: 0.0, want: "1.000"}, + {name: "p50", p: 0.5, want: "5.500"}, + {name: "p95", p: 0.95, want: "9.550"}, + {name: "p100", p: 1.0, want: "10.000"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d128, err := percentileDecimal64Vals(vals, tt.p, 2) + require.NoError(t, err) + require.Equal(t, tt.want, d128.Format(3)) + }) + } +} + +func TestPercentileDecimal64Vals_EdgeCases(t *testing.T) { + // Empty + d128, err := percentileDecimal64Vals(nil, 0.5, 2) + require.NoError(t, err) + require.Equal(t, types.Decimal128{}, d128) + + d128, err = percentileDecimal64Vals(nil, 1.5, 2) + require.NoError(t, err) + require.Equal(t, types.Decimal128{}, d128) + + // Single value + vals := mustDecimal64s(t, "42.00") + d128, err = percentileDecimal64Vals(vals, 0.5, 2) + require.NoError(t, err) + require.Equal(t, "42.000", d128.Format(3)) + + // Two values + vals2 := mustDecimal64s(t, "10.00", "20.00") + d128, err = percentileDecimal64Vals(vals2, 0.5, 2) + require.NoError(t, err) + require.Equal(t, "15.000", d128.Format(3)) + + // Negative values + valsNeg := mustDecimal64s(t, "-5.00", "5.00", "-3.00", "3.00") + d128, err = percentileDecimal64Vals(valsNeg, 0.5, 2) + require.NoError(t, err) + require.Equal(t, "0.000", d128.Format(3)) +} + +func TestPercentileDecimal128Vals(t *testing.T) { + vals := mustDecimal128s(t, "1.00", "2.00", "3.00", "4.00", "5.00", "6.00", "7.00", "8.00", "9.00", "10.00") + + tests := []struct { + name string + p float64 + want string + }{ + {name: "p00", p: 0.0, want: "1.000"}, + {name: "p50", p: 0.5, want: "5.500"}, + {name: "p95", p: 0.95, want: "9.550"}, + {name: "p100", p: 1.0, want: "10.000"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d128, err := percentileDecimal128Vals(vals, tt.p, 20, 2) + require.NoError(t, err) + require.Equal(t, tt.want, d128.Format(3)) + }) + } +} + +func TestPercentileDecimal128Vals_EdgeCases(t *testing.T) { + // Empty + d128, err := percentileDecimal128Vals(nil, 0.5, 20, 2) + require.NoError(t, err) + require.Equal(t, types.Decimal128{}, d128) + + // Single value + vals := mustDecimal128s(t, "99.99") + d128, err = percentileDecimal128Vals(vals, 0.5, 20, 2) + require.NoError(t, err) + require.Equal(t, "99.990", d128.Format(3)) + + // Two values + vals2 := mustDecimal128s(t, "10.00", "20.00") + d128, err = percentileDecimal128Vals(vals2, 0.75, 20, 2) + require.NoError(t, err) + require.Equal(t, "17.500", d128.Format(3)) +} + +func TestPercentileDecimal128MaxPrecisionEndpoints(t *testing.T) { + mp := mpool.MustNewZero() + values := NewVectors[types.Decimal128](types.New(types.T_decimal128, 38, 0)) + t.Cleanup(func() { + values.Free(mp) + require.Equal(t, int64(0), mp.CurrNB()) + }) + + maxValue, err := types.ParseDecimal128("99999999999999999999999999999999999999", 38, 0) + require.NoError(t, err) + minValue, err := types.ParseDecimal128("-99999999999999999999999999999999999999", 38, 0) + require.NoError(t, err) + require.NoError(t, vector.AppendFixedList(values.vecs[0], []types.Decimal128{minValue, maxValue}, nil, mp)) + + got, err := PercentileDecimal128(values, 0, 0) + require.NoError(t, err) + require.Equal(t, "-99999999999999999999999999999999999999", got.Format(0)) + got, err = PercentileDecimal128(values, 1, 0) + require.NoError(t, err) + require.Equal(t, "99999999999999999999999999999999999999", got.Format(0)) +} + +func TestPercentileDecimal128ScaleBoundaries(t *testing.T) { + tests := []struct { + name string + typ types.Type + values []string + p float64 + want string + outScale int32 + }{ + { + name: "width 37 adds one result scale digit", + typ: types.New(types.T_decimal128, 37, 2), + values: []string{"1.00", "2.00"}, + p: 0.5, + want: "1.500", + outScale: 3, + }, + { + name: "width 38 retains maximum scale", + typ: types.New(types.T_decimal128, 38, 38), + values: []string{"0.1", "0.2"}, + p: 0.5, + want: "0.15000000000000000000000000000000000000", + outScale: 38, + }, + { + name: "width 38 retained scale rounds midpoint", + typ: types.New(types.T_decimal128, 38, 0), + values: []string{"-1", "0"}, + p: 0.5, + want: "-1", + outScale: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mp := mpool.MustNewZero() + values := NewVectors[types.Decimal128](tt.typ) + t.Cleanup(func() { + values.Free(mp) + require.Equal(t, int64(0), mp.CurrNB()) + }) + + parsed := make([]types.Decimal128, len(tt.values)) + for i, value := range tt.values { + var err error + parsed[i], err = types.ParseDecimal128(value, tt.typ.Width, tt.typ.Scale) + require.NoError(t, err) + } + require.NoError(t, vector.AppendFixedList(values.vecs[0], parsed, nil, mp)) + + got, err := PercentileDecimal128(values, tt.p, tt.typ.Scale) + require.NoError(t, err) + require.Equal(t, tt.want, got.Format(tt.outScale)) + }) + } +} + +// --- Executor tests --- + +func formatFloatConfig(p float64) string { + return strconv.FormatFloat(p, 'f', -1, 64) +} + +func TestApproxPercentileReturnType(t *testing.T) { + require.Equal(t, types.New(types.T_decimal128, 38, 3), + ApproxPercentileReturnType([]types.Type{types.New(types.T_decimal128, 37, 2)})) + require.Equal(t, types.New(types.T_decimal128, 38, 0), + ApproxPercentileReturnType([]types.Type{types.New(types.T_decimal128, 38, 0)})) + require.Equal(t, types.New(types.T_decimal128, 38, 38), + ApproxPercentileReturnType([]types.Type{types.New(types.T_decimal128, 38, 38)})) + require.Equal(t, types.T_float64.ToType(), + ApproxPercentileReturnType([]types.Type{types.T_int64.ToType()})) +} + +func TestApproxPercentileExecAcrossSupportedTypes(t *testing.T) { + mp := mpool.MustNewZero() + + cases := []struct { + name string + typ types.Type + values any + p float64 + want any + }{ + {name: "bit", typ: types.T_bit.ToType(), values: []uint64{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "int8", typ: types.T_int8.ToType(), values: []int8{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "int16", typ: types.T_int16.ToType(), values: []int16{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "int32", typ: types.T_int32.ToType(), values: []int32{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "int64", typ: types.T_int64.ToType(), values: []int64{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "int64_large_opposite", typ: types.T_int64.ToType(), values: []int64{-9007199254740993, 9007199254740992}, p: 0.5, want: -0.5}, + {name: "uint8", typ: types.T_uint8.ToType(), values: []uint8{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "uint16", typ: types.T_uint16.ToType(), values: []uint16{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "uint32", typ: types.T_uint32.ToType(), values: []uint32{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "uint64", typ: types.T_uint64.ToType(), values: []uint64{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "float32", typ: types.T_float32.ToType(), values: []float32{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "float64", typ: types.T_float64.ToType(), values: []float64{1, 3, 2}, p: 0.5, want: 2.0}, + {name: "decimal64", typ: types.New(types.T_decimal64, 10, 2), values: mustDecimal64s(t, "1.00", "2.00", "3.00"), p: 0.5, want: "2.000"}, + {name: "decimal128", typ: types.New(types.T_decimal128, 20, 2), values: mustDecimal128s(t, "1.00", "2.00", "3.00"), p: 0.5, want: "2.000"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + exec, err := makeApproxPercentile(mp, 1, false, tc.typ) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + + vec := medianTestVector(t, mp, tc.typ, tc.values) + require.NoError(t, exec.BulkFill(0, []*vector.Vector{vec})) + + // Set percentile via SetExtraInformation + require.NoError(t, exec.SetExtraInformation([]byte(formatFloatConfig(tc.p)), 0)) + + require.GreaterOrEqual(t, exec.Size(), int64(0)) + + ret, err := exec.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + + switch want := tc.want.(type) { + case float64: + require.Equal(t, want, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + case string: + require.Equal(t, want, vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 0).Format(ret[0].GetType().Scale)) + } + + vec.Free(mp) + ret[0].Free(mp) + exec.Free() + }) + } +} + +func TestApproxPercentileExec_MaxPrecisionDecimal(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + + tests := []struct { + name string + typ types.Type + values []string + p string + want string + }{ + { + name: "decimal38 scale0 minimum endpoint", + typ: types.New(types.T_decimal128, 38, 0), + values: []string{"-99999999999999999999999999999999999999", "99999999999999999999999999999999999999"}, + p: "0", + want: "-99999999999999999999999999999999999999", + }, + { + name: "decimal38 scale0 maximum endpoint", + typ: types.New(types.T_decimal128, 38, 0), + values: []string{"-99999999999999999999999999999999999999", "99999999999999999999999999999999999999"}, + p: "1", + want: "99999999999999999999999999999999999999", + }, + { + name: "decimal38 scale38 endpoint", + typ: types.New(types.T_decimal128, 38, 38), + values: []string{"0.1", "0.2"}, + p: "1", + want: "0.20000000000000000000000000000000000000", + }, + { + name: "decimal38 scale0 midpoint rounds at retained scale", + typ: types.New(types.T_decimal128, 38, 0), + values: []string{"0", "1"}, + p: "0.5", + want: "1", + }, + { + name: "decimal38 scale0 negative midpoint rounds away from zero", + typ: types.New(types.T_decimal128, 38, 0), + values: []string{"-1", "0"}, + p: "0.5", + want: "-1", + }, + { + name: "decimal37 gains interpolation scale", + typ: types.New(types.T_decimal128, 37, 0), + values: []string{"0", "1"}, + p: "0.5", + want: "0.5", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, test.typ) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte(test.p), 0)) + + values := make([]types.Decimal128, len(test.values)) + for i, value := range test.values { + values[i], err = types.ParseDecimal128(value, test.typ.Width, test.typ.Scale) + require.NoError(t, err) + } + vec := medianTestVector(t, mp, test.typ, values) + require.NoError(t, exec.BulkFill(0, []*vector.Vector{vec})) + + ret, err := exec.Flush() + require.NoError(t, err) + require.Equal(t, test.want, + vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 0).Format(ret[0].GetType().Scale)) + + vec.Free(mp) + ret[0].Free(mp) + exec.Free() + }) + } +} + +func TestApproxPercentileExec_DifferentPercentiles(t *testing.T) { + mp := mpool.MustNewZero() + + cases := []struct { + label string + p float64 + want float64 + }{ + {"p000", 0.00, 1.0}, + {"p025", 0.25, 3.25}, + {"p050", 0.50, 5.5}, + {"p075", 0.75, 7.75}, + {"p095", 0.95, 9.55}, + {"p100", 1.00, 10.0}, + } + + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + e, err := makeApproxPercentile(mp, 1, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, e.GroupGrow(1)) + vc := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + defer vc.Free(mp) + require.NoError(t, e.BulkFill(0, []*vector.Vector{vc})) + require.NoError(t, e.SetExtraInformation([]byte(formatFloatConfig(tc.p)), 0)) + + ret, err := e.Flush() + require.NoError(t, err) + require.InDelta(t, tc.want, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0), 1e-10) + ret[0].Free(mp) + e.Free() + }) + } +} + +func TestApproxPercentileExec_Float64ExtremeInterpolation(t *testing.T) { + mp := mpool.MustNewZero() + exec, err := makeApproxPercentile(mp, 1, false, types.T_float64.ToType()) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + + vec := buildFixedVec(t, mp, types.T_float64.ToType(), []float64{-math.MaxFloat64, math.MaxFloat64}) + require.NoError(t, exec.BulkFill(0, []*vector.Vector{vec})) + for _, test := range []struct { + percentile string + want float64 + }{ + {percentile: "0", want: -math.MaxFloat64}, + {percentile: "0.5", want: 0}, + {percentile: "1", want: math.MaxFloat64}, + } { + require.NoError(t, exec.SetExtraInformation([]byte(test.percentile), 0)) + ret, err := exec.Flush() + require.NoError(t, err) + require.Equal(t, test.want, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + ret[0].Free(mp) + } + + vec.Free(mp) + exec.Free() + require.Equal(t, int64(0), mp.CurrNB()) +} + +func TestApproxPercentileExec_Float64InfiniteEndpoints(t *testing.T) { + for _, test := range []struct { + name string + values []float64 + percentile string + wantInf int + wantNaN bool + }{ + {name: "positive singleton p0", values: []float64{math.Inf(1)}, percentile: "0", wantInf: 1}, + {name: "positive singleton p50", values: []float64{math.Inf(1)}, percentile: "0.5", wantInf: 1}, + {name: "positive singleton p100", values: []float64{math.Inf(1)}, percentile: "1", wantInf: 1}, + {name: "negative singleton p0", values: []float64{math.Inf(-1)}, percentile: "0", wantInf: -1}, + {name: "negative singleton p50", values: []float64{math.Inf(-1)}, percentile: "0.5", wantInf: -1}, + {name: "negative singleton p100", values: []float64{math.Inf(-1)}, percentile: "1", wantInf: -1}, + {name: "opposite endpoints p0", values: []float64{math.Inf(-1), math.Inf(1)}, percentile: "0", wantInf: -1}, + {name: "opposite endpoints p50", values: []float64{math.Inf(-1), math.Inf(1)}, percentile: "0.5", wantNaN: true}, + {name: "opposite endpoints p100", values: []float64{math.Inf(-1), math.Inf(1)}, percentile: "1", wantInf: 1}, + } { + t.Run(test.name, func(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + exec, err := makeApproxPercentile(mp, 1, false, types.T_float64.ToType()) + require.NoError(t, err) + defer exec.Free() + require.NoError(t, exec.GroupGrow(1)) + vec := buildFixedVec(t, mp, types.T_float64.ToType(), test.values) + defer vec.Free(mp) + require.NoError(t, exec.BulkFill(0, []*vector.Vector{vec})) + require.NoError(t, exec.SetExtraInformation([]byte(test.percentile), 0)) + + ret, err := exec.Flush() + require.NoError(t, err) + defer ret[0].Free(mp) + got := vector.GetFixedAtNoTypeCheck[float64](ret[0], 0) + if test.wantNaN { + require.True(t, math.IsNaN(got)) + } else { + require.True(t, math.IsInf(got, test.wantInf)) + } + }) + } +} + +func TestApproxPercentileExec_Fill(t *testing.T) { + t.Run("const row uses physical index zero", func(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_float64.ToType()) + require.NoError(t, err) + defer exec.Free() + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte("0.5"), 0)) + vec, err := vector.NewConstFixed(types.T_float64.ToType(), math.Inf(1), 3, mp) + require.NoError(t, err) + defer vec.Free(mp) + + require.NoError(t, exec.Fill(0, 2, []*vector.Vector{vec})) + ret, err := exec.Flush() + require.NoError(t, err) + defer ret[0].Free(mp) + require.True(t, math.IsInf(vector.GetFixedAtNoTypeCheck[float64](ret[0], 0), 1)) + }) + + t.Run("null row does not create a value", func(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_float64.ToType()) + require.NoError(t, err) + defer exec.Free() + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte("0.5"), 0)) + vec := vector.NewConstNull(types.T_float64.ToType(), 3, mp) + defer vec.Free(mp) + + require.NoError(t, exec.Fill(0, 2, []*vector.Vector{vec})) + ret, err := exec.Flush() + require.NoError(t, err) + defer ret[0].Free(mp) + require.True(t, ret[0].IsNull(0)) + }) +} + +func TestApproxPercentileExec_DistinctNotSupported(t *testing.T) { + mp := mpool.MustNewZero() + + _, err := makeApproxPercentile(mp, 1, true, types.T_int64.ToType()) + require.Error(t, err) + require.Contains(t, err.Error(), "distinct") +} + +func TestApproxPercentileExec_UnsupportedType(t *testing.T) { + mp := mpool.MustNewZero() + + _, err := makeApproxPercentile(mp, 1, false, types.T_varchar.ToType()) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported type") +} + +func TestApproxPercentileExec_SetExtraInformation_Invalid(t *testing.T) { + mp := mpool.MustNewZero() + + exec, err := makeApproxPercentile(mp, 1, false, types.T_int64.ToType()) + require.NoError(t, err) + + // Not []byte + err = exec.SetExtraInformation("not-bytes", 0) + require.Error(t, err) + require.Contains(t, err.Error(), "expected []byte config") + + // Not a valid float + err = exec.SetExtraInformation([]byte("not-a-float"), 0) + require.Error(t, err) + + // Percentile out of range + err = exec.SetExtraInformation([]byte("1.5"), 0) + require.Error(t, err) + require.Contains(t, err.Error(), "percentile must be in [0,1]") + + err = exec.SetExtraInformation([]byte("-0.5"), 0) + require.Error(t, err) + require.Contains(t, err.Error(), "percentile must be in [0,1]") + + for _, value := range []string{"NaN", "+Inf", "-Inf"} { + err = exec.SetExtraInformation([]byte(value), 0) + require.Error(t, err, "non-finite percentile %s must be rejected", value) + } + + exec.Free() +} + +func TestApproxPercentileExec_MultipleGroups(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(3)) + + // Set percentile config once (shared across groups) + require.NoError(t, exec.SetExtraInformation([]byte("0.5"), 0)) + + // Group 1: values 1,2 -> median 1.5 + // Group 2: values 10,20 -> median 15 + // Group 3: value 100 -> median 100 + vec := vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixedList(vec, []int64{1, 2, 10, 20, 0, 0, 100}, []bool{false, false, false, false, true, true, false}, mp)) + defer vec.Free(mp) + + require.NoError(t, exec.BatchFill(0, []uint64{1, 1, 2, 2, 2, GroupNotMatched, 3}, []*vector.Vector{vec})) + + ret, err := exec.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + require.Equal(t, 1.5, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + require.Equal(t, 15.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 1)) + require.Equal(t, 100.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 2)) + ret[0].Free(mp) + exec.Free() +} + +func TestApproxPercentileExec_NullHandling(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_float64.ToType()) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte("0.5"), 0)) + + vec := vector.NewVec(types.T_float64.ToType()) + require.NoError(t, vector.AppendFixedList(vec, []float64{0, 1, 2, 3, 4, 5}, []bool{true, false, true, false, true, false}, mp)) + defer vec.Free(mp) + + require.NoError(t, exec.BatchFill(0, []uint64{1, 1, 1, 1, 1, 1}, []*vector.Vector{vec})) + + ret, err := exec.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + // Values: 1, 3, 5 -> sorted -> median = 3 + require.Equal(t, 3.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + ret[0].Free(mp) + exec.Free() +} + +func TestApproxPercentileExec_EmptyGroup(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte("0.5"), 0)) + + ret, err := exec.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + // Empty group should return NULL + require.True(t, ret[0].IsNull(0)) + ret[0].Free(mp) + exec.Free() +} + +func TestApproxPercentileExec_BatchMerge(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, left.GroupGrow(2)) + require.NoError(t, right.GroupGrow(2)) + + // Set percentile for both + require.NoError(t, left.SetExtraInformation([]byte("0.5"), 0)) + require.NoError(t, right.SetExtraInformation([]byte("0.5"), 0)) + + vecLeft := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{1, 9, 3, 11}) + vecRight := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{5, 13, 7, 15}) + defer vecLeft.Free(mp) + defer vecRight.Free(mp) + + require.NoError(t, left.BatchFill(0, []uint64{1, 1, 2, 2}, []*vector.Vector{vecLeft})) + require.NoError(t, right.BatchFill(0, []uint64{1, 1, 2, 2}, []*vector.Vector{vecRight})) + require.NoError(t, left.BatchMerge(right, 0, []uint64{1, 2})) + + ret, err := left.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + // Group 1: 1,5,9,13 -> median = (5+9)/2 = 7.0 + // Group 2: 3,7,11,15 -> median = (7+11)/2 = 9.0 + require.Equal(t, 7.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + require.Equal(t, 9.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 1)) + ret[0].Free(mp) + left.Free() + right.Free() +} + +func TestApproxPercentileExec_Merge(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, left.GroupGrow(1)) + require.NoError(t, right.GroupGrow(1)) + require.NoError(t, left.SetExtraInformation([]byte("0.5"), 0)) + require.NoError(t, right.SetExtraInformation([]byte("0.5"), 0)) + + vecLeft := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{1, 9}) + vecRight := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{5, 13}) + defer vecLeft.Free(mp) + defer vecRight.Free(mp) + + require.NoError(t, left.BulkFill(0, []*vector.Vector{vecLeft})) + require.NoError(t, right.BulkFill(0, []*vector.Vector{vecRight})) + require.NoError(t, left.Merge(right, 0, 0)) + + ret, err := left.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + // Values: 1,5,9,13 -> median = (5+9)/2 = 7.0 + require.Equal(t, 7.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + ret[0].Free(mp) + left.Free() + right.Free() +} + +func TestApproxPercentileExec_BatchMerge_DifferentPercentile(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, left.GroupGrow(1)) + require.NoError(t, right.GroupGrow(1)) + require.NoError(t, left.SetExtraInformation([]byte("0.95"), 0)) + require.NoError(t, right.SetExtraInformation([]byte("0.95"), 0)) + + vecLeft := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{1, 2, 3, 4, 5}) + vecRight := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{6, 7, 8, 9, 10}) + defer vecLeft.Free(mp) + defer vecRight.Free(mp) + + require.NoError(t, left.BulkFill(0, []*vector.Vector{vecLeft})) + require.NoError(t, right.BulkFill(0, []*vector.Vector{vecRight})) + require.NoError(t, left.Merge(right, 0, 0)) + + ret, err := left.Flush() + require.NoError(t, err) + // Values: 1..10, p=0.95, idx=0.95*9=8.55, lo=8(9), hi=9(10), frac=0.55 + // 9 + 0.55*1 = 9.55 + require.InDelta(t, 9.55, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0), 1e-10) + ret[0].Free(mp) + left.Free() + right.Free() +} + +func TestApproxPercentileExec_IntermediateRoundTrip(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(3)) + require.NoError(t, exec.SetExtraInformation([]byte("0.5"), 0)) + + vec := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{1, 1, 2, 4, 6, 8, 10}) + defer vec.Free(mp) + require.NoError(t, exec.BatchFill(0, []uint64{1, 1, 1, 2, 2, 3, 3}, []*vector.Vector{vec})) + + var buf bytes.Buffer + require.NoError(t, exec.SaveIntermediateResult(3, [][]uint8{{1, 1, 1}}, &buf)) + + restored, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, restored.UnmarshalFromReader(bytes.NewReader(buf.Bytes()), mp)) + require.NoError(t, restored.SetExtraInformation([]byte("0.5"), 0)) + + ret, err := restored.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + require.Equal(t, 1.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + require.Equal(t, 5.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 1)) + require.Equal(t, 9.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 2)) + ret[0].Free(mp) + exec.Free() + restored.Free() +} + +func TestApproxPercentileExec_Float64InfiniteMergeRoundTrip(t *testing.T) { + for _, test := range []struct { + name string + left float64 + right float64 + percentile string + wantInf int + wantNaN bool + }{ + {name: "equal positive infinity", left: math.Inf(1), right: math.Inf(1), percentile: "0.5", wantInf: 1}, + {name: "equal negative infinity", left: math.Inf(-1), right: math.Inf(-1), percentile: "0.5", wantInf: -1}, + {name: "opposite left endpoint", left: math.Inf(-1), right: math.Inf(1), percentile: "0", wantInf: -1}, + {name: "opposite interpolation", left: math.Inf(-1), right: math.Inf(1), percentile: "0.5", wantNaN: true}, + {name: "opposite right endpoint", left: math.Inf(-1), right: math.Inf(1), percentile: "1", wantInf: 1}, + } { + t.Run(test.name, func(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_float64.ToType()) + require.NoError(t, err) + defer left.Free() + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_float64.ToType()) + require.NoError(t, err) + defer right.Free() + require.NoError(t, left.GroupGrow(1)) + require.NoError(t, right.GroupGrow(1)) + leftVec := buildFixedVec(t, mp, types.T_float64.ToType(), []float64{test.left}) + defer leftVec.Free(mp) + rightVec := buildFixedVec(t, mp, types.T_float64.ToType(), []float64{test.right}) + defer rightVec.Free(mp) + require.NoError(t, left.BulkFill(0, []*vector.Vector{leftVec})) + require.NoError(t, right.BulkFill(0, []*vector.Vector{rightVec})) + require.NoError(t, left.Merge(right, 0, 0)) + + var buf bytes.Buffer + require.NoError(t, left.SaveIntermediateResult(1, [][]uint8{{1}}, &buf)) + restored, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_float64.ToType()) + require.NoError(t, err) + defer restored.Free() + require.NoError(t, restored.UnmarshalFromReader(bytes.NewReader(buf.Bytes()), mp)) + require.NoError(t, restored.SetExtraInformation([]byte(test.percentile), 0)) + + ret, err := restored.Flush() + require.NoError(t, err) + defer ret[0].Free(mp) + got := vector.GetFixedAtNoTypeCheck[float64](ret[0], 0) + if test.wantNaN { + require.True(t, math.IsNaN(got)) + } else { + require.True(t, math.IsInf(got, test.wantInf)) + } + }) + } +} + +func TestApproxPercentileExec_DecimalMerge(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + typ := types.New(types.T_decimal64, 10, 2) + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, typ) + require.NoError(t, err) + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, typ) + require.NoError(t, err) + require.NoError(t, left.GroupGrow(1)) + require.NoError(t, right.GroupGrow(1)) + require.NoError(t, left.SetExtraInformation([]byte("0.5"), 0)) + require.NoError(t, right.SetExtraInformation([]byte("0.5"), 0)) + + vecLeft := medianTestVector(t, mp, typ, mustDecimal64s(t, "1.00", "3.00", "5.00")) + vecRight := medianTestVector(t, mp, typ, mustDecimal64s(t, "3.00", "7.00", "9.00")) + defer vecLeft.Free(mp) + defer vecRight.Free(mp) + + require.NoError(t, left.BulkFill(0, []*vector.Vector{vecLeft})) + require.NoError(t, right.BulkFill(0, []*vector.Vector{vecRight})) + require.NoError(t, left.Merge(right, 0, 0)) + + ret, err := left.Flush() + require.NoError(t, err) + // Values: 1,3,3,5,7,9 -> median = (3+5)/2 = 4.0 -> "4.000" with scale+1=3 + require.Equal(t, "4.000", vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 0).Format(ret[0].GetType().Scale)) + ret[0].Free(mp) + left.Free() + right.Free() +} + +func TestApproxPercentileExec_P95LargerDataset(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte("0.95"), 0)) + + vec := buildFixedVec(t, mp, types.T_int64.ToType(), []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}) + defer vec.Free(mp) + require.NoError(t, exec.BatchFill(0, []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, []*vector.Vector{vec})) + + ret, err := exec.Flush() + require.NoError(t, err) + // idx = 0.95 * 19 = 18.05, lo=18(19), hi=19(20), frac=0.05 + // 19 + 0.05*1 = 19.05 + require.InDelta(t, 19.05, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0), 1e-10) + ret[0].Free(mp) + exec.Free() +} + +func TestApproxPercentileExec_Decimal128WithP75(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + typ := types.New(types.T_decimal128, 20, 2) + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, typ) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte("0.75"), 0)) + + vals := mustDecimal128s(t, "10.00", "20.00", "30.00", "40.00") + vec := medianTestVector(t, mp, typ, vals) + defer vec.Free(mp) + require.NoError(t, exec.BulkFill(0, []*vector.Vector{vec})) + + ret, err := exec.Flush() + require.NoError(t, err) + // idx = 0.75 * 3 = 2.25, lo=2(30), hi=3(40), frac=0.25 + // 30 + 0.25*10 = 32.5 + require.Equal(t, "32.500", vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 0).Format(ret[0].GetType().Scale)) + ret[0].Free(mp) + exec.Free() +} + +func TestApproxPercentileExec_DecimalBatchMerge(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + typ := types.New(types.T_decimal64, 10, 2) + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, typ) + require.NoError(t, err) + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, typ) + require.NoError(t, err) + require.NoError(t, left.GroupGrow(2)) + require.NoError(t, right.GroupGrow(2)) + require.NoError(t, left.SetExtraInformation([]byte("0.5"), 0)) + require.NoError(t, right.SetExtraInformation([]byte("0.5"), 0)) + + vecLeft := medianTestVector(t, mp, typ, mustDecimal64s(t, "1.00", "9.00", "3.00", "11.00")) + vecRight := medianTestVector(t, mp, typ, mustDecimal64s(t, "5.00", "13.00", "7.00", "15.00")) + defer vecLeft.Free(mp) + defer vecRight.Free(mp) + + require.NoError(t, left.BatchFill(0, []uint64{1, 1, 2, 2}, []*vector.Vector{vecLeft})) + require.NoError(t, right.BatchFill(0, []uint64{1, 1, 2, 2}, []*vector.Vector{vecRight})) + require.NoError(t, left.BatchMerge(right, 0, []uint64{1, 2})) + + ret, err := left.Flush() + require.NoError(t, err) + require.Len(t, ret, 1) + // Group 1: 1,5,9,13 -> median = 7.0 -> "7.000" + // Group 2: 3,7,11,15 -> median = 9.0 -> "9.000" + require.Equal(t, "7.000", vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 0).Format(ret[0].GetType().Scale)) + require.Equal(t, "9.000", vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 1).Format(ret[0].GetType().Scale)) + ret[0].Free(mp) + left.Free() + right.Free() +} + +func TestApproxPercentileExec_Decimal128Merge(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + typ := types.New(types.T_decimal128, 20, 2) + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, typ) + require.NoError(t, err) + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, typ) + require.NoError(t, err) + require.NoError(t, left.GroupGrow(1)) + require.NoError(t, right.GroupGrow(1)) + require.NoError(t, left.SetExtraInformation([]byte("0.5"), 0)) + require.NoError(t, right.SetExtraInformation([]byte("0.5"), 0)) + + vecLeft := medianTestVector(t, mp, typ, mustDecimal128s(t, "2.00", "4.00", "6.00")) + vecRight := medianTestVector(t, mp, typ, mustDecimal128s(t, "8.00", "10.00", "12.00")) + defer vecLeft.Free(mp) + defer vecRight.Free(mp) + + require.NoError(t, left.BulkFill(0, []*vector.Vector{vecLeft})) + require.NoError(t, right.BulkFill(0, []*vector.Vector{vecRight})) + require.NoError(t, left.Merge(right, 0, 0)) + + ret, err := left.Flush() + require.NoError(t, err) + // Values: 2,4,6,8,10,12 -> median = 7.0 -> "7.000" + require.Equal(t, "7.000", vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 0).Format(ret[0].GetType().Scale)) + ret[0].Free(mp) + left.Free() + right.Free() +} + +func TestApproxPercentileExec_LargeDecimalDoesNotUseFloat64(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + + tests := []struct { + name string + typ types.Type + values any + p string + want string + }{ + { + name: "decimal64 min boundary", + typ: types.New(types.T_decimal64, 18, 2), + values: mustDecimal64sWithType(t, 18, 2, "9007199254740993.00", "9007199254740994.00"), + p: "0", + want: "9007199254740993.000", + }, + { + name: "decimal64 interpolation", + typ: types.New(types.T_decimal64, 18, 2), + values: mustDecimal64sWithType(t, 18, 2, "9007199254740993.00", "9007199254740994.00"), + p: "0.5", + want: "9007199254740993.500", + }, + { + name: "decimal128 min boundary", + typ: types.New(types.T_decimal128, 30, 2), + values: mustDecimal128s(t, "9007199254740993.00", "9007199254740994.00"), + p: "0", + want: "9007199254740993.000", + }, + { + name: "decimal128 negative interpolation", + typ: types.New(types.T_decimal128, 30, 2), + values: mustDecimal128s(t, "-9007199254740994.00", "-9007199254740993.00"), + p: "0.5", + want: "-9007199254740993.500", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, test.typ) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte(test.p), 0)) + vec := medianTestVector(t, mp, test.typ, test.values) + require.NoError(t, exec.BulkFill(0, []*vector.Vector{vec})) + + ret, err := exec.Flush() + require.NoError(t, err) + require.Equal(t, test.want, + vector.GetFixedAtNoTypeCheck[types.Decimal128](ret[0], 0).Format(ret[0].GetType().Scale)) + + vec.Free(mp) + ret[0].Free(mp) + exec.Free() + }) + } +} + +func mustDecimal64sWithType(t *testing.T, precision, scale int32, values ...string) []types.Decimal64 { + t.Helper() + result := make([]types.Decimal64, len(values)) + for i, value := range values { + decimal, err := types.ParseDecimal64(value, precision, scale) + require.NoError(t, err) + result[i] = decimal + } + return result +} + +func TestApproxPercentileExec_BoundedSketchPartialMerge10K(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + + const ( + rowCount = 10_000 + parts = 4 + ) + execs := make([]AggFuncExec, parts) + exact := make([]int64, 0, rowCount) + for part := range parts { + exec, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + require.NoError(t, exec.GroupGrow(1)) + require.NoError(t, exec.SetExtraInformation([]byte("0.95"), 0)) + execs[part] = exec + + values := make([]int64, rowCount/parts) + nulls := make([]bool, len(values)) + for row := range values { + globalRow := part*len(values) + row + values[row] = int64(globalRow % 1_000) + nulls[row] = globalRow%11 == 0 + if !nulls[row] { + exact = append(exact, values[row]) + } + } + vec := vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixedList(vec, values, nulls, mp)) + require.NoError(t, exec.BulkFill(0, []*vector.Vector{vec})) + vec.Free(mp) + } + defer func() { + for _, exec := range execs { + exec.Free() + } + }() + + for part := 1; part < parts; part++ { + require.NoError(t, execs[0].Merge(execs[part], 0, 0)) + } + impl := execs[0].(*approxPercentileNumericExec[int64]) + sketch := impl.state[0].mobs[0].(*quantileSketch[int64]) + require.Equal(t, uint64(len(exact)), sketch.count) + require.Less(t, sketch.retained(), 1_000, "retained state must stay bounded well below input rows") + + var intermediate bytes.Buffer + require.NoError(t, execs[0].SaveIntermediateResult(1, [][]uint8{{1}}, &intermediate)) + require.Less(t, intermediate.Len(), 20_000, "distributed intermediate state must be bounded") + + restored, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + defer restored.Free() + require.NoError(t, restored.UnmarshalFromReader(bytes.NewReader(intermediate.Bytes()), mp)) + require.NoError(t, restored.SetExtraInformation([]byte("0.95"), 0)) + + ret, err := restored.Flush() + require.NoError(t, err) + got := vector.GetFixedAtNoTypeCheck[float64](ret[0], 0) + want := percentileNumericVals(exact, 0.95) + require.InDelta(t, want, got, 25, "bounded sketch rank error exceeded test tolerance") + ret[0].Free(mp) + + require.NoError(t, restored.SetExtraInformation([]byte("0"), 0)) + ret, err = restored.Flush() + require.NoError(t, err) + require.Equal(t, 0.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + ret[0].Free(mp) + + require.NoError(t, restored.SetExtraInformation([]byte("1"), 0)) + ret, err = restored.Flush() + require.NoError(t, err) + require.Equal(t, 999.0, vector.GetFixedAtNoTypeCheck[float64](ret[0], 0)) + ret[0].Free(mp) +} + +func TestApproxPercentileExec_RejectsDifferentMergeConfig(t *testing.T) { + mp := mpool.MustNewZero() + left, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + right, err := makeApproxPercentile(mp, AggIdOfApproxPercentile, false, types.T_int64.ToType()) + require.NoError(t, err) + defer left.Free() + defer right.Free() + require.NoError(t, left.GroupGrow(1)) + require.NoError(t, right.GroupGrow(1)) + require.NoError(t, left.SetExtraInformation([]byte("0.95"), 0)) + require.NoError(t, right.SetExtraInformation([]byte("0.99"), 0)) + require.ErrorContains(t, left.Merge(right, 0, 0), "different percentile configurations") +} + +func TestQuantileSketchMPoolLifecycle(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + + left := newQuantileSketch[int64](mp, orderedCompare[int64]) + right := newQuantileSketch[int64](mp, orderedCompare[int64]) + restored := newQuantileSketch[int64](mp, orderedCompare[int64]) + defer left.Free() + defer right.Free() + defer restored.Free() + + for value := range int64(2_000) { + require.NoError(t, left.Add(value)) + require.NoError(t, right.Add(value+2_000)) + } + require.Positive(t, mp.CurrNB(), "retained samples must be tracked by the mpool") + + require.NoError(t, left.Merge(right)) + beforeQuantile := mp.CurrNB() + _, _, _, err := left.Quantile(big.NewRat(1, 2)) + require.NoError(t, err) + require.Equal(t, beforeQuantile, mp.CurrNB(), "quantile scratch space must be released") + encoded, err := left.MarshalBinary() + require.NoError(t, err) + require.NoError(t, restored.UnmarshalBinary(encoded)) + require.Equal(t, left.count, restored.count) + require.Equal(t, left.retained(), restored.retained()) + + beforeInvalid := mp.CurrNB() + invalid := append(append([]byte(nil), encoded...), 0) + require.Error(t, restored.UnmarshalBinary(invalid)) + require.Equal(t, beforeInvalid, mp.CurrNB(), "failed unmarshal must release its temporary state") + require.Error(t, restored.UnmarshalBinary(encoded[:len(encoded)-1])) + require.Equal(t, beforeInvalid, mp.CurrNB(), "truncated unmarshal must release partially decoded levels") +} + +func TestQuantileSketchMergeEmptyParityLevelIntoEmptyDestination(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { require.Equal(t, int64(0), mp.CurrNB()) }() + + dst := newQuantileSketch[int64](mp, orderedCompare[int64]) + src := newQuantileSketch[int64](mp, orderedCompare[int64]) + defer dst.Free() + defer src.Free() + + for value := range int64(2 * approxPercentileSketchCapacity) { + require.NoError(t, src.Add(value)) + } + require.Empty(t, src.levels[0]) + require.True(t, src.parity[0]) + + require.NoError(t, dst.Merge(src)) + require.Equal(t, src.count, dst.count) + require.True(t, dst.hasValue) +} diff --git a/pkg/sql/colexec/aggexec/function_id.go b/pkg/sql/colexec/aggexec/function_id.go new file mode 100644 index 0000000000000..5ee167f866f19 --- /dev/null +++ b/pkg/sql/colexec/aggexec/function_id.go @@ -0,0 +1,59 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aggexec + +// Aggregate executors receive the encoded function overload ID. All aggregate +// and window functions below have overload zero, so these constants match the +// IDs encoded by the function catalog without requiring init-time registration. +const ( + AggIdOfAny int64 = 44 << 32 + AggIdOfApproxCount int64 = 45 << 32 + AggIdOfAvg int64 = 57 << 32 + AggIdOfAvgTwCache int64 = 58 << 32 + AggIdOfAvgTwResult int64 = 59 << 32 + AggIdOfBitAnd int64 = 62 << 32 + AggIdOfBitOr int64 = 65 << 32 + AggIdOfBitXor int64 = 66 << 32 + AggIdOfCountColumn int64 = 82 << 32 + WinIdOfCumeDist int64 = 87 << 32 + WinIdOfDenseRank int64 = 96 << 32 + WinIdOfFirstValue int64 = 102 << 32 + WinIdOfLag int64 = 119 << 32 + WinIdOfLastValue int64 = 120 << 32 + WinIdOfLead int64 = 121 << 32 + AggIdOfMax int64 = 137 << 32 + AggIdOfMedian int64 = 138 << 32 + AggIdOfMin int64 = 139 << 32 + WinIdOfNthValue int64 = 143 << 32 + WinIdOfNtile int64 = 144 << 32 + WinIdOfPercentRank int64 = 146 << 32 + WinIdOfRank int64 = 155 << 32 + WinIdOfRowNumber int64 = 168 << 32 + AggIdOfCountStar int64 = 178 << 32 + AggIdOfStdDevPop int64 = 180 << 32 + AggIdOfStdDevSample int64 = 181 << 32 + AggIdOfSum int64 = 183 << 32 + AggIdOfGroupConcat int64 = 185 << 32 + AggIdOfVarPop int64 = 199 << 32 + AggIdOfVarSample int64 = 200 << 32 + AggIdOfApproxCountDistinct int64 = 226 << 32 + AggIdOfBitmapConstruct int64 = 333 << 32 + AggIdOfBitmapOr int64 = 334 << 32 + AggIdOfJsonArrayAgg int64 = 400 << 32 + AggIdOfJsonObjectAgg int64 = 401 << 32 + AggIdOfHllAdd int64 = 453 << 32 + AggIdOfHllMerge int64 = 454 << 32 + AggIdOfApproxPercentile int64 = 557 << 32 +) diff --git a/pkg/sql/colexec/aggexec/jsonagg2.go b/pkg/sql/colexec/aggexec/jsonagg2.go index 83eb12c65f130..ebab25d0c8516 100644 --- a/pkg/sql/colexec/aggexec/jsonagg2.go +++ b/pkg/sql/colexec/aggexec/jsonagg2.go @@ -25,16 +25,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/vector" ) -func RegisterJsonArrayAgg(id int64) { - specialAgg[id] = true - AggIdOfJsonArrayAgg = id -} - -func RegisterJsonObjectAgg(id int64) { - specialAgg[id] = true - AggIdOfJsonObjectAgg = id -} - type jsonArrayAggExec struct { aggExec distinct bool diff --git a/pkg/sql/colexec/aggexec/jsonagg_test.go b/pkg/sql/colexec/aggexec/jsonagg_test.go index 23e69369e978f..9f38cf9b3ffbc 100644 --- a/pkg/sql/colexec/aggexec/jsonagg_test.go +++ b/pkg/sql/colexec/aggexec/jsonagg_test.go @@ -256,12 +256,7 @@ func TestBuildValueByteJsonCoversTypes(t *testing.T) { } } -func TestJsonAggRegistersAndHelpers(t *testing.T) { - RegisterJsonArrayAgg(101) - RegisterJsonObjectAgg(202) - require.Equal(t, int64(101), AggIdOfJsonArrayAgg) - require.Equal(t, int64(202), AggIdOfJsonObjectAgg) - +func TestJsonAggHelpers(t *testing.T) { exec := newJsonArrayAggExec(mpool.MustNewZero(), multiAggInfo{ aggID: 0, distinct: false, diff --git a/pkg/sql/colexec/aggexec/register.go b/pkg/sql/colexec/aggexec/register.go deleted file mode 100644 index ed8b7cdfc19f1..0000000000000 --- a/pkg/sql/colexec/aggexec/register.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package aggexec - -import ( - "github.com/matrixorigin/matrixone/pkg/container/types" -) - -/* - methods to register the aggregation function. - after registered, the function `MakeAgg` can make the aggregation function executor. -*/ - -func RegisterBitmapConstruct(id int64) { - specialAgg[id] = true - AggIdOfBitmapConstruct = id -} - -func RegisterBitmapOr(id int64) { - specialAgg[id] = true - AggIdOfBitmapOr = id -} - -func RegisterBitXorAgg(id int64) { - specialAgg[id] = true - AggIdOfBitXor = id -} - -func RegisterBitAndAgg(id int64) { - specialAgg[id] = true - AggIdOfBitAnd = id -} - -func RegisterBitOrAgg(id int64) { - specialAgg[id] = true - AggIdOfBitOr = id -} - -func RegisterVarPop(id int64) { - specialAgg[id] = true - AggIdOfVarPop = id -} - -func RegisterStdDevPop(id int64) { - specialAgg[id] = true - AggIdOfStdDevPop = id -} - -func RegisterVarSample(id int64) { - specialAgg[id] = true - AggIdOfVarSample = id -} - -func RegisterStdDevSample(id int64) { - specialAgg[id] = true - AggIdOfStdDevSample = id -} - -func RegisterAny(id int64) { - specialAgg[id] = true - AggIdOfAny = id -} - -func RegisterMin(id int64) { - specialAgg[id] = true - AggIdOfMin = id -} - -func RegisterMax(id int64) { - specialAgg[id] = true - AggIdOfMax = id -} - -func RegisterSum(id int64) { - specialAgg[id] = true - AggIdOfSum = id -} - -func RegisterAvg(id int64) { - specialAgg[id] = true - AggIdOfAvg = id -} - -func RegisterCountColumnAgg(id int64) { - specialAgg[id] = true - AggIdOfCountColumn = id -} - -func RegisterCountStarAgg(id int64) { - specialAgg[id] = true - AggIdOfCountStar = id -} - -func RegisterGroupConcatAgg(id int64, sep string) { - specialAgg[id] = true - AggIdOfGroupConcat = id - groupConcatSep = sep -} - -func RegisterApproxCountAgg(id int64) { - specialAgg[id] = true - AggIdOfApproxCount = id -} - -func RegisterHllAddAgg(id int64) { - specialAgg[id] = true - AggIdOfHllAdd = id -} - -func RegisterHllMergeAgg(id int64) { - specialAgg[id] = true - AggIdOfHllMerge = id -} - -func RegisterMedian(id int64) { - specialAgg[id] = true - AggIdOfMedian = id -} - -func RegisterAvgTwCache(id int64) { - specialAgg[id] = true - AggIdOfAvgTwCache = id -} - -func RegisterAvgTwResult(id int64) { - specialAgg[id] = true - AggIdOfAvgTwResult = id -} - -func RegisterRowNumberWin(id int64) { - specialAgg[id] = true - WinIdOfRowNumber = id -} - -func RegisterRankWin(id int64) { - specialAgg[id] = true - WinIdOfRank = id -} - -func RegisterDenseRankWin(id int64) { - specialAgg[id] = true - WinIdOfDenseRank = id -} - -func RegisterPercentRankWin(id int64) { - specialAgg[id] = true - WinIdOfPercentRank = id -} - -func RegisterNtileWin(id int64) { - specialAgg[id] = true - WinIdOfNtile = id -} - -func RegisterLagWin(id int64) { - specialAgg[id] = true - WinIdOfLag = id -} - -func RegisterLeadWin(id int64) { - specialAgg[id] = true - WinIdOfLead = id -} - -func RegisterFirstValueWin(id int64) { - specialAgg[id] = true - WinIdOfFirstValue = id -} - -func RegisterLastValueWin(id int64) { - specialAgg[id] = true - WinIdOfLastValue = id -} - -func RegisterNthValueWin(id int64) { - specialAgg[id] = true - WinIdOfNthValue = id -} - -func RegisterCumeDistWin(id int64) { - specialAgg[id] = true - WinIdOfCumeDist = id -} - -var ( - specialAgg = make(map[int64]bool) - - // list of special aggregation function IDs. - AggIdOfCountColumn = int64(-1) - AggIdOfCountStar = int64(-2) - AggIdOfGroupConcat = int64(-3) - AggIdOfApproxCount = int64(-4) - AggIdOfMedian = int64(-5) - AggIdOfJsonArrayAgg = int64(-6) - AggIdOfJsonObjectAgg = int64(-7) - WinIdOfRowNumber = int64(-8) - WinIdOfRank = int64(-9) - WinIdOfDenseRank = int64(-10) - WinIdOfLag = int64(-11) - WinIdOfLead = int64(-12) - WinIdOfFirstValue = int64(-13) - WinIdOfLastValue = int64(-14) - WinIdOfNthValue = int64(-15) - AggIdOfSum = int64(-16) - AggIdOfAvg = int64(-17) - AggIdOfMin = int64(-18) - AggIdOfMax = int64(-19) - AggIdOfAny = int64(-20) - AggIdOfVarPop = int64(-21) - AggIdOfStdDevPop = int64(-22) - AggIdOfVarSample = int64(-23) - AggIdOfStdDevSample = int64(-24) - AggIdOfBitXor = int64(-25) - AggIdOfBitAnd = int64(-26) - AggIdOfBitOr = int64(-27) - AggIdOfBitmapConstruct = int64(-28) - AggIdOfBitmapOr = int64(-29) - WinIdOfCumeDist = int64(-30) - WinIdOfNtile = int64(-31) - WinIdOfPercentRank = int64(-32) - AggIdOfAvgTwCache = int64(-33) - AggIdOfAvgTwResult = int64(-34) - AggIdOfHllAdd = int64(-35) - AggIdOfHllMerge = int64(-36) - groupConcatSep = "," - getGroupConcatRet = func(args ...types.Type) types.Type { - for _, p := range args { - if p.Oid == types.T_binary || p.Oid == types.T_varbinary || p.Oid == types.T_blob { - return types.T_blob.ToType() - } - } - return types.T_text.ToType() - } -) diff --git a/pkg/sql/colexec/aggexec/sumavg2.go b/pkg/sql/colexec/aggexec/sumavg2.go index 46130ef4c9f83..4287fdac85eb3 100644 --- a/pkg/sql/colexec/aggexec/sumavg2.go +++ b/pkg/sql/colexec/aggexec/sumavg2.go @@ -28,9 +28,6 @@ import ( // // This returned type thing definitely belongs to plan, function. // However, exec cannot import plan, function due to circular dependency. -// Also this is the reason for all those RegisterAgg crap. This is the -// weirdest thing I've ever seen. We need to untangle all this mess. -// // See list_agg.go, we need to remove dependency of plan on exec. // diff --git a/pkg/sql/colexec/aggexec/types.go b/pkg/sql/colexec/aggexec/types.go index 1c0769caa4a96..915ff0792ee44 100644 --- a/pkg/sql/colexec/aggexec/types.go +++ b/pkg/sql/colexec/aggexec/types.go @@ -152,7 +152,6 @@ var ( ) // MakeAgg is the only exporting method to create an aggregation function executor. -// all the aggID should be registered before calling this function. func MakeAgg( mg *mpool.MPool, aggID int64, isDistinct bool, @@ -173,79 +172,80 @@ func makeSpecialAggExec( mp *mpool.MPool, id int64, isDistinct bool, params ...types.Type, ) (AggFuncExec, bool, error) { - if _, ok := specialAgg[id]; ok { - switch id { - case AggIdOfBitmapConstruct: - return makeBmpConstructExec(mp, id, params[0]), true, nil - case AggIdOfBitmapOr: - return makeBmpOrExec(mp, id, params[0]), true, nil - case AggIdOfBitXor: - return makeBitXorExec(mp, id, isDistinct, params[0]), true, nil - case AggIdOfBitAnd: - return makeBitAndExec(mp, id, isDistinct, params[0]), true, nil - case AggIdOfBitOr: - return makeBitOrExec(mp, id, isDistinct, params[0]), true, nil - case AggIdOfVarPop: - return makeVarPopExec(mp, id, isDistinct, params[0]), true, nil - case AggIdOfStdDevPop: - return makeStdDevPopExec(mp, id, isDistinct, params[0]), true, nil - case AggIdOfVarSample: - return makeVarSampleExec(mp, id, isDistinct, params[0]), true, nil - case AggIdOfStdDevSample: - return makeStdDevSampleExec(mp, id, isDistinct, params[0]), true, nil - case AggIdOfAny: - return makeAnyValueExec(mp, id, params[0]), true, nil - case AggIdOfMin: - return makeMinMaxExec(mp, id, true, params[0]), true, nil - case AggIdOfMax: - return makeMinMaxExec(mp, id, false, params[0]), true, nil - case AggIdOfSum: - return makeSumAvgExec(mp, true, id, isDistinct, params[0]), true, nil - case AggIdOfAvg: - return makeSumAvgExec(mp, false, id, isDistinct, params[0]), true, nil - case AggIdOfCountColumn: - return makeCount(mp, false, id, isDistinct, params), true, nil - case AggIdOfCountStar: - return makeCount(mp, true, id, isDistinct, params), true, nil - case AggIdOfMedian: - exec, err := makeMedian(mp, id, isDistinct, params[0]) - return exec, true, err - case AggIdOfGroupConcat: - return makeGroupConcat(mp, id, isDistinct, params, getGroupConcatRet(params...), groupConcatSep), true, nil - case AggIdOfApproxCount: - return makeApproxCount(mp, id, params[0]), true, nil - case AggIdOfHllAdd: - return makeHllAdd(mp, id, params[0]), true, nil - case AggIdOfHllMerge: - return makeHllMerge(mp, id, params[0]), true, nil - case AggIdOfJsonArrayAgg: - exec, err := makeJsonArrayAgg(mp, id, isDistinct, params) - return exec, true, err - case AggIdOfJsonObjectAgg: - exec, err := makeJsonObjectAgg(mp, id, isDistinct, params) - return exec, true, err - case AggIdOfAvgTwCache: - exec, err := makeAvgTwCacheExec(mp, id, params[0]) - return exec, true, err - case AggIdOfAvgTwResult: - exec, err := makeAvgTwResultExec(mp, id, params[0]) - return exec, true, err - case WinIdOfRowNumber, WinIdOfRank, WinIdOfDenseRank: - exec, err := makeWindowExec(mp, id, isDistinct) - return exec, true, err - case WinIdOfPercentRank: - exec, err := makePercentRankExec(mp, id, isDistinct) - return exec, true, err - case WinIdOfNtile: - exec, err := makeNtileExec(mp, id, isDistinct, params) - return exec, true, err - case WinIdOfCumeDist: - exec, err := makeWindowExec(mp, id, isDistinct) - return exec, true, err - case WinIdOfLag, WinIdOfLead, WinIdOfFirstValue, WinIdOfLastValue, WinIdOfNthValue: - exec, err := makeValueWindowExec(mp, id, isDistinct, params) - return exec, true, err - } + switch id { + case AggIdOfBitmapConstruct: + return makeBmpConstructExec(mp, id, params[0]), true, nil + case AggIdOfBitmapOr: + return makeBmpOrExec(mp, id, params[0]), true, nil + case AggIdOfBitXor: + return makeBitXorExec(mp, id, isDistinct, params[0]), true, nil + case AggIdOfBitAnd: + return makeBitAndExec(mp, id, isDistinct, params[0]), true, nil + case AggIdOfBitOr: + return makeBitOrExec(mp, id, isDistinct, params[0]), true, nil + case AggIdOfVarPop: + return makeVarPopExec(mp, id, isDistinct, params[0]), true, nil + case AggIdOfStdDevPop: + return makeStdDevPopExec(mp, id, isDistinct, params[0]), true, nil + case AggIdOfVarSample: + return makeVarSampleExec(mp, id, isDistinct, params[0]), true, nil + case AggIdOfStdDevSample: + return makeStdDevSampleExec(mp, id, isDistinct, params[0]), true, nil + case AggIdOfAny: + return makeAnyValueExec(mp, id, params[0]), true, nil + case AggIdOfMin: + return makeMinMaxExec(mp, id, true, params[0]), true, nil + case AggIdOfMax: + return makeMinMaxExec(mp, id, false, params[0]), true, nil + case AggIdOfSum: + return makeSumAvgExec(mp, true, id, isDistinct, params[0]), true, nil + case AggIdOfAvg: + return makeSumAvgExec(mp, false, id, isDistinct, params[0]), true, nil + case AggIdOfCountColumn: + return makeCount(mp, false, id, isDistinct, params), true, nil + case AggIdOfCountStar: + return makeCount(mp, true, id, isDistinct, params), true, nil + case AggIdOfMedian: + exec, err := makeMedian(mp, id, isDistinct, params[0]) + return exec, true, err + case AggIdOfGroupConcat: + return makeGroupConcat(mp, id, isDistinct, params, GroupConcatReturnType(params), ","), true, nil + case AggIdOfApproxCount, AggIdOfApproxCountDistinct: + return makeApproxCount(mp, id, params[0]), true, nil + case AggIdOfHllAdd: + return makeHllAdd(mp, id, params[0]), true, nil + case AggIdOfHllMerge: + return makeHllMerge(mp, id, params[0]), true, nil + case AggIdOfApproxPercentile: + exec, err := makeApproxPercentile(mp, id, isDistinct, params[0]) + return exec, true, err + case AggIdOfJsonArrayAgg: + exec, err := makeJsonArrayAgg(mp, id, isDistinct, params) + return exec, true, err + case AggIdOfJsonObjectAgg: + exec, err := makeJsonObjectAgg(mp, id, isDistinct, params) + return exec, true, err + case AggIdOfAvgTwCache: + exec, err := makeAvgTwCacheExec(mp, id, params[0]) + return exec, true, err + case AggIdOfAvgTwResult: + exec, err := makeAvgTwResultExec(mp, id, params[0]) + return exec, true, err + case WinIdOfRowNumber, WinIdOfRank, WinIdOfDenseRank: + exec, err := makeWindowExec(mp, id, isDistinct) + return exec, true, err + case WinIdOfPercentRank: + exec, err := makePercentRankExec(mp, id, isDistinct) + return exec, true, err + case WinIdOfNtile: + exec, err := makeNtileExec(mp, id, isDistinct, params) + return exec, true, err + case WinIdOfCumeDist: + exec, err := makeWindowExec(mp, id, isDistinct) + return exec, true, err + case WinIdOfLag, WinIdOfLead, WinIdOfFirstValue, WinIdOfLastValue, WinIdOfNthValue: + exec, err := makeValueWindowExec(mp, id, isDistinct, params) + return exec, true, err } return nil, false, nil } @@ -306,6 +306,23 @@ func makeMedian( return newMedianExec(mp, aggID, isDistinct, param) } +func makeApproxPercentile( + mp *mpool.MPool, aggID int64, isDistinct bool, param types.Type) (AggFuncExec, error) { + info := singleAggInfo{ + aggID: aggID, + distinct: isDistinct, + argType: param, + emptyNull: true, + } + switch param.Oid { + case types.T_decimal64, types.T_decimal128: + info.retType = ApproxPercentileReturnType([]types.Type{param}) + default: + info.retType = types.T_float64.ToType() + } + return newApproxPercentileExec(mp, info) +} + func makeWindowExec( mp *mpool.MPool, aggID int64, isDistinct bool) (AggFuncExec, error) { if isDistinct { diff --git a/pkg/sql/colexec/timewin/timewin_test.go b/pkg/sql/colexec/timewin/timewin_test.go index 4be5cacefae4c..502e52219f885 100644 --- a/pkg/sql/colexec/timewin/timewin_test.go +++ b/pkg/sql/colexec/timewin/timewin_test.go @@ -161,6 +161,66 @@ func TestTimeWin(t *testing.T) { } } +func TestTimeWinApproxPercentileEndpointConfigs(t *testing.T) { + for _, tc := range []struct { + name string + config string + want float64 + }{ + {name: "lower endpoint", config: "0", want: 1}, + {name: "upper endpoint", config: "1", want: 1000}, + } { + t.Run(tc.name, func(t *testing.T) { + proc := testutil.NewProcessWithMPool(t, "", mpool.MustNewZero()) + input := testutil.MakeInt32Vector([]int32{1, 4, 5, 1000}, nil, proc.Mp()) + arg := &TimeWin{ + Types: []types.Type{types.T_int32.ToType()}, + Aggs: []aggexec.AggFuncExecExpression{ + aggexec.MakeAggFunctionExpression( + aggexec.AggIdOfApproxPercentile, + false, + []*plan.Expr{newExpression(1)}, + []byte(tc.config)), + }, + } + + aggs, err := makeAggExecutors(arg, proc, false) + require.NoError(t, err) + require.Len(t, aggs, 1) + require.NoError(t, aggs[0].GroupGrow(1)) + require.NoError(t, aggs[0].BatchFill(0, []uint64{1, 1, 1, 1}, []*vector.Vector{input})) + results, err := aggs[0].Flush() + require.NoError(t, err) + require.Equal(t, []float64{tc.want}, vector.MustFixedColWithTypeCheck[float64](results[0])) + + results[0].Free(proc.Mp()) + aggs[0].Free() + input.Free(proc.Mp()) + proc.Free() + require.Equal(t, int64(0), proc.Mp().CurrNB()) + }) + } +} + +func TestTimeWinApproxPercentileRejectsInvalidExecutorConfig(t *testing.T) { + proc := testutil.NewProcessWithMPool(t, "", mpool.MustNewZero()) + arg := &TimeWin{ + Types: []types.Type{types.T_int32.ToType()}, + Aggs: []aggexec.AggFuncExecExpression{ + aggexec.MakeAggFunctionExpression( + aggexec.AggIdOfApproxPercentile, + false, + []*plan.Expr{newExpression(1)}, + []byte("1.01")), + }, + } + + _, err := makeAggExecutors(arg, proc, false) + require.ErrorContains(t, err, "percentile must be in [0,1]") + proc.Free() + require.Equal(t, int64(0), proc.Mp().CurrNB()) +} + // TestTimeWinSplitDistinctResultAndReplace verifies the complete non-final // flush transition: split physical results are materialized as one logical // batch, and the flushed DISTINCT executor is freed before its replacement is diff --git a/pkg/sql/compile/operator.go b/pkg/sql/compile/operator.go index 1313ae42748d5..aa3996c70b3c2 100644 --- a/pkg/sql/compile/operator.go +++ b/pkg/sql/compile/operator.go @@ -19,6 +19,8 @@ import ( "encoding/json" "fmt" "math" + "math/big" + "strconv" "strings" "time" @@ -1554,9 +1556,10 @@ func constructTimeWindow(_ context.Context, node *plan.Node, proc *process.Proce // Every slot the layout hands out must get an aggregate, or the // operator's columns stop matching the positions the planner projects. e := f.F.Args[0] + args, cfg := constructAggregateConfig(f.F, proc) aggregationExpressions = append( aggregationExpressions, - aggexec.MakeAggFunctionExpression(functionID, isDistinct, f.F.Args, nil)) + aggexec.MakeAggFunctionExpression(functionID, isDistinct, args, cfg)) typs = append(typs, types.New(types.T(e.Typ.Id), e.Typ.Width, e.Typ.Scale)) } wStart := layout.WStartSlot != plan2.TimeWindowSlotNone @@ -1698,6 +1701,24 @@ func constructAggregateConfig(f *plan.Function, proc *process.Process) ([]*plan. config := evaluateAggregateConfigString(proc, args[len(args)-1]) return args[:len(args)-1], []byte(config) } + + case plan2.NameApproxPercentile: + if len(args) > 1 { + configExpr := args[len(args)-1] + if err := validateApproxPercentileExpr(configExpr); err != nil { + panic(err) + } + vec, free, err := colexec.GetReadonlyResultFromNoColumnExpression(proc, configExpr) + if err != nil { + panic(err) + } + defer free() + config, err := getPercentileConfig(vec) + if err != nil { + panic(err) + } + return args[:len(args)-1], config + } } return args, nil } @@ -2550,6 +2571,66 @@ func constructTableClone( return metaCopy, nil } +func validateApproxPercentileExpr(expr *plan.Expr) error { + if expr == nil || !rule.IsConstant(expr, false) { + return moerr.NewInvalidInputNoCtx( + "percentile argument of approx_percentile must be a constant") + } + return nil +} + +// getPercentileConfig extracts the percentile value from a vector for approx_percentile. +func getPercentileConfig(vec *vector.Vector) ([]byte, error) { + if vec == nil || !vec.IsConst() { + return nil, moerr.NewInvalidInputNoCtx( + "percentile argument of approx_percentile must be a constant") + } + if vec.Length() == 0 || vec.IsConstNull() { + return nil, moerr.NewInvalidInputNoCtx( + "percentile argument of approx_percentile cannot be NULL") + } + + var p float64 + var config string + switch vec.GetType().Oid { + case types.T_float64: + p = vector.MustFixedColWithTypeCheck[float64](vec)[0] + config = strconv.FormatFloat(p, 'f', -1, 64) + case types.T_float32: + p = float64(vector.MustFixedColWithTypeCheck[float32](vec)[0]) + config = strconv.FormatFloat(p, 'f', -1, 32) + case types.T_int64: + v := vector.MustFixedColWithTypeCheck[int64](vec)[0] + p = float64(v) + config = strconv.FormatInt(v, 10) + case types.T_int32: + v := vector.MustFixedColWithTypeCheck[int32](vec)[0] + p = float64(v) + config = strconv.FormatInt(int64(v), 10) + case types.T_decimal64: + d := vector.MustFixedColWithTypeCheck[types.Decimal64](vec)[0] + p = types.Decimal64ToFloat64(d, vec.GetType().Scale) + config = d.Format(vec.GetType().Scale) + case types.T_decimal128: + d := vector.MustFixedColWithTypeCheck[types.Decimal128](vec)[0] + p = types.Decimal128ToFloat64(d, vec.GetType().Scale) + config = d.Format(vec.GetType().Scale) + default: + return nil, moerr.NewInvalidInputNoCtxf( + "unsupported percentile type %s for approx_percentile", vec.GetType().String()) + } + if math.IsNaN(p) || math.IsInf(p, 0) || p < 0 || p > 1 { + return nil, moerr.NewInvalidInputNoCtxf( + "percentile argument of approx_percentile must be finite and in [0,1], got %v", p) + } + exact, ok := new(big.Rat).SetString(config) + if !ok || exact.Sign() < 0 || exact.Cmp(big.NewRat(1, 1)) > 0 { + return nil, moerr.NewInvalidInputNoCtxf( + "percentile argument of approx_percentile must be in [0,1], got %s", config) + } + return []byte(config), nil +} + type cloneIndexAutoIncrementTable struct { srcTableName string sourceKey string diff --git a/pkg/sql/compile/operator_test.go b/pkg/sql/compile/operator_test.go index 4284e11c2690d..797f0f0f7f4a5 100644 --- a/pkg/sql/compile/operator_test.go +++ b/pkg/sql/compile/operator_test.go @@ -16,9 +16,12 @@ package compile import ( "context" + "math" "testing" + "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/colexec/aggexec" "github.com/matrixorigin/matrixone/pkg/sql/colexec/dedupjoin" @@ -274,6 +277,133 @@ func TestConstructTimeWindowUsesRegularSumForPartialSum(t *testing.T) { require.Equal(t, types.T_decimal128, timeWin.Types[0].Oid) } +func TestConstructTimeWindowApproxPercentileConfig(t *testing.T) { + proc := testutil.NewProcessWithMPool(t, "", mpool.MustNewZero()) + defer proc.Free() + + fn, err := function.GetFunctionByName(context.Background(), plan2.NameApproxPercentile, []types.Type{ + types.T_int32.ToType(), types.T_float64.ToType(), + }) + require.NoError(t, err) + + for _, tc := range []struct { + name string + percentile *plan.Expr + want string + }{ + {name: "lower endpoint", percentile: plan2.MakePlan2Float64ConstExprWithType(0), want: "0"}, + {name: "upper endpoint", percentile: plan2.MakePlan2Float64ConstExprWithType(1), want: "1"}, + } { + t.Run(tc.name, func(t *testing.T) { + node := makeTimeWindowAggNode(fn.GetEncodedOverloadID(), plan2.NameApproxPercentile, tc.percentile) + arg := constructTimeWindow(context.Background(), node, proc) + require.Len(t, arg.Aggs, 1) + require.Len(t, arg.Aggs[0].GetArgExpressions(), 1) + require.Equal(t, tc.want, string(arg.Aggs[0].GetExtraConfig())) + }) + } +} + +func TestConstructTimeWindowApproxPercentileRejectsInvalidConfig(t *testing.T) { + proc := testutil.NewProcessWithMPool(t, "", mpool.MustNewZero()) + defer proc.Free() + + fn, err := function.GetFunctionByName(context.Background(), plan2.NameApproxPercentile, []types.Type{ + types.T_int32.ToType(), types.T_float64.ToType(), + }) + require.NoError(t, err) + nonConstant := &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_float64)}, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ColPos: 2}}, + } + + for _, tc := range []struct { + name string + percentile *plan.Expr + want string + }{ + { + name: "below range", + percentile: plan2.MakePlan2Float64ConstExprWithType(-0.01), + want: "invalid input: percentile argument of approx_percentile must be finite and in [0,1], got -0.01", + }, + { + name: "above range", + percentile: plan2.MakePlan2Float64ConstExprWithType(1.01), + want: "invalid input: percentile argument of approx_percentile must be finite and in [0,1], got 1.01", + }, + { + name: "non constant", + percentile: nonConstant, + want: "invalid input: percentile argument of approx_percentile must be a constant", + }, + } { + t.Run(tc.name, func(t *testing.T) { + node := makeTimeWindowAggNode(fn.GetEncodedOverloadID(), plan2.NameApproxPercentile, tc.percentile) + require.PanicsWithError(t, tc.want, func() { + constructTimeWindow(context.Background(), node, proc) + }) + }) + } +} + +func TestConstructAggregateConfigPreservesOtherSpecialConfigs(t *testing.T) { + proc := testutil.NewProcessWithMPool(t, "", mpool.MustNewZero()) + defer proc.Free() + proc.SetResolveVariableFunc(func(name string, system, global bool) (interface{}, error) { + require.Equal(t, "group_concat_max_len", name) + require.True(t, system) + require.False(t, global) + return int64(1024), nil + }) + value := &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_int32)}, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ColPos: 1}}, + } + + for _, tc := range []struct { + name string + config string + wantConfig []byte + }{ + {name: plan2.NameGroupConcat, config: "|", wantConfig: aggexec.EncodeGroupConcatConfig("|", 1024)}, + {name: plan2.NameClusterCenters, config: "k=3,init=random", wantConfig: []byte("k=3,init=random")}, + } { + t.Run(tc.name, func(t *testing.T) { + f := &plan.Function{ + Func: &plan.ObjectRef{ObjName: tc.name}, + Args: []*plan.Expr{value, plan2.MakePlan2StringConstExprWithType(tc.config)}, + } + args, config := constructAggregateConfig(f, proc) + require.Len(t, args, 1) + require.Equal(t, tc.wantConfig, config) + }) + } +} + +func makeTimeWindowAggNode(functionID int64, name string, config *plan.Expr) *plan.Node { + value := &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_int32)}, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ColPos: 1}}, + } + ts := &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_datetime)}, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ColPos: 0}}, + } + return &plan.Node{ + AggList: []*plan.Expr{{ + Typ: plan.Type{Id: int32(types.T_float64)}, + Expr: &plan.Expr_F{F: &plan.Function{ + Func: &plan.ObjectRef{Obj: functionID, ObjName: name}, + Args: []*plan.Expr{value, config}, + }}, + }}, + GroupBy: []*plan.Expr{ts}, + Timestamp: ts, + Interval: makeTimeWindowIntervalExpr(1, "second"), + } +} + func TestDupOperatorLoopJoinMarkPos(t *testing.T) { op := loopjoin.NewArgument() op.MarkPos = 3 @@ -359,6 +489,143 @@ func TestConstructShuffleOperatorForJoinSupportsColumnsAndExpressions(t *testing require.Nil(t, rightShuffle.RuntimeFilterSpec) } +func TestGetPercentileConfig(t *testing.T) { + mp, err := mpool.NewMPool("test_pct_config", 0, mpool.NoFixed) + require.NoError(t, err) + defer mpool.DeleteMPool(mp) + + t.Run("float64", func(t *testing.T) { + vec, err := vector.NewConstFixed(types.T_float64.ToType(), float64(0.95), 1, mp) + require.NoError(t, err) + defer vec.Free(mp) + cfg, err := getPercentileConfig(vec) + require.NoError(t, err) + require.Equal(t, "0.95", string(cfg)) + }) + + t.Run("float32", func(t *testing.T) { + vec, err := vector.NewConstFixed(types.T_float32.ToType(), float32(0.5), 1, mp) + require.NoError(t, err) + defer vec.Free(mp) + cfg, err := getPercentileConfig(vec) + require.NoError(t, err) + require.Equal(t, "0.5", string(cfg)) + }) + + t.Run("int64", func(t *testing.T) { + vec, err := vector.NewConstFixed(types.T_int64.ToType(), int64(0), 1, mp) + require.NoError(t, err) + defer vec.Free(mp) + cfg, err := getPercentileConfig(vec) + require.NoError(t, err) + require.Equal(t, "0", string(cfg)) + }) + + t.Run("int32", func(t *testing.T) { + vec, err := vector.NewConstFixed(types.T_int32.ToType(), int32(1), 1, mp) + require.NoError(t, err) + defer vec.Free(mp) + cfg, err := getPercentileConfig(vec) + require.NoError(t, err) + require.Equal(t, "1", string(cfg)) + }) + + t.Run("decimal64 preserves exact text", func(t *testing.T) { + typ := types.New(types.T_decimal64, 10, 6) + value, err := types.ParseDecimal64("0.123456", typ.Width, typ.Scale) + require.NoError(t, err) + vec, err := vector.NewConstFixed(typ, value, 1, mp) + require.NoError(t, err) + defer vec.Free(mp) + cfg, err := getPercentileConfig(vec) + require.NoError(t, err) + require.Equal(t, "0.123456", string(cfg)) + }) + + t.Run("decimal128 preserves exact text", func(t *testing.T) { + typ := types.New(types.T_decimal128, 38, 30) + value, err := types.ParseDecimal128("0.123456789012345678901234567890", typ.Width, typ.Scale) + require.NoError(t, err) + vec, err := vector.NewConstFixed(typ, value, 1, mp) + require.NoError(t, err) + defer vec.Free(mp) + cfg, err := getPercentileConfig(vec) + require.NoError(t, err) + require.Equal(t, "0.123456789012345678901234567890", string(cfg)) + }) +} + +func TestGetPercentileConfigRejectsInvalidVectors(t *testing.T) { + mp, err := mpool.NewMPool("test_pct_config_invalid", 0, mpool.NoFixed) + require.NoError(t, err) + defer mpool.DeleteMPool(mp) + + flat := vector.NewVec(types.T_float64.ToType()) + require.NoError(t, vector.AppendFixed(flat, 0.5, false, mp)) + nullVec := vector.NewConstNull(types.T_float64.ToType(), 1, mp) + unsupported, err := vector.NewConstBytes(types.T_varchar.ToType(), []byte("0.5"), 1, mp) + require.NoError(t, err) + below, err := vector.NewConstFixed(types.T_float64.ToType(), -0.1, 1, mp) + require.NoError(t, err) + above, err := vector.NewConstFixed(types.T_float64.ToType(), 1.1, 1, mp) + require.NoError(t, err) + nan, err := vector.NewConstFixed(types.T_float64.ToType(), math.NaN(), 1, mp) + require.NoError(t, err) + inf, err := vector.NewConstFixed(types.T_float64.ToType(), math.Inf(1), 1, mp) + require.NoError(t, err) + decimalType := types.New(types.T_decimal128, 38, 37) + decimalAboveValue, err := types.ParseDecimal128( + "1.0000000000000000000000000000000000001", decimalType.Width, decimalType.Scale) + require.NoError(t, err) + decimalAbove, err := vector.NewConstFixed(decimalType, decimalAboveValue, 1, mp) + require.NoError(t, err) + + tests := []struct { + name string + vec *vector.Vector + }{ + {name: "non-constant", vec: flat}, + {name: "null", vec: nullVec}, + {name: "unsupported", vec: unsupported}, + {name: "below range", vec: below}, + {name: "above range", vec: above}, + {name: "nan", vec: nan}, + {name: "infinity", vec: inf}, + {name: "decimal above range beyond float64 precision", vec: decimalAbove}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + defer tc.vec.Free(mp) + require.NotPanics(t, func() { + _, err := getPercentileConfig(tc.vec) + require.Error(t, err) + }) + }) + } +} + +func TestValidateApproxPercentileExpr(t *testing.T) { + column := &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_float64)}, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ColPos: 1}}, + } + parameter := &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_float64)}, + Expr: &plan.Expr_P{P: &plan.ParamRef{Pos: 0}}, + } + + require.Error(t, validateApproxPercentileExpr(nil)) + require.Error(t, validateApproxPercentileExpr(column)) + literal := &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_int64)}, + Expr: &plan.Expr_Lit{Lit: &plan.Literal{ + Value: &plan.Literal_I64Val{I64Val: 1}, + }}, + } + require.NoError(t, validateApproxPercentileExpr(literal)) + require.Error(t, validateApproxPercentileExpr(parameter)) +} + func makeTimeWindowIntervalExpr(value int64, unit string) *plan.Expr { return &plan.Expr{ Expr: &plan.Expr_List{ diff --git a/pkg/sql/compile/remoterun_test.go b/pkg/sql/compile/remoterun_test.go index c2997c4b0b600..ec24e5f6581a0 100644 --- a/pkg/sql/compile/remoterun_test.go +++ b/pkg/sql/compile/remoterun_test.go @@ -40,7 +40,6 @@ import ( planpb "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/pb/txn" "github.com/matrixorigin/matrixone/pkg/sql/colexec" - "github.com/matrixorigin/matrixone/pkg/sql/colexec/aggexec" "github.com/matrixorigin/matrixone/pkg/sql/colexec/apply" "github.com/matrixorigin/matrixone/pkg/sql/colexec/connector" "github.com/matrixorigin/matrixone/pkg/sql/colexec/dedupjoin" @@ -981,7 +980,6 @@ func Test_convertToProcessSessionInfo(t *testing.T) { func Test_decodeBatch(t *testing.T) { mp := &mpool.MPool{} - aggexec.RegisterGroupConcatAgg(0, ",") bat := &batch.Batch{ Recursive: 0, ShuffleIDX: 0, diff --git a/pkg/sql/plan/base_binder.go b/pkg/sql/plan/base_binder.go index 713a7620a8b24..b698e61aad2ec 100644 --- a/pkg/sql/plan/base_binder.go +++ b/pkg/sql/plan/base_binder.go @@ -31,6 +31,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/plan/function" + "github.com/matrixorigin/matrixone/pkg/sql/plan/rule" "github.com/matrixorigin/matrixone/pkg/sql/util" "github.com/matrixorigin/matrixone/pkg/util/errutil" "github.com/matrixorigin/matrixone/pkg/vm/process" @@ -2857,6 +2858,18 @@ func bindSerialFuncOverExprList(ctx context.Context, name string, args []*Expr) return args[0], true, nil } +func validateApproxPercentileArgs(ctx context.Context, args []*Expr) error { + if len(args) != 2 { + return nil + } + percentile := args[1] + if percentile == nil || isNullExpr(percentile) || !rule.IsConstant(percentile, false) { + return moerr.NewInvalidInput(ctx, + "percentile argument of approx_percentile must be a non-null constant") + } + return nil +} + // bindMixedInListComparison applies MySQL's REAL comparison semantics for a // string left operand and a numeric IN-list value. It is deliberately limited // to IN/NOT IN fallback comparisons: applying it to every comparison would @@ -2881,6 +2894,11 @@ func bindMixedInListComparison(ctx context.Context, operator string, left, right func BindFuncExprImplByPlanExpr(ctx context.Context, name string, args []*Expr) (*plan.Expr, error) { var err error + if name == NameApproxPercentile { + if err = validateApproxPercentileArgs(ctx, args); err != nil { + return nil, err + } + } // deal with some special function if listExpr, ok, err := bindSerialFuncOverExprList(ctx, name, args); ok || err != nil { diff --git a/pkg/sql/plan/base_binder_approx_percentile_test.go b/pkg/sql/plan/base_binder_approx_percentile_test.go new file mode 100644 index 0000000000000..ceecbe6d2cf16 --- /dev/null +++ b/pkg/sql/plan/base_binder_approx_percentile_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "context" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + planpb "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" + "github.com/stretchr/testify/require" +) + +func approxPercentileValueColumn() *planpb.Expr { + return &planpb.Expr{ + Typ: planpb.Type{Id: int32(types.T_int64)}, + Expr: &planpb.Expr_Col{Col: &planpb.ColRef{ + ColPos: 0, + Name: "v", + }}, + } +} + +func TestBindApproxPercentileRequiresStableNonNullPercentile(t *testing.T) { + ctx := context.Background() + percentileColumn := &planpb.Expr{ + Typ: planpb.Type{Id: int32(types.T_float64)}, + Expr: &planpb.Expr_Col{Col: &planpb.ColRef{ + ColPos: 1, + Name: "p", + }}, + } + + _, err := BindFuncExprImplByPlanExpr(ctx, "approx_percentile", []*planpb.Expr{ + approxPercentileValueColumn(), + makePlan2NullConstExprWithType(), + }) + require.ErrorContains(t, err, "percentile argument of approx_percentile must be a non-null constant") + + _, err = BindFuncExprImplByPlanExpr(ctx, "approx_percentile", []*planpb.Expr{ + approxPercentileValueColumn(), + nil, + }) + require.ErrorContains(t, err, "percentile argument of approx_percentile must be a non-null constant") + + _, err = BindFuncExprImplByPlanExpr(ctx, "approx_percentile", []*planpb.Expr{ + approxPercentileValueColumn(), + percentileColumn, + }) + require.ErrorContains(t, err, "percentile argument of approx_percentile must be a non-null constant") +} + +func TestBindApproxPercentileAcceptsFoldableConstants(t *testing.T) { + ctx := context.Background() + foldable, err := BindFuncExprImplByPlanExpr(ctx, "+", []*planpb.Expr{ + makePlan2Float64ConstExprWithType(0.4), + makePlan2Float64ConstExprWithType(0.1), + }) + require.NoError(t, err) + + for _, percentile := range []*planpb.Expr{ + makePlan2Float64ConstExprWithType(0.95), + foldable, + } { + _, err = BindFuncExprImplByPlanExpr(ctx, "approx_percentile", []*planpb.Expr{ + approxPercentileValueColumn(), + percentile, + }) + require.NoError(t, err) + } + + parameter := &planpb.Expr{ + Typ: planpb.Type{Id: int32(types.T_text)}, + Expr: &planpb.Expr_P{P: &planpb.ParamRef{Pos: 0}}, + } + _, err = BindFuncExprImplByPlanExpr(ctx, "approx_percentile", []*planpb.Expr{ + approxPercentileValueColumn(), + parameter, + }) + require.ErrorContains(t, err, "must be a non-null constant") +} + +func TestBuildPlanApproxPercentileRejectsInvalidPercentileSQL(t *testing.T) { + ctx := NewMockCompilerContext(true) + tests := []string{ + "select approx_percentile(a, null) from select_test.bind_select", + "select approx_percentile(a, b) from select_test.bind_select", + "select approx_percentile(a, null) over () from select_test.bind_select", + "select approx_percentile(a, b) over () from select_test.bind_select", + } + + for _, sql := range tests { + t.Run(sql, func(t *testing.T) { + stmts, err := parsers.Parse(context.Background(), dialect.MYSQL, sql, 1) + require.NoError(t, err) + _, err = BuildPlan(ctx, stmts[0], false) + require.ErrorContains(t, err, + "percentile argument of approx_percentile must be a non-null constant") + }) + } +} diff --git a/pkg/sql/plan/function/func_approx_percentile_test.go b/pkg/sql/plan/function/func_approx_percentile_test.go new file mode 100644 index 0000000000000..d8406a4ea6521 --- /dev/null +++ b/pkg/sql/plan/function/func_approx_percentile_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package function + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +func TestApproxPercentileCheckFn(t *testing.T) { + fn := allSupportedFunctions[APPROX_PERCENTILE] + require.NotNil(t, fn.checkFn) + + check := fn.checkFn + + // wrong number of arguments + result := check(nil, []types.Type{}) + require.Equal(t, failedAggParametersWrong, result.status) + + result = check(nil, []types.Type{types.T_int64.ToType()}) + require.Equal(t, failedAggParametersWrong, result.status) + + // T_any in arg0 should request cast + result = check(nil, []types.Type{types.T_any.ToType(), types.T_float64.ToType()}) + require.Equal(t, succeedWithCast, result.status) + require.Equal(t, 0, result.idx) + + // unsupported type in arg0 + result = check(nil, []types.Type{types.T_varchar.ToType(), types.T_float64.ToType()}) + require.Equal(t, failedAggParametersWrong, result.status) + + // T_any in arg1 should request cast + result = check(nil, []types.Type{types.T_int64.ToType(), types.T_any.ToType()}) + require.Equal(t, succeedWithCast, result.status) + + // unsupported type in arg1 + result = check(nil, []types.Type{types.T_int64.ToType(), types.T_varchar.ToType()}) + require.Equal(t, failedAggParametersWrong, result.status) + + // valid types + for _, oid := range []types.T{ + types.T_bit, types.T_int8, types.T_int16, types.T_int32, types.T_int64, + types.T_uint8, types.T_uint16, types.T_uint32, types.T_uint64, + types.T_float32, types.T_float64, + types.T_decimal64, types.T_decimal128, + } { + result = check(nil, []types.Type{oid.ToType(), types.T_float64.ToType()}) + require.Equal(t, succeedMatched, result.status, "oid %d should succeed", oid) + } + + // valid arg1 types + for _, arg1Oid := range []types.T{ + types.T_int32, types.T_int64, types.T_float32, types.T_float64, + types.T_decimal64, types.T_decimal128, + } { + result = check(nil, []types.Type{types.T_int64.ToType(), arg1Oid.ToType()}) + require.Equal(t, succeedMatched, result.status, "arg1 oid %d should succeed", arg1Oid) + } +} + +func TestApproxPercentileCatalogEntry(t *testing.T) { + fn := allSupportedFunctions[APPROX_PERCENTILE] + require.Equal(t, APPROX_PERCENTILE, fn.functionId) + require.True(t, fn.isAggregate()) + require.Equal(t, 1, len(fn.Overloads)) + require.True(t, fn.Overloads[0].isAgg) + require.Equal(t, "approx_percentile", fn.Overloads[0].aggName) + require.Equal(t, types.New(types.T_decimal128, 38, 0), + fn.Overloads[0].retType([]types.Type{types.New(types.T_decimal128, 38, 0)})) + require.Equal(t, types.New(types.T_decimal128, 38, 38), + fn.Overloads[0].retType([]types.Type{types.New(types.T_decimal128, 38, 38)})) +} diff --git a/pkg/sql/plan/function/function.go b/pkg/sql/plan/function/function.go index ec5a1d3a7bc27..4a3c3993865d1 100644 --- a/pkg/sql/plan/function/function.go +++ b/pkg/sql/plan/function/function.go @@ -52,15 +52,9 @@ func initAllSupportedFunctions() { } for _, fn := range supportedWindowInNewFramework { - for _, ov := range fn.Overloads { - ov.aggFramework.aggRegister(encodeOverloadID(int32(fn.functionId), int32(ov.overloadId))) - } allSupportedFunctions[fn.functionId] = fn } for _, fn := range supportedAggInNewFramework { - for _, ov := range fn.Overloads { - ov.aggFramework.aggRegister(encodeOverloadID(int32(fn.functionId), int32(ov.overloadId))) - } allSupportedFunctions[fn.functionId] = fn } } @@ -280,7 +274,7 @@ func GetAggFunctionNameByID(overloadID int64) string { if !exist { return "unknown function" } - return f.aggFramework.str + return f.aggName } // DeduceNotNullable helps optimization sometimes. @@ -428,14 +422,6 @@ type executeFreeOfOverload func() error // in case we need it in the future. type executeResetOfOverload func() error -type aggregationLogicOfOverload struct { - // agg related string for error message. - str string - - // how to register the aggregation. - aggRegister func(overloadID int64) -} - // an overload of a function. // stores all information about execution logic. type overload struct { @@ -463,9 +449,11 @@ type overload struct { // in fact, the function framework does not directly run aggregate functions and window functions. // we use two flags to mark whether function is one of them. - isAgg bool - isWin bool - aggFramework aggregationLogicOfOverload + isAgg bool + isWin bool + + // aggName is used in aggregate-related error messages. + aggName string // if true, overload was unable to run in parallel. // For example, diff --git a/pkg/sql/plan/function/function_id.go b/pkg/sql/plan/function/function_id.go index c46465f309f9a..c2e27da258fcc 100644 --- a/pkg/sql/plan/function/function_id.go +++ b/pkg/sql/plan/function/function_id.go @@ -785,11 +785,12 @@ const ( VECF16_FROM_BASE64 = 551 VECINT8_FROM_BASE64 = 552 VECUINT8_FROM_BASE64 = 553 - // function `cast_assign` CAST_ASSIGN = 554 // function `cast_ignore` CAST_IGNORE = 555 + // function `approx_percentile` + APPROX_PERCENTILE = 557 // onnx_run: evaluate an ONNX model. Renumbered as main merges claim ids // (549->554->556); referenced by name only, so renumbering is safe. @@ -797,7 +798,7 @@ const ( // FUNCTION_END_NUMBER is not a function, just a flag to record the max number of function. // TODO: every one should put the new function id in front of this one if you want to make a new function. - FUNCTION_END_NUMBER = 557 + FUNCTION_END_NUMBER = 558 ) // functionIdRegister is what function we have registered already. @@ -894,6 +895,7 @@ var functionIdRegister = map[string]int32{ "hll_merge_agg": HLL_MERGE_AGG, "any_value": ANY_VALUE, "median": MEDIAN, + "approx_percentile": APPROX_PERCENTILE, // count window "rank": RANK, "row_number": ROW_NUMBER, diff --git a/pkg/sql/plan/function/function_id_test.go b/pkg/sql/plan/function/function_id_test.go index 5ac1553e06519..fbe36ae47d444 100644 --- a/pkg/sql/plan/function/function_id_test.go +++ b/pkg/sql/plan/function/function_id_test.go @@ -17,9 +17,67 @@ package function import ( "testing" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/aggexec" "github.com/stretchr/testify/require" ) +func TestAggregateExecutorIDs(t *testing.T) { + tests := []struct { + name string + functionID int + executorID int64 + }{ + {"any_value", ANY_VALUE, aggexec.AggIdOfAny}, + {"approx_count", APPROX_COUNT, aggexec.AggIdOfApproxCount}, + {"avg", AVG, aggexec.AggIdOfAvg}, + {"avg_tw_cache", AVG_TW_CACHE, aggexec.AggIdOfAvgTwCache}, + {"avg_tw_result", AVG_TW_RESULT, aggexec.AggIdOfAvgTwResult}, + {"bit_and", BIT_AND, aggexec.AggIdOfBitAnd}, + {"bit_or", BIT_OR, aggexec.AggIdOfBitOr}, + {"bit_xor", BIT_XOR, aggexec.AggIdOfBitXor}, + {"count", COUNT, aggexec.AggIdOfCountColumn}, + {"cume_dist", CUME_DIST, aggexec.WinIdOfCumeDist}, + {"dense_rank", DENSE_RANK, aggexec.WinIdOfDenseRank}, + {"first_value", FIRST_VALUE, aggexec.WinIdOfFirstValue}, + {"lag", LAG, aggexec.WinIdOfLag}, + {"last_value", LAST_VALUE, aggexec.WinIdOfLastValue}, + {"lead", LEAD, aggexec.WinIdOfLead}, + {"max", MAX, aggexec.AggIdOfMax}, + {"median", MEDIAN, aggexec.AggIdOfMedian}, + {"min", MIN, aggexec.AggIdOfMin}, + {"nth_value", NTH_VALUE, aggexec.WinIdOfNthValue}, + {"ntile", NTILE, aggexec.WinIdOfNtile}, + {"percent_rank", PERCENT_RANK, aggexec.WinIdOfPercentRank}, + {"rank", RANK, aggexec.WinIdOfRank}, + {"row_number", ROW_NUMBER, aggexec.WinIdOfRowNumber}, + {"count_star", STARCOUNT, aggexec.AggIdOfCountStar}, + {"stddev_pop", STDDEV_POP, aggexec.AggIdOfStdDevPop}, + {"stddev_sample", STDDEV_SAMPLE, aggexec.AggIdOfStdDevSample}, + {"sum", SUM, aggexec.AggIdOfSum}, + {"group_concat", GROUP_CONCAT, aggexec.AggIdOfGroupConcat}, + {"var_pop", VAR_POP, aggexec.AggIdOfVarPop}, + {"var_sample", VAR_SAMPLE, aggexec.AggIdOfVarSample}, + {"approx_count_distinct", APPROX_COUNT_DISTINCT, aggexec.AggIdOfApproxCountDistinct}, + {"bitmap_construct_agg", BITMAP_CONSTRUCT_AGG, aggexec.AggIdOfBitmapConstruct}, + {"bitmap_or_agg", BITMAP_OR_AGG, aggexec.AggIdOfBitmapOr}, + {"json_arrayagg", JSON_ARRAYAGG, aggexec.AggIdOfJsonArrayAgg}, + {"json_objectagg", JSON_OBJECTAGG, aggexec.AggIdOfJsonObjectAgg}, + {"hll_add_agg", HLL_ADD_AGG, aggexec.AggIdOfHllAdd}, + {"hll_merge_agg", HLL_MERGE_AGG, aggexec.AggIdOfHllMerge}, + {"approx_percentile", APPROX_PERCENTILE, aggexec.AggIdOfApproxPercentile}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + overloads := allSupportedFunctions[test.functionID].Overloads + require.Len(t, overloads, 1) + require.Equal(t, + encodeOverloadID(int32(test.functionID), int32(overloads[0].overloadId)), + test.executorID) + }) + } +} + // all fixed function ids defined at 2024-12-12 var predefinedFunids = map[int]int{ EQUAL: 0, @@ -610,10 +668,11 @@ var predefinedFunids = map[int]int{ CAST_ASSIGN: 554, CAST_IGNORE: 555, ONNX_RUN: 556, + APPROX_PERCENTILE: 557, // FUNCTION_END_NUMBER is not a function, just a flag to record the max number of function. // TODO: every one should put the new function id in front of this one if you want to make a new function. - FUNCTION_END_NUMBER: 557, + FUNCTION_END_NUMBER: 558, } func Test_funids(t *testing.T) { diff --git a/pkg/sql/plan/function/list_agg.go b/pkg/sql/plan/function/list_agg.go index dee8768ff5d07..f624fdeb4446f 100644 --- a/pkg/sql/plan/function/list_agg.go +++ b/pkg/sql/plan/function/list_agg.go @@ -20,10 +20,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/colexec/aggexec" ) -func registerGroupConcatWithDefaultSeparator(id int64) { - aggexec.RegisterGroupConcatAgg(id, ",") -} - var supportedAggInNewFramework = []FuncNew{ { functionId: COUNT, @@ -60,10 +56,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "count", - aggRegister: aggexec.RegisterCountColumnAgg, - }, + aggName: "count", }, }, }, @@ -89,10 +82,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "count(*)", - aggRegister: aggexec.RegisterCountStarAgg, - }, + aggName: "count(*)", }, }, }, @@ -110,10 +100,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: ReturnFirstArgType, - aggFramework: aggregationLogicOfOverload{ - str: "min", - aggRegister: aggexec.RegisterMin, - }, + aggName: "min", }, }, }, @@ -131,10 +118,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: ReturnFirstArgType, - aggFramework: aggregationLogicOfOverload{ - str: "max", - aggRegister: aggexec.RegisterMax, - }, + aggName: "max", }, }, }, @@ -152,10 +136,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.SumReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "sum", - aggRegister: aggexec.RegisterSum, - }, + aggName: "sum", }, }, }, @@ -173,10 +154,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.AvgReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "avg", - aggRegister: aggexec.RegisterAvg, - }, + aggName: "avg", }, }, }, @@ -194,10 +172,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.AvgTwCacheReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "avg_tw_cache", - aggRegister: aggexec.RegisterAvgTwCache, - }, + aggName: "avg_tw_cache", }, }, }, @@ -215,10 +190,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.AvgTwResultReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "avg_tw_result", - aggRegister: aggexec.RegisterAvgTwResult, - }, + aggName: "avg_tw_result", }, }, }, @@ -256,10 +228,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.GroupConcatReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "group_concat", - aggRegister: registerGroupConcatWithDefaultSeparator, - }, + aggName: "group_concat", }, }, }, @@ -288,10 +257,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_json.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "json_arrayagg", - aggRegister: aggexec.RegisterJsonArrayAgg, - }, + aggName: "json_arrayagg", }, }, }, @@ -325,10 +291,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_json.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "json_objectagg", - aggRegister: aggexec.RegisterJsonObjectAgg, - }, + aggName: "json_objectagg", }, }, }, @@ -354,10 +317,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_uint64.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "approx_count", - aggRegister: aggexec.RegisterApproxCountAgg, - }, + aggName: "approx_count", }, }, }, @@ -384,10 +344,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_uint64.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "approx_count_distinct", - aggRegister: aggexec.RegisterApproxCountAgg, - }, + aggName: "approx_count_distinct", }, }, }, @@ -405,10 +362,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: ReturnFirstArgType, - aggFramework: aggregationLogicOfOverload{ - str: "any_value", - aggRegister: aggexec.RegisterAny, - }, + aggName: "any_value", }, }, }, @@ -426,10 +380,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: BitOpsReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "bit_and", - aggRegister: aggexec.RegisterBitAndAgg, - }, + aggName: "bit_and", }, }, }, @@ -447,10 +398,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: BitOpsReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "bit_or", - aggRegister: aggexec.RegisterBitOrAgg, - }, + aggName: "bit_or", }, }, }, @@ -468,10 +416,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: BitOpsReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "bit_xor", - aggRegister: aggexec.RegisterBitXorAgg, - }, + aggName: "bit_xor", }, }, }, @@ -489,10 +434,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.VarStdDevReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "var_pop", - aggRegister: aggexec.RegisterVarPop, - }, + aggName: "var_pop", }, }, }, @@ -510,10 +452,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.VarStdDevReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "stddev_pop", - aggRegister: aggexec.RegisterStdDevPop, - }, + aggName: "stddev_pop", }, }, }, @@ -531,10 +470,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.VarStdDevReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "var_sample", - aggRegister: aggexec.RegisterVarSample, - }, + aggName: "var_sample", }, }, }, @@ -552,10 +488,7 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.VarStdDevReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "stddev_sample", - aggRegister: aggexec.RegisterStdDevSample, - }, + aggName: "stddev_sample", }, }, }, @@ -573,10 +506,58 @@ var supportedAggInNewFramework = []FuncNew{ overloadId: 0, isAgg: true, retType: aggexec.MedianReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "median", - aggRegister: aggexec.RegisterMedian, - }, + aggName: "median", + }, + }, + }, + { + functionId: APPROX_PERCENTILE, + class: plan.Function_AGG, + layout: STANDARD_FUNCTION, + checkFn: func(overloads []overload, inputs []types.Type) checkResult { + if len(inputs) != 2 { + return newCheckResultWithFailure(failedAggParametersWrong) + } + + // check Arg[0]: must be numeric (same as median) + t0 := inputs[0] + if t0.Oid == types.T_any { + // cast to first supported type + return newCheckResultWithCast(0, []types.Type{aggexec.MedianSupportedType[0].ToType(), types.T_float64.ToType()}) + } + + supported := false + for _, st := range aggexec.MedianSupportedType { + if t0.Oid == st { + supported = true + break + } + } + if !supported { + return newCheckResultWithFailure(failedAggParametersWrong) + } + + // check Arg[1]: must be a supported integer, float, or decimal type + t1 := inputs[1] + if t1.Oid == types.T_any { + return newCheckResultWithCast(0, []types.Type{inputs[0], types.T_float64.ToType()}) + } + + switch t1.Oid { + case types.T_int32, types.T_int64, types.T_float32, types.T_float64, types.T_decimal64, types.T_decimal128: + default: + return newCheckResultWithFailure(failedAggParametersWrong) + } + + return newCheckResultWithSuccess(0) + }, + + Overloads: []overload{ + { + overloadId: 0, + isAgg: true, + retType: aggexec.ApproxPercentileReturnType, + aggName: "approx_percentile", }, }, }, @@ -598,11 +579,8 @@ var supportedAggInNewFramework = []FuncNew{ return types.T_varbinary.ToType() }, - isAgg: true, - aggFramework: aggregationLogicOfOverload{ - str: "bitmap_construct_agg", - aggRegister: aggexec.RegisterBitmapConstruct, - }, + isAgg: true, + aggName: "bitmap_construct_agg", }, }, }, @@ -624,11 +602,8 @@ var supportedAggInNewFramework = []FuncNew{ return types.T_varbinary.ToType() }, - isAgg: true, - aggFramework: aggregationLogicOfOverload{ - str: "bitmap_or_agg", - aggRegister: aggexec.RegisterBitmapOr, - }, + isAgg: true, + aggName: "bitmap_or_agg", }, }, }, @@ -655,10 +630,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_varbinary.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "hll_add_agg", - aggRegister: aggexec.RegisterHllAddAgg, - }, + aggName: "hll_add_agg", }, }, }, @@ -688,10 +660,7 @@ var supportedAggInNewFramework = []FuncNew{ retType: func(parameters []types.Type) types.Type { return types.T_varbinary.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "hll_merge_agg", - aggRegister: aggexec.RegisterHllMergeAgg, - }, + aggName: "hll_merge_agg", }, }, }, diff --git a/pkg/sql/plan/function/list_window.go b/pkg/sql/plan/function/list_window.go index 027c30fa78046..23dfd6c5d17f2 100644 --- a/pkg/sql/plan/function/list_window.go +++ b/pkg/sql/plan/function/list_window.go @@ -36,10 +36,7 @@ var supportedWindowInNewFramework = []FuncNew{ overloadId: 0, isWin: true, retType: aggexec.SingleWindowReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "rank", - aggRegister: aggexec.RegisterRankWin, - }, + aggName: "rank", }, }, }, @@ -58,10 +55,7 @@ var supportedWindowInNewFramework = []FuncNew{ overloadId: 0, isWin: true, retType: aggexec.SingleWindowReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "row_number", - aggRegister: aggexec.RegisterRowNumberWin, - }, + aggName: "row_number", }, }, }, @@ -80,10 +74,7 @@ var supportedWindowInNewFramework = []FuncNew{ overloadId: 0, isWin: true, retType: aggexec.SingleWindowReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "dense_rank", - aggRegister: aggexec.RegisterDenseRankWin, - }, + aggName: "dense_rank", }, }, }, @@ -104,10 +95,7 @@ var supportedWindowInNewFramework = []FuncNew{ retType: func(_ []types.Type) types.Type { return types.T_float64.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "percent_rank", - aggRegister: aggexec.RegisterPercentRankWin, - }, + aggName: "percent_rank", }, }, }, @@ -126,10 +114,7 @@ var supportedWindowInNewFramework = []FuncNew{ overloadId: 0, isWin: true, retType: aggexec.SingleWindowReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "ntile", - aggRegister: aggexec.RegisterNtileWin, - }, + aggName: "ntile", }, }, }, @@ -148,10 +133,7 @@ var supportedWindowInNewFramework = []FuncNew{ overloadId: 0, isWin: true, retType: aggexec.CumeDistReturnType, - aggFramework: aggregationLogicOfOverload{ - str: "cume_dist", - aggRegister: aggexec.RegisterCumeDistWin, - }, + aggName: "cume_dist", }, }, }, @@ -178,10 +160,7 @@ var supportedWindowInNewFramework = []FuncNew{ } return types.T_any.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "lag", - aggRegister: aggexec.RegisterLagWin, - }, + aggName: "lag", }, }, }, @@ -208,10 +187,7 @@ var supportedWindowInNewFramework = []FuncNew{ } return types.T_any.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "lead", - aggRegister: aggexec.RegisterLeadWin, - }, + aggName: "lead", }, }, }, @@ -236,10 +212,7 @@ var supportedWindowInNewFramework = []FuncNew{ } return types.T_any.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "first_value", - aggRegister: aggexec.RegisterFirstValueWin, - }, + aggName: "first_value", }, }, }, @@ -264,10 +237,7 @@ var supportedWindowInNewFramework = []FuncNew{ } return types.T_any.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "last_value", - aggRegister: aggexec.RegisterLastValueWin, - }, + aggName: "last_value", }, }, }, @@ -293,10 +263,7 @@ var supportedWindowInNewFramework = []FuncNew{ } return types.T_any.ToType() }, - aggFramework: aggregationLogicOfOverload{ - str: "nth_value", - aggRegister: aggexec.RegisterNthValueWin, - }, + aggName: "nth_value", }, }, }, diff --git a/pkg/sql/plan/query_builder.go b/pkg/sql/plan/query_builder.go index 9606468049c0f..4b5ca1e578a3a 100644 --- a/pkg/sql/plan/query_builder.go +++ b/pkg/sql/plan/query_builder.go @@ -3738,6 +3738,7 @@ func (builder *QueryBuilder) numericSetProjectionTypes(ctx *BindContext, stmts [ const NameGroupConcat = "group_concat" const NameClusterCenters = "cluster_centers" +const NameApproxPercentile = "approx_percentile" func (builder *QueryBuilder) bindNoRecursiveCte( ctx *BindContext, diff --git a/test/distributed/cases/function/func_aggr_approx_percentile.result b/test/distributed/cases/function/func_aggr_approx_percentile.result new file mode 100644 index 0000000000000..f727b4e03bd66 --- /dev/null +++ b/test/distributed/cases/function/func_aggr_approx_percentile.result @@ -0,0 +1,77 @@ +select approx_percentile(null, 0.5); +➤ approx_percentile(null, 0.5)[8,54,0] 𝄀 +null +drop table if exists t1; +create table t1 (a int, b int); +insert into t1 values (1, 1), (2, 3), (3, 5), (4, 7), (5, 9); +select approx_percentile(b, 0.5) from t1; +➤ approx_percentile(b, 0.5)[8,54,0] 𝄀 +5.0 +select approx_percentile(b, 0.0) from t1; +➤ approx_percentile(b, 0.0)[8,54,0] 𝄀 +1.0 +select approx_percentile(b, 1.0) from t1; +➤ approx_percentile(b, 1.0)[8,54,0] 𝄀 +9.0 +insert into t1 values (1, 2), (1, 4), (2, 6), (2, 8); +select a, approx_percentile(b, 0.5) from t1 group by a order by a; +➤ a[4,32,0] ¦ approx_percentile(b, 0.5)[8,54,0] 𝄀 +1 ¦ 2.0 𝄀 +2 ¦ 6.0 𝄀 +3 ¦ 5.0 𝄀 +4 ¦ 7.0 𝄀 +5 ¦ 9.0 +drop table if exists t1; +create table t1 (a int, b float); +insert into t1 values (1, 1.1), (2, 2.2), (3, 3.3); +select approx_percentile(b, 0.5) from t1; +➤ approx_percentile(b, 0.5)[8,54,0] 𝄀 +2.2 +select approx_percentile(b, 0.95) from t1; +➤ approx_percentile(b, 0.95)[8,54,0] 𝄀 +3.19 +drop table t1; +drop table if exists t_time_window; +create table t_time_window (ts timestamp primary key, v int); +insert into t_time_window values ('2026-01-01 00:00:00', 1), ('2026-01-01 00:00:30', 3), ('2026-01-01 00:01:00', 5), ('2026-01-01 00:01:30', 7); +select _wstart, _wend, approx_percentile(v, 0), approx_percentile(v, 1) from t_time_window interval(ts, 1, minute); +➤ _wstart[93,64,0] ¦ _wend[93,64,0] ¦ approx_percentile(v, 0)[8,54,0] ¦ approx_percentile(v, 1)[8,54,0] 𝄀 +2026-01-01 00:00:00 ¦ 2026-01-01 00:01:00 ¦ 1.0 ¦ 3.0 𝄀 +2026-01-01 00:01:00 ¦ 2026-01-01 00:02:00 ¦ 5.0 ¦ 7.0 +drop table t_time_window; +drop table if exists t_approx_decimal38_0; +create table t_approx_decimal38_0 (v decimal(38, 0)); +insert into t_approx_decimal38_0 values (-99999999999999999999999999999999999999), (99999999999999999999999999999999999999); +select approx_percentile(v, 0), approx_percentile(v, 1) from t_approx_decimal38_0; +➤ approx_percentile(v, 0)[3,38,0] ¦ approx_percentile(v, 1)[3,38,0] 𝄀 +-99999999999999999999999999999999999999 ¦ 99999999999999999999999999999999999999 +truncate table t_approx_decimal38_0; +insert into t_approx_decimal38_0 values (0), (1); +select approx_percentile(v, 0.5) from t_approx_decimal38_0; +➤ approx_percentile(v, 0.5)[3,38,0] 𝄀 +1 +drop table t_approx_decimal38_0; +drop table if exists t_approx_decimal38_38; +create table t_approx_decimal38_38 (v decimal(38, 38)); +insert into t_approx_decimal38_38 values (0.1), (0.2); +select approx_percentile(v, 0), approx_percentile(v, 1) from t_approx_decimal38_38; +➤ approx_percentile(v, 0)[3,38,38] ¦ approx_percentile(v, 1)[3,38,38] 𝄀 +0.10000000000000000000000000000000000000 ¦ 0.20000000000000000000000000000000000000 +drop table t_approx_decimal38_38; +drop table if exists t_approx_float_infinity; +create table t_approx_float_infinity (v double); +insert into t_approx_float_infinity values (cast('Inf' as double)); +select approx_percentile(v, 0) = cast('Inf' as double) as p0_is_pos_inf, approx_percentile(v, 0.5) = cast('Inf' as double) as p50_is_pos_inf, approx_percentile(v, 1) = cast('Inf' as double) as p100_is_pos_inf from t_approx_float_infinity; +➤ p0_is_pos_inf[-7,1,0] ¦ p50_is_pos_inf[-7,1,0] ¦ p100_is_pos_inf[-7,1,0] 𝄀 +1 ¦ 1 ¦ 1 +truncate table t_approx_float_infinity; +insert into t_approx_float_infinity values (cast('-Inf' as double)); +select approx_percentile(v, 0) = cast('-Inf' as double) as p0_is_neg_inf, approx_percentile(v, 0.5) = cast('-Inf' as double) as p50_is_neg_inf, approx_percentile(v, 1) = cast('-Inf' as double) as p100_is_neg_inf from t_approx_float_infinity; +➤ p0_is_neg_inf[-7,1,0] ¦ p50_is_neg_inf[-7,1,0] ¦ p100_is_neg_inf[-7,1,0] 𝄀 +1 ¦ 1 ¦ 1 +truncate table t_approx_float_infinity; +insert into t_approx_float_infinity values (cast('-Inf' as double)), (cast('Inf' as double)); +select approx_percentile(v, 0) = cast('-Inf' as double) as p0_is_neg_inf, approx_percentile(v, 1) = cast('Inf' as double) as p100_is_pos_inf from t_approx_float_infinity; +➤ p0_is_neg_inf[-7,1,0] ¦ p100_is_pos_inf[-7,1,0] 𝄀 +1 ¦ 1 +drop table t_approx_float_infinity; diff --git a/test/distributed/cases/function/func_aggr_approx_percentile.test b/test/distributed/cases/function/func_aggr_approx_percentile.test new file mode 100644 index 0000000000000..16f391b5d734d --- /dev/null +++ b/test/distributed/cases/function/func_aggr_approx_percentile.test @@ -0,0 +1,51 @@ +select approx_percentile(null, 0.5); + +drop table if exists t1; +create table t1 (a int, b int); +insert into t1 values (1, 1), (2, 3), (3, 5), (4, 7), (5, 9); +select approx_percentile(b, 0.5) from t1; +select approx_percentile(b, 0.0) from t1; +select approx_percentile(b, 1.0) from t1; + +insert into t1 values (1, 2), (1, 4), (2, 6), (2, 8); +select a, approx_percentile(b, 0.5) from t1 group by a order by a; + +drop table if exists t1; +create table t1 (a int, b float); +insert into t1 values (1, 1.1), (2, 2.2), (3, 3.3); +select approx_percentile(b, 0.5) from t1; +select approx_percentile(b, 0.95) from t1; +drop table t1; + +drop table if exists t_time_window; +create table t_time_window (ts timestamp primary key, v int); +insert into t_time_window values ('2026-01-01 00:00:00', 1), ('2026-01-01 00:00:30', 3), ('2026-01-01 00:01:00', 5), ('2026-01-01 00:01:30', 7); +select _wstart, _wend, approx_percentile(v, 0), approx_percentile(v, 1) from t_time_window interval(ts, 1, minute); +drop table t_time_window; + +drop table if exists t_approx_decimal38_0; +create table t_approx_decimal38_0 (v decimal(38, 0)); +insert into t_approx_decimal38_0 values (-99999999999999999999999999999999999999), (99999999999999999999999999999999999999); +select approx_percentile(v, 0), approx_percentile(v, 1) from t_approx_decimal38_0; +truncate table t_approx_decimal38_0; +insert into t_approx_decimal38_0 values (0), (1); +select approx_percentile(v, 0.5) from t_approx_decimal38_0; +drop table t_approx_decimal38_0; + +drop table if exists t_approx_decimal38_38; +create table t_approx_decimal38_38 (v decimal(38, 38)); +insert into t_approx_decimal38_38 values (0.1), (0.2); +select approx_percentile(v, 0), approx_percentile(v, 1) from t_approx_decimal38_38; +drop table t_approx_decimal38_38; + +drop table if exists t_approx_float_infinity; +create table t_approx_float_infinity (v double); +insert into t_approx_float_infinity values (cast('Inf' as double)); +select approx_percentile(v, 0) = cast('Inf' as double) as p0_is_pos_inf, approx_percentile(v, 0.5) = cast('Inf' as double) as p50_is_pos_inf, approx_percentile(v, 1) = cast('Inf' as double) as p100_is_pos_inf from t_approx_float_infinity; +truncate table t_approx_float_infinity; +insert into t_approx_float_infinity values (cast('-Inf' as double)); +select approx_percentile(v, 0) = cast('-Inf' as double) as p0_is_neg_inf, approx_percentile(v, 0.5) = cast('-Inf' as double) as p50_is_neg_inf, approx_percentile(v, 1) = cast('-Inf' as double) as p100_is_neg_inf from t_approx_float_infinity; +truncate table t_approx_float_infinity; +insert into t_approx_float_infinity values (cast('-Inf' as double)), (cast('Inf' as double)); +select approx_percentile(v, 0) = cast('-Inf' as double) as p0_is_neg_inf, approx_percentile(v, 1) = cast('Inf' as double) as p100_is_pos_inf from t_approx_float_infinity; +drop table t_approx_float_infinity;