Skip to content

Commit fb646be

Browse files
[CH] Support skewness aggregate function (#12294)
* [CH] Support skewness aggregate function * removed comment * added new test logic * [CH] Cover skewness with grouping keys and drop duplicate fallback check * fix by splitting line 108
1 parent fa83280 commit fb646be

3 files changed

Lines changed: 13 additions & 5 deletions

File tree

backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,6 @@ object CHExpressionUtil {
192192
URL_DECODE -> DefaultValidator(),
193193
URL_ENCODE -> DefaultValidator(),
194194
FORMAT_STRING -> FormatStringValidator(),
195-
SKEWNESS -> DefaultValidator(),
196195
MAKE_YM_INTERVAL -> DefaultValidator(),
197196
MAP_ZIP_WITH -> DefaultValidator(),
198197
KURTOSIS -> DefaultValidator(),

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,14 @@ class GlutenClickhouseCountDistinctSuite extends GlutenClickHouseWholeStageTrans
100100
}
101101

102102
test("check count distinct with agg fallback") {
103-
// skewness agg is not supported, will cause fallback
104103
val sql = "select count(distinct(a,b)) , skewness(b) from " +
105104
"values (0, null,1), (0,null,1), (1, 1,1), (2, 2, 1) ,(2,2,2),(3,3,3) as data(a,b,c)"
106-
assertThrows[UnsupportedOperationException] {
107-
spark.sql(sql).show
108-
}
105+
compareResultsAgainstVanillaSpark(sql, true, { _ => })
106+
107+
val sqlWithKeys = "select a, count(distinct(b)) , skewness(b) from " +
108+
"values (0, null,1), (0,null,1), (1, 1,1), (2, 2, 1) ,(2,2,2),(3,3,3) as data(a,b,c) " +
109+
"group by a"
110+
compareResultsAgainstVanillaSpark(sqlWithKeys, true, { _ => })
109111
}
110112

111113
test("check count distinct with expr fallback") {

cpp-ch/local-engine/Parser/aggregate_function_parser/SimpleStatisticsFunctions.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,14 @@ class AggregateFunctionParserStddev final : public AggregateFunctionParser
6363
return func_node;
6464
}
6565
};
66+
// for skewness
67+
struct SkewnessNameStruct
68+
{
69+
static constexpr auto spark_name = "skewness";
70+
static constexpr auto ch_name = "skewSamp";
71+
};
6672

73+
static const AggregateFunctionParserRegister<AggregateFunctionParserStddev<SkewnessNameStruct>> registerer_skewness;
6774
static const AggregateFunctionParserRegister<AggregateFunctionParserStddev<StddevNameStruct>> registerer_stddev;
6875
static const AggregateFunctionParserRegister<AggregateFunctionParserStddev<StddevSampNameStruct>> registerer_stddev_samp;
6976
}

0 commit comments

Comments
 (0)