Skip to content

Commit 3bfe812

Browse files
MazterQyouMazterQyou
authored
feat(cubesql): Support PatchMeasure for view measures (cube-js#10571)
1 parent f053837 commit 3bfe812

8 files changed

Lines changed: 272 additions & 87 deletions

File tree

‎packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ export class BaseMeasure {
1818

1919
protected preparePatchedMeasure(sourceMeasure: string, newMeasureType: string | null, addFilters: Array<{sql: Function}>): MeasureDefinition {
2020
const source = this.query.cubeEvaluator.measureByPath(sourceMeasure);
21+
const aggType = source.aggType ?? source.type;
2122

22-
let resultMeasureType = source.type;
23+
let resultMeasureType = aggType;
2324
if (newMeasureType !== null) {
24-
switch (source.type) {
25+
switch (aggType) {
2526
case 'sum':
2627
case 'avg':
2728
case 'min':
@@ -32,29 +33,35 @@ export class BaseMeasure {
3233
case 'min':
3334
case 'max':
3435
case 'count_distinct':
36+
case 'countDistinct':
3537
case 'count_distinct_approx':
38+
case 'countDistinctApprox':
3639
// Can change from avg/... to count_distinct
3740
// Latter does not care what input value is
3841
// ok, do nothing
3942
break;
4043
default:
4144
throw new UserError(
42-
`Unsupported measure type replacement for ${sourceMeasure}: ${source.type} => ${newMeasureType}`
45+
`Unsupported measure type replacement for ${sourceMeasure}: ${aggType} => ${newMeasureType}`
4346
);
4447
}
4548
break;
4649
case 'count_distinct':
50+
case 'countDistinct':
4751
case 'count_distinct_approx':
52+
case 'countDistinctApprox':
4853
switch (newMeasureType) {
4954
case 'count_distinct':
55+
case 'countDistinct':
5056
case 'count_distinct_approx':
57+
case 'countDistinctApprox':
5158
// ok, do nothing
5259
break;
5360
default:
5461
// Can not change from count_distinct to avg/...
5562
// Latter do care what input value is, and original measure can be defined on strings
5663
throw new UserError(
57-
`Unsupported measure type replacement for ${sourceMeasure}: ${source.type} => ${newMeasureType}`
64+
`Unsupported measure type replacement for ${sourceMeasure}: ${aggType} => ${newMeasureType}`
5865
);
5966
}
6067
break;
@@ -64,7 +71,7 @@ export class BaseMeasure {
6471
// Can not change from count
6572
// There's no SQL at all
6673
throw new UserError(
67-
`Unsupported measure type replacement for ${sourceMeasure}: ${source.type} => ${newMeasureType}`
74+
`Unsupported measure type replacement for ${sourceMeasure}: ${aggType} => ${newMeasureType}`
6875
);
6976
}
7077

@@ -81,14 +88,16 @@ export class BaseMeasure {
8188
case 'max':
8289
case 'count':
8390
case 'count_distinct':
91+
case 'countDistinct':
8492
case 'count_distinct_approx':
93+
case 'countDistinctApprox':
8594
// ok, do nothing
8695
break;
8796
default:
8897
// Can not add filters to string, time, boolean, number
8998
// Aggregation is already included in SQL, it's hard to patch that
9099
throw new UserError(
91-
`Unsupported additional filters for measure ${sourceMeasure} type ${source.type}`
100+
`Unsupported additional filters for measure ${sourceMeasure} type ${aggType}`
92101
);
93102
}
94103

@@ -97,9 +106,16 @@ export class BaseMeasure {
97106

98107
const patchedFrom = this.query.cubeEvaluator.parsePath('measures', sourceMeasure);
99108

109+
// For view measures, `type` is `number` (aggregation is embedded in SQL)
110+
// while `aggType` carries the real aggregation kind. We must preserve that
111+
// distinction to avoid double-wrapping (e.g. SUM(SUM(...))).
112+
const typeFields = source.aggType != null
113+
? { type: source.type, aggType: resultMeasureType }
114+
: { type: resultMeasureType };
115+
100116
return {
101117
...source,
102-
type: resultMeasureType,
118+
...typeFields,
103119
filters: resultFilters,
104120
patchedFrom: {
105121
cubeName: patchedFrom[0],

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3319,13 +3319,28 @@ export class BaseQuery {
33193319
const primaryKeys = this.cubeEvaluator.primaryKeys[cubeName];
33203320
const orderBySql = (symbol.orderBy || []).map(o => ({ sql: this.evaluateSql(cubeName, o.sql), dir: o.dir }));
33213321
let sql;
3322+
let patchedSymbol = symbol;
33223323
if (symbol.type !== 'rank') {
3323-
sql = symbol.sql && this.evaluateSql(cubeName, symbol.sql) ||
3324+
const evaluateSql = () => symbol.sql && this.evaluateSql(cubeName, symbol.sql) ||
33243325
primaryKeys.length && (
33253326
primaryKeys.length > 1 ?
33263327
this.concatStringsSql(primaryKeys.map((pk) => this.castToString(this.primaryKeySql(pk, cubeName))))
33273328
: this.primaryKeySql(primaryKeys[0], cubeName)
33283329
) || '*';
3330+
// For patched view measures (aggType is set), the view's sql resolves to
3331+
// already-aggregated SQL (e.g. SUM(col)). Filters must be applied inside
3332+
// that aggregation, not outside. We pre-evaluate the filter SQL at the
3333+
// view level, push it down via context, and skip filters at this level.
3334+
const isPatchedViewMeasure = symbol.aggType && symbol.patchedFrom && symbol.filters?.length;
3335+
if (isPatchedViewMeasure) {
3336+
const pushDownFilterSql = this.evaluateFiltersArray(symbol.filters, cubeName);
3337+
sql = this.evaluateSymbolSqlWithContext(evaluateSql, {
3338+
patchMeasurePushDownFilterSql: pushDownFilterSql,
3339+
});
3340+
patchedSymbol = { ...symbol, filters: [] };
3341+
} else {
3342+
sql = evaluateSql();
3343+
}
33293344
}
33303345
const result = this.renderSqlMeasure(
33313346
name,
@@ -3335,7 +3350,7 @@ export class BaseQuery {
33353350
sql,
33363351
isMemberExpr,
33373352
),
3338-
symbol,
3353+
patchedSymbol,
33393354
cubeName
33403355
),
33413356
symbol,
@@ -3836,11 +3851,21 @@ export class BaseQuery {
38363851
}
38373852

38383853
applyMeasureFilters(evaluateSql, symbol, cubeName) {
3839-
if (!symbol.filters || !symbol.filters.length) {
3854+
const pushDownFilterSql = this.safeEvaluateSymbolContext().patchMeasurePushDownFilterSql;
3855+
const hasOwnFilters = symbol.filters && symbol.filters.length;
3856+
3857+
if (!hasOwnFilters && !pushDownFilterSql) {
38403858
return evaluateSql;
38413859
}
38423860

3843-
const where = this.evaluateMeasureFilters(symbol, cubeName);
3861+
const parts = [];
3862+
if (hasOwnFilters) {
3863+
parts.push(this.evaluateMeasureFilters(symbol, cubeName));
3864+
}
3865+
if (pushDownFilterSql) {
3866+
parts.push(pushDownFilterSql);
3867+
}
3868+
const where = parts.join(' AND ');
38443869

38453870
return `CASE WHEN ${where} THEN ${evaluateSql === '*' ? '1' : evaluateSql} END`;
38463871
}

‎packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ export type TimeShiftDefinitionReference = {
6060

6161
export type MeasureDefinition = {
6262
type: string;
63+
aggType?: string,
6364
sql(): string;
6465
ownedByCube: boolean;
6566
rollingWindow?: any

‎packages/cubejs-testing/birdbox-fixtures/postgresql/schema/Orders.js‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ cube(`Orders`, {
2424
type: `count_distinct`,
2525
sql: `CASE WHEN ${Orders.status} = 'shipped' THEN ${CUBE}.id END`
2626
},
27+
approxOrderCount: {
28+
type: `count_distinct_approx`,
29+
sql: `CASE WHEN ${Orders.status} = 'shipped' THEN ${CUBE}.id END`
30+
},
2731
netCollectionCompleted: {
2832
type: `sum`,
2933
sql: `CASE WHEN ${Orders.status} = 'shipped' THEN ${CUBE}.amount END`
@@ -50,6 +54,24 @@ cube(`Orders`, {
5054
format: `currency`,
5155
currency: `usd`,
5256
},
57+
avgAmount: {
58+
sql: `amount`,
59+
type: `avg`,
60+
format: `currency`,
61+
currency: `usd`,
62+
},
63+
minAmount: {
64+
sql: `amount`,
65+
type: `min`,
66+
format: `currency`,
67+
currency: `usd`,
68+
},
69+
maxAmount: {
70+
sql: `amount`,
71+
type: `max`,
72+
format: `currency`,
73+
currency: `usd`,
74+
},
5375
toRemove: {
5476
type: `count`,
5577
},

‎packages/cubejs-testing/test/__snapshots__/smoke-cubesql.test.ts.snap‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,8 @@ Object {
149149
exports[`SQL API Cube SQL over HTTP sql4sql regular query with missing column 1`] = `
150150
Object {
151151
"body": Object {
152-
"error": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
153-
"stack": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
152+
"error": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.approxOrderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.avgAmount, Orders.minAmount, Orders.maxAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
153+
"stack": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.approxOrderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.avgAmount, Orders.minAmount, Orders.maxAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
154154
},
155155
"headers": Headers {
156156
Symbol(map): Object {
@@ -161,7 +161,7 @@ Object {
161161
"keep-alive",
162162
],
163163
"content-length": Array [
164-
"1431",
164+
"1589",
165165
],
166166
"content-type": Array [
167167
"application/json; charset=utf-8",
@@ -557,6 +557,18 @@ Array [
557557
]
558558
`;
559559

560+
exports[`SQL API Postgres (Data) measure in view with ad-hoc filter: measure-in-view-with-ad-hoc-filters 1`] = `
561+
Array [
562+
Object {
563+
"new_amount": 800,
564+
"new_avg_amount": 400,
565+
"new_count_distinct": "1",
566+
"new_max_amount": 500,
567+
"new_min_amount": 300,
568+
},
569+
]
570+
`;
571+
560572
exports[`SQL API Postgres (Data) measure with ad-hoc filter and original measure: measure-with-ad-hoc-filters-and-original-measure 1`] = `
561573
Array [
562574
Object {

‎packages/cubejs-testing/test/smoke-cubesql.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -980,6 +980,38 @@ filter_subq AS (
980980
expect(res.rows).toMatchSnapshot('measure-with-ad-hoc-filters-and-original-measure');
981981
});
982982

983+
test('measure in view with ad-hoc filter', async () => {
984+
const query = `
985+
SELECT
986+
SUM(CASE
987+
WHEN status = 'processed' THEN totalAmount
988+
END) AS new_amount,
989+
AVG(CASE
990+
WHEN status = 'processed' THEN avgAmount
991+
END) AS new_avg_amount,
992+
MIN(CASE
993+
WHEN status = 'processed' THEN minAmount
994+
END) AS new_min_amount,
995+
MAX(CASE
996+
WHEN status = 'processed' THEN maxAmount
997+
END) AS new_max_amount,
998+
COUNT(DISTINCT CASE
999+
WHEN status = 'shipped' THEN orderCount
1000+
END) AS new_count_distinct
1001+
1002+
/* Works but testing Postgres does not include "hll_hash_any" function
1003+
APPROX_DISTINCT(CASE
1004+
WHEN status = 'shipped' THEN approxOrderCount
1005+
END) AS new_approx_distinct
1006+
*/
1007+
FROM
1008+
OrdersView
1009+
`;
1010+
1011+
const res = await connection.query(query);
1012+
expect(res.rows).toMatchSnapshot('measure-in-view-with-ad-hoc-filters');
1013+
});
1014+
9831015
/// Query references `updatedAt` in three places: in outer projection, in grouping key and in window
9841016
/// Incoming query is consistent: all three references same column
9851017
/// This tests that generated SQL for pushdown remains consistent:

0 commit comments

Comments
 (0)