Skip to content

Commit da1d68d

Browse files
ovrovr
authored
fix(tesseract): Support time series queries for MySQL 8+ (cube-js#11154)
1 parent b33b95c commit da1d68d

11 files changed

Lines changed: 126 additions & 39 deletions

File tree

‎.github/actions/integration/mysql.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ set -eo pipefail
44
# Debug log for test containers
55
export DEBUG=testcontainers
66

7+
# Recursive CTE based generated time series require MySQL 8.0+.
8+
# Keep it disabled for 5.6/5.7 (falls back to the portable VALUES series) and
9+
# enable it only for the 8.0.x run below.
10+
export CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES=false
11+
712
export TEST_MYSQL_VERSION=5.6
813

914
echo "::group::MySQL ${TEST_MYSQL_VERSION}";
@@ -19,6 +24,7 @@ yarn lerna run --concurrency 1 --stream --no-prefix integration:mysql
1924
echo "::endgroup::"
2025

2126
export TEST_MYSQL_VERSION=8.0.24
27+
export CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES=true
2228

2329
echo "::group::MySQL ${TEST_MYSQL_VERSION}";
2430
docker pull mysql:${TEST_MYSQL_VERSION}

‎.github/workflows/push.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,9 @@ jobs:
416416
- db: firebolt
417417
node-version: 22.x
418418
use_tesseract_sql_planner: true
419+
- db: mysql
420+
node-version: 22.x
421+
use_tesseract_sql_planner: true
419422
- db: clickhouse
420423
node-version: 22.x
421424
use_tesseract_sql_planner: true

‎packages/cubejs-backend-shared/src/env.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,6 +1011,19 @@ const variables: Record<string, (...args: any) => any> = {
10111011
.asBool()
10121012
),
10131013

1014+
/**
1015+
* Use the generated (recursive CTE based) time series for the Tesseract SQL
1016+
* planner instead of the portable VALUES/UNION ALL series. Defaults to TRUE.
1017+
* Recursive CTEs require MySQL 8.0+ — set this to FALSE for MySQL < 8.0,
1018+
* which has no CTE support. When disabled, time series are materialized as a
1019+
* VALUES list.
1020+
*/
1021+
mysqlUseGeneratedTimeSeries: ({ dataSource, preAggregations }: DataSourceOpts) => (
1022+
get(keyByDataSource('CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES', dataSource, preAggregations))
1023+
.default('true')
1024+
.asBool()
1025+
),
1026+
10141027
/** ****************************************************************
10151028
* MSSQL Driver *
10161029
***************************************************************** */

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4559,7 +4559,7 @@ export class BaseQuery {
45594559
PERCENTILECONT: 'PERCENTILE_CONT({{ args_concat }})',
45604560
},
45614561
statements: {
4562-
select: '{% if ctes %} WITH \n' +
4562+
select: '{% if ctes %} WITH {% if recursive %}RECURSIVE {% endif %}\n' +
45634563
'{{ ctes | join(\',\n\') }}\n' +
45644564
'{% endif %}' +
45654565
'SELECT {% if distinct %}DISTINCT {% endif %}' +

‎packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts‎

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,13 @@ class MysqlFilter extends BaseFilter {
2727
export class MysqlQuery extends BaseQuery {
2828
private readonly useNamedTimezones: boolean;
2929

30+
private readonly useGeneratedTimeSeries: boolean;
31+
3032
public constructor(compilers: any, options: any) {
3133
super(compilers, options);
3234

3335
this.useNamedTimezones = getEnv('mysqlUseNamedTimezones', { dataSource: this.dataSource });
36+
this.useGeneratedTimeSeries = getEnv('mysqlUseGeneratedTimeSeries', { dataSource: this.dataSource });
3437
}
3538

3639
public newFilter(filter) {
@@ -173,7 +176,7 @@ export class MysqlQuery extends BaseQuery {
173176
}
174177

175178
public supportGeneratedSeriesForCustomTd(): boolean {
176-
return true;
179+
return this.useGeneratedTimeSeries;
177180
}
178181

179182
public intervalString(interval: string): string {
@@ -211,31 +214,43 @@ export class MysqlQuery extends BaseQuery {
211214
'{% endfor %}' +
212215
') AS dates';
213216

214-
templates.statements.generated_time_series_select =
215-
'WITH RECURSIVE date_series AS (\n' +
216-
' SELECT TIMESTAMP({{ start }}) AS date_from\n' +
217-
' UNION ALL\n' +
218-
' SELECT DATE_ADD(date_from, INTERVAL {{ granularity }})\n' +
219-
' FROM date_series\n' +
220-
' WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= TIMESTAMP({{ end }})\n' +
221-
')\n' +
222-
'SELECT CAST(date_from AS DATETIME) AS date_from,\n' +
223-
' CAST(DATE_SUB(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME) AS date_to\n' +
224-
'FROM date_series';
225-
226-
templates.statements.generated_time_series_with_cte_range_source =
227-
'WITH RECURSIVE date_series AS (\n' +
228-
' SELECT {{ range_source }}.{{ min_name }} AS date_from,\n' +
229-
' {{ range_source }}.{{ max_name }} AS max_date\n' +
230-
' FROM {{ range_source }}\n' +
231-
' UNION ALL\n' +
232-
' SELECT DATE_ADD(date_from, INTERVAL {{ granularity }}), max_date\n' +
233-
' FROM date_series\n' +
234-
' WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= max_date\n' +
235-
')\n' +
236-
'SELECT CAST(date_from AS DATETIME) AS date_from,\n' +
237-
' CAST(DATE_SUB(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME) AS date_to\n' +
238-
'FROM date_series';
217+
// Generated (recursive CTE based) time series. Recursive CTEs require
218+
// MySQL 8.0+, so they are only registered when explicitly enabled via
219+
// CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES. When absent, the Tesseract
220+
// planner falls back to the portable VALUES/UNION ALL `time_series_select`
221+
// template, which also works on MySQL 5.6/5.7.
222+
//
223+
// The template body becomes the content of `time_series AS (...)` CTE, so
224+
// it self-references `time_series` for recursion (no nested WITH). The
225+
// outer WITH is emitted as `WITH RECURSIVE` because the
226+
// `generated_time_series_recursive` marker below is present.
227+
if (this.useGeneratedTimeSeries) {
228+
templates.statements.generated_time_series_select =
229+
'SELECT CAST(TIMESTAMP({{ start }}) AS DATETIME(6)) AS date_from,\n' +
230+
' CAST(DATE_SUB(DATE_ADD(TIMESTAMP({{ start }}), INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6)) AS date_to\n' +
231+
'UNION ALL\n' +
232+
'SELECT DATE_ADD(date_from, INTERVAL {{ granularity }}),\n' +
233+
' CAST(DATE_SUB(DATE_ADD(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6))\n' +
234+
'FROM time_series\n' +
235+
'WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= TIMESTAMP({{ end }})';
236+
237+
templates.statements.generated_time_series_with_cte_range_source =
238+
'SELECT CAST({{ range_source }}.{{ min_name }} AS DATETIME(6)) AS date_from,\n' +
239+
' CAST(DATE_SUB(DATE_ADD({{ range_source }}.{{ min_name }}, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6)) AS date_to,\n' +
240+
' {{ range_source }}.{{ max_name }} AS max_date\n' +
241+
'FROM {{ range_source }}\n' +
242+
'UNION ALL\n' +
243+
'SELECT DATE_ADD(date_from, INTERVAL {{ granularity }}),\n' +
244+
' CAST(DATE_SUB(DATE_ADD(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6)),\n' +
245+
' max_date\n' +
246+
'FROM time_series\n' +
247+
'WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= max_date';
248+
249+
// Marker (presence-only) telling the Tesseract planner that the generated
250+
// time series is a self-referencing recursive CTE and the outer WITH must
251+
// be emitted as `WITH RECURSIVE`.
252+
templates.statements.generated_time_series_recursive = 'true';
253+
}
239254
templates.expressions.wrap_segment_select = 'IF({{ expr }}, 1, 0)';
240255
templates.expressions.wrap_segment_filter = '{{ expr }} = 1';
241256

‎packages/cubejs-schema-compiler/test/integration/mysql/custom-granularities.test.ts‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
1+
import { getEnv } from '@cubejs-backend/shared';
12
import { prepareYamlCompiler } from '../../unit/PrepareCompiler';
23
import { MySqlDbRunner } from './MySqlDbRunner';
34

5+
// Tesseract renders rolling-window queries using recursive CTEs, which require
6+
// MySQL 8.0+. That capability is controlled by mysqlUseGeneratedTimeSeries
7+
// (CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES), which is disabled for MySQL < 8.0.
8+
// The legacy planner does not use CTEs and works on all versions, so only skip
9+
// these cases when running under Tesseract with generated time series disabled.
10+
const nativeSqlPlanner = getEnv('nativeSqlPlanner');
11+
const useGeneratedTimeSeries = getEnv('mysqlUseGeneratedTimeSeries', { dataSource: 'default' });
12+
const itRollingWindow = nativeSqlPlanner && !useGeneratedTimeSeries ? it.skip : it;
13+
414
describe('Custom Granularities', () => {
515
jest.setTimeout(200000);
616

@@ -387,7 +397,7 @@ describe('Custom Granularities', () => {
387397
{ joinGraph, cubeEvaluator, compiler }
388398
));
389399

390-
it('works with half_year custom granularity w/o dimensions with unbounded rolling window query', async () => dbRunner.runQueryTest(
400+
itRollingWindow('works with half_year custom granularity w/o dimensions with unbounded rolling window query', async () => dbRunner.runQueryTest(
391401
{
392402
measures: ['orders.rollingCountByUnbounded'],
393403
timeDimensions: [{
@@ -420,7 +430,7 @@ describe('Custom Granularities', () => {
420430
{ joinGraph, cubeEvaluator, compiler }
421431
));
422432

423-
it('works with half_year custom granularity with dimension with unbounded rolling window query', async () => dbRunner.runQueryTest(
433+
itRollingWindow('works with half_year custom granularity with dimension with unbounded rolling window query', async () => dbRunner.runQueryTest(
424434
{
425435
measures: ['orders.rollingCountByUnbounded'],
426436
timeDimensions: [{
@@ -498,7 +508,7 @@ describe('Custom Granularities', () => {
498508
{ joinGraph, cubeEvaluator, compiler }
499509
));
500510

501-
it('works with half_year_by_1st_april custom granularity with dimension with unbounded rolling window query', async () => dbRunner.runQueryTest(
511+
itRollingWindow('works with half_year_by_1st_april custom granularity with dimension with unbounded rolling window query', async () => dbRunner.runQueryTest(
502512
{
503513
measures: ['orders.rollingCountByUnbounded'],
504514
timeDimensions: [{
@@ -591,7 +601,7 @@ describe('Custom Granularities', () => {
591601
{ joinGraph, cubeEvaluator, compiler }
592602
));
593603

594-
it('works with half_year custom granularity w/o dimensions with trailing rolling window query', async () => dbRunner.runQueryTest(
604+
itRollingWindow('works with half_year custom granularity w/o dimensions with trailing rolling window query', async () => dbRunner.runQueryTest(
595605
{
596606
measures: ['orders.rollingCountByTrailing3Months'],
597607
timeDimensions: [{
@@ -624,7 +634,7 @@ describe('Custom Granularities', () => {
624634
{ joinGraph, cubeEvaluator, compiler }
625635
));
626636

627-
it('works with half_year custom granularity with dimension with trailing rolling window query', async () => dbRunner.runQueryTest(
637+
itRollingWindow('works with half_year custom granularity with dimension with trailing rolling window query', async () => dbRunner.runQueryTest(
628638
{
629639
measures: ['orders.rollingCountByTrailing3Months'],
630640
timeDimensions: [{
@@ -702,7 +712,7 @@ describe('Custom Granularities', () => {
702712
{ joinGraph, cubeEvaluator, compiler }
703713
));
704714

705-
it('works with half_year_by_1st_april custom granularity w/o dimensions with trailing rolling window query', async () => dbRunner.runQueryTest(
715+
itRollingWindow('works with half_year_by_1st_april custom granularity w/o dimensions with trailing rolling window query', async () => dbRunner.runQueryTest(
706716
{
707717
measures: ['orders.rollingCountByTrailing3Months'],
708718
timeDimensions: [{
@@ -739,7 +749,7 @@ describe('Custom Granularities', () => {
739749
{ joinGraph, cubeEvaluator, compiler }
740750
));
741751

742-
it('works with half_year custom granularity w/o dimensions with leading rolling window query', async () => dbRunner.runQueryTest(
752+
itRollingWindow('works with half_year custom granularity w/o dimensions with leading rolling window query', async () => dbRunner.runQueryTest(
743753
{
744754
measures: ['orders.rollingCountByLeading4Months'],
745755
timeDimensions: [{
@@ -772,7 +782,7 @@ describe('Custom Granularities', () => {
772782
{ joinGraph, cubeEvaluator, compiler }
773783
));
774784

775-
it('works with half_year custom granularity with dimension with leading rolling window query', async () => dbRunner.runQueryTest(
785+
itRollingWindow('works with half_year custom granularity with dimension with leading rolling window query', async () => dbRunner.runQueryTest(
776786
{
777787
measures: ['orders.rollingCountByLeading4Months'],
778788
timeDimensions: [{
@@ -850,7 +860,7 @@ describe('Custom Granularities', () => {
850860
{ joinGraph, cubeEvaluator, compiler }
851861
));
852862

853-
it('works with half_year_by_1st_april custom granularity w/o dimensions with leading rolling window query', async () => dbRunner.runQueryTest(
863+
itRollingWindow('works with half_year_by_1st_april custom granularity w/o dimensions with leading rolling window query', async () => dbRunner.runQueryTest(
854864
{
855865
measures: ['orders.rollingCountByLeading4Months'],
856866
timeDimensions: [{

‎rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/cte.rs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,26 @@ use std::rc::Rc;
88
pub struct Cte {
99
query: Rc<QueryPlan>,
1010
name: String,
11+
/// Whether this CTE is a self-referencing recursive CTE. When any CTE in a
12+
/// `WITH` clause is recursive, dialects such as MySQL require the clause to
13+
/// be emitted as `WITH RECURSIVE`.
14+
is_recursive: bool,
1115
}
1216

1317
impl Cte {
14-
pub fn new(query: Rc<QueryPlan>, name: String) -> Self {
15-
Self { query, name }
18+
pub fn new(query: Rc<QueryPlan>, name: String, is_recursive: bool) -> Self {
19+
Self {
20+
query,
21+
name,
22+
is_recursive,
23+
}
1624
}
1725

1826
pub fn new_from_select(select: Rc<Select>, name: String) -> Self {
1927
Self {
2028
query: Rc::new(QueryPlan::Select(select)),
2129
name,
30+
is_recursive: false,
2231
}
2332
}
2433

@@ -30,6 +39,10 @@ impl Cte {
3039
&self.name
3140
}
3241

42+
pub fn is_recursive(&self) -> bool {
43+
self.is_recursive
44+
}
45+
3346
pub fn to_sql(&self, templates: &PlanSqlTemplates) -> Result<String, CubeError> {
3447
let sql = format!("({})", self.query.to_sql(templates)?);
3548
Ok(sql)

‎rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/filter/operators/filter_sql_context.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ impl<'a> FilterSqlContext<'a> {
172172
None,
173173
None,
174174
false,
175+
false,
175176
)?;
176177
let to = self.plan_templates.select(
177178
vec![],
@@ -184,6 +185,7 @@ impl<'a> FilterSqlContext<'a> {
184185
None,
185186
None,
186187
false,
188+
false,
187189
)?;
188190
Ok((format!("({})", from), format!("({})", to)))
189191
}

‎rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/select.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,10 @@ impl Select {
115115
})
116116
.collect::<Result<Vec<_>, _>>()?;
117117

118+
// Dialects like MySQL require `WITH RECURSIVE` when any CTE is a
119+
// self-referencing recursive CTE (e.g. a generated time series).
120+
let recursive = self.ctes.iter().any(|cte| cte.is_recursive());
121+
118122
let order_by = self
119123
.order_by
120124
.iter()
@@ -141,6 +145,7 @@ impl Select {
141145
self.limit,
142146
self.offset,
143147
self.is_distinct,
148+
recursive,
144149
)?;
145150

146151
/* let res = format!(

‎rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/root_query.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,16 @@ impl<'a> LogicalNodeProcessor<'a, RootQuery> for RootQueryProcessor<'a> {
4444
query.schema(),
4545
);
4646
}
47-
ctes.push(Rc::new(Cte::new(Rc::new(query), alias)));
47+
// A generated time series is a self-referencing recursive CTE in
48+
// some dialects (e.g. MySQL), which need the `WITH RECURSIVE` form.
49+
let is_recursive = matches!(
50+
&cte_member.member_type,
51+
MultiStageMemberLogicalType::TimeSeries(_)
52+
) && self
53+
.builder
54+
.templates()
55+
.generated_time_series_is_recursive();
56+
ctes.push(Rc::new(Cte::new(Rc::new(query), alias, is_recursive)));
4857
}
4958

5059
let select = self

0 commit comments

Comments
 (0)