[pull] master from cube-js:master

.github/actions/integration/clickhouse.sh

@@ -1,24 +1,34 @@
 #!/bin/bash
 set -eo pipefail
 
+# Cube officially maintained upstream LTS releases (as of 2026-06): 25.8, 26.3. + 1 EOL release
+# 24.8 can be removed, but let's test that it still works
+
 # Debug log for test containers
 export DEBUG=testcontainers
 
-export TEST_CLICKHOUSE_VERSION=23.11
+export TEST_CLICKHOUSE_VERSION=26.3
+
+echo "::group::Clickhouse ${TEST_CLICKHOUSE_VERSION}";
+docker pull clickhouse/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
+yarn lerna run --concurrency 1 --stream --no-prefix integration:clickhouse
+echo "::endgroup::"
+
+export TEST_CLICKHOUSE_VERSION=25.8
 
 echo "::group::Clickhouse ${TEST_CLICKHOUSE_VERSION}";
 docker pull clickhouse/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
 yarn lerna run --concurrency 1 --stream --no-prefix integration:clickhouse
 echo "::endgroup::"
 
-export TEST_CLICKHOUSE_VERSION=22.8
+export TEST_CLICKHOUSE_VERSION=25.3
 
 echo "::group::Clickhouse ${TEST_CLICKHOUSE_VERSION}";
 docker pull clickhouse/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
 yarn lerna run --concurrency 1 --stream --no-prefix integration:clickhouse
 echo "::endgroup::"
 
-export TEST_CLICKHOUSE_VERSION=21.8
+export TEST_CLICKHOUSE_VERSION=24.8
 
 echo "::group::Clickhouse ${TEST_CLICKHOUSE_VERSION}";
 docker pull clickhouse/clickhouse-server:${TEST_CLICKHOUSE_VERSION}

.github/actions/integration/mysql.sh

@@ -4,6 +4,11 @@ set -eo pipefail
 # Debug log for test containers
 export DEBUG=testcontainers
 
+# Recursive CTE based generated time series require MySQL 8.0+.
+# Keep it disabled for 5.6/5.7 (falls back to the portable VALUES series) and
+# enable it only for the 8.0.x run below.
+export CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES=false
+
 export TEST_MYSQL_VERSION=5.6
 
 echo "::group::MySQL ${TEST_MYSQL_VERSION}";
@@ -19,6 +24,7 @@ yarn lerna run --concurrency 1 --stream --no-prefix integration:mysql
 echo "::endgroup::"
 
 export TEST_MYSQL_VERSION=8.0.24
+export CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES=true
 
 echo "::group::MySQL ${TEST_MYSQL_VERSION}";
 docker pull mysql:${TEST_MYSQL_VERSION}

.github/workflows/push.yml

@@ -416,6 +416,12 @@ jobs:
           - db: firebolt
             node-version: 22.x
             use_tesseract_sql_planner: true
+          - db: mysql
+            node-version: 22.x
+            use_tesseract_sql_planner: true
+          - db: clickhouse
+            node-version: 22.x
+            use_tesseract_sql_planner: true
       fail-fast: false
 
     steps:

packages/cubejs-backend-native/src/stream.rs

@@ -261,6 +261,27 @@ pub struct JsValueObject<'a> {
     pub handle: Handle<'a, JsArray>,
 }
 
+fn js_value_to_json_string<'a, C: Context<'a>>(
+    cx: &mut C,
+    value: Handle<'a, JsValue>,
+) -> Result<String, CubeError> {
+    let global = cx.global_object();
+    let json = global
+        .get::<JsObject, _, _>(cx, "JSON")
+        .map_err(|e| CubeError::internal(format!("Can't get JSON global: {}", e)))?;
+    let stringify = json
+        .get::<JsFunction, _, _>(cx, "stringify")
+        .map_err(|e| CubeError::internal(format!("Can't get JSON.stringify: {}", e)))?;
+    let undefined = cx.undefined().upcast::<JsValue>();
+    let result = stringify
+        .call(cx, undefined, [value])
+        .map_err(|e| CubeError::internal(format!("JSON.stringify failed: {}", e)))?;
+    let s = result.downcast::<JsString, _>(cx).map_err(|e| {
+        CubeError::internal(format!("JSON.stringify did not return a string: {}", e))
+    })?;
+    Ok(s.value(cx))
+}
+
 impl ValueObject for JsValueObject<'_> {
     fn len(&mut self) -> Result<usize, CubeError> {
         Ok(self.handle.len(&mut self.cx) as usize)
@@ -287,17 +308,22 @@ impl ValueObject for JsValueObject<'_> {
             || value.downcast::<JsNull, _>(&mut self.cx).is_ok()
         {
             Ok(FieldValue::Null)
-        } else if let Ok(b) = value.downcast::<JsArray, _>(&mut self.cx) {
-            Err(CubeError::internal(format!(
-                "Expected primitive value but found JsArray({:?})",
-                b
-            )))
+        } else if value.is_a::<JsArray, _>(&mut self.cx) {
+            Ok(FieldValue::String(Cow::Owned(js_value_to_json_string(
+                &mut self.cx,
+                value,
+            )?)))
         } else if let Ok(b) = value.downcast::<JsDate, _>(&mut self.cx) {
             // TODO: Support it?
             Err(CubeError::internal(format!(
                 "Expected primitive value but found JsDate({:?})",
                 b
             )))
+        } else if value.is_a::<JsObject, _>(&mut self.cx) {
+            Ok(FieldValue::String(Cow::Owned(js_value_to_json_string(
+                &mut self.cx,
+                value,
+            )?)))
         } else {
             Err(CubeError::internal(format!(
                 "Expected primitive value but found: {:?}",
@@ -321,7 +347,10 @@ fn js_stream_push_chunk(mut cx: FunctionContext) -> JsResult<JsUndefined> {
         handle: chunk_array,
     };
     let value =
-        transform_response(&mut value_object, this.schema.clone(), &this.member_fields).unwrap();
+        match transform_response(&mut value_object, this.schema.clone(), &this.member_fields) {
+            Ok(value) => value,
+            Err(e) => return value_object.cx.throw_error(e.message),
+        };
     let future = this.push_chunk(value);
     wait_for_future_and_execute_callback(
         this.tokio_handle.clone(),

packages/cubejs-backend-shared/src/env.ts

@@ -1011,6 +1011,19 @@ const variables: Record<string, (...args: any) => any> = {
       .asBool()
   ),
 
+  /**
+   * Use the generated (recursive CTE based) time series for the Tesseract SQL
+   * planner instead of the portable VALUES/UNION ALL series. Defaults to TRUE.
+   * Recursive CTEs require MySQL 8.0+ — set this to FALSE for MySQL < 8.0,
+   * which has no CTE support. When disabled, time series are materialized as a
+   * VALUES list.
+   */
+  mysqlUseGeneratedTimeSeries: ({ dataSource, preAggregations }: DataSourceOpts) => (
+    get(keyByDataSource('CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES', dataSource, preAggregations))
+      .default('true')
+      .asBool()
+  ),
+
   /** ****************************************************************
    * MSSQL Driver                                                    *
    ***************************************************************** */

packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts

@@ -21,7 +21,6 @@ import {
   StreamOptions,
   StreamTableDataWithTypes,
   TableColumn,
-  TableColumnQueryResult,
   TableQueryResult,
   TableStructure,
   UnloadOptions,

packages/cubejs-postgres-driver/src/PostgresDriver.ts

@@ -296,19 +296,22 @@ export class PostgresDriver<Config extends PostgresDriverConfiguration = Postgre
     if (!this.userDefinedTypes) {
       // Postgres enum types defined as typcategory = 'E' these can be assumed
       // to be of type varchar for the drivers purposes.
+      // Postgres array types defined as typcategory = 'A' these can be assumed
+      // to be of type text for the drivers purposes.
       // TODO: if full implmentation the constraints can be looked up via pg_enum
       // https://www.postgresql.org/docs/9.1/catalog-pg-enum.html
       const customTypes = await conn.query(
         `SELECT
             oid,
             CASE
                 WHEN typcategory = 'E' THEN 'varchar'
+                WHEN typcategory = 'A' THEN 'text'
                 ELSE typname
-            END
+            END AS typname
         FROM
             pg_type
         WHERE
-            typcategory in ('U', 'E')`,
+            typcategory in ('U', 'E', 'A')`,
         []
       );
 

packages/cubejs-postgres-driver/test/PostgresDriver.test.ts

@@ -114,6 +114,39 @@ describe('PostgresDriver', () => {
     }
   });
 
+  test('stream (array-typed columns)', async () => {
+    // Streaming must not fail when a query returns array-typed columns.
+    // Array types are reported as `text` and node-postgres parses them into
+    // JS arrays. See CORE-522.
+    const tableData = await driver.stream(
+      `SELECT
+        ARRAY['oops', 'test']::text[] as text_array,
+        ARRAY[1, 2, 3]::int[] as int_array`,
+      [],
+      {
+        highWaterMark: 1000,
+      }
+    );
+
+    try {
+      expect(await tableData.types).toEqual([
+        {
+          name: 'text_array',
+          type: 'text'
+        },
+        {
+          name: 'int_array',
+          type: 'text'
+        },
+      ]);
+      expect(await streamToArray(tableData.rowStream)).toEqual([
+        { text_array: ['oops', 'test'], int_array: [1, 2, 3] },
+      ]);
+    } finally {
+      await (<any> tableData).release();
+    }
+  });
+
   test('stream (exception)', async () => {
     try {
       await driver.stream('select * from test.random_name_for_table_that_doesnot_exist_sql_must_fail', [], {

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -1791,6 +1791,11 @@ export class BaseQuery {
       multiStageDimensions: withQuery.multiStageDimensions,
       multiStageTimeDimensions: withQuery.multiStageTimeDimensions,
       filters: withQuery.filters,
+      // Propagate SQL API member aliases so dimension columns produced by this stage
+      // match how the outer aggregate (and renderedReference) reference them. Without it
+      // the stage names columns with full internal aliases while the consumer references
+      // them by memberToAlias, producing `invalid identifier` errors.
+      memberToAlias: this.options.memberToAlias,
       // TODO do we need it?
       multiStageQuery: true, // !!fromDimensions.find(d => this.newDimension(d).isMultiStage())
       disableExternalPreAggregations: true,
@@ -1813,6 +1818,8 @@ export class BaseQuery {
       multiStageTimeDimensions: withQuery.multiStageTimeDimensions,
       filters: withQuery.filters,
       segments: withQuery.segments,
+      // See note above: keep member aliases consistent across multi-stage CTEs.
+      memberToAlias: this.options.memberToAlias,
       from: fromSql && {
         sql: fromSql,
         alias: `${withQuery.alias}_join`,
@@ -4552,7 +4559,7 @@ export class BaseQuery {
         PERCENTILECONT: 'PERCENTILE_CONT({{ args_concat }})',
       },
       statements: {
-        select: '{% if ctes %} WITH \n' +
+        select: '{% if ctes %} WITH {% if recursive %}RECURSIVE {% endif %}\n' +
           '{{ ctes | join(\',\n\') }}\n' +
           '{% endif %}' +
           'SELECT {% if distinct %}DISTINCT {% endif %}' +

packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts

@@ -27,10 +27,13 @@ class MysqlFilter extends BaseFilter {
 export class MysqlQuery extends BaseQuery {
   private readonly useNamedTimezones: boolean;
 
+  private readonly useGeneratedTimeSeries: boolean;
+
   public constructor(compilers: any, options: any) {
     super(compilers, options);
 
     this.useNamedTimezones = getEnv('mysqlUseNamedTimezones', { dataSource: this.dataSource });
+    this.useGeneratedTimeSeries = getEnv('mysqlUseGeneratedTimeSeries', { dataSource: this.dataSource });
   }
 
   public newFilter(filter) {
@@ -173,7 +176,7 @@ export class MysqlQuery extends BaseQuery {
   }
 
   public supportGeneratedSeriesForCustomTd(): boolean {
-    return true;
+    return this.useGeneratedTimeSeries;
   }
 
   public intervalString(interval: string): string {
@@ -211,31 +214,43 @@ export class MysqlQuery extends BaseQuery {
       '{% endfor %}' +
       ') AS dates';
 
-    templates.statements.generated_time_series_select =
-      'WITH RECURSIVE date_series AS (\n' +
-      '  SELECT TIMESTAMP({{ start }}) AS date_from\n' +
-      '  UNION ALL\n' +
-      '  SELECT DATE_ADD(date_from, INTERVAL {{ granularity }})\n' +
-      '  FROM date_series\n' +
-      '  WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= TIMESTAMP({{ end }})\n' +
-      ')\n' +
-      'SELECT CAST(date_from AS DATETIME) AS date_from,\n' +
-      '       CAST(DATE_SUB(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME) AS date_to\n' +
-      'FROM date_series';
-
-    templates.statements.generated_time_series_with_cte_range_source =
-      'WITH RECURSIVE date_series AS (\n' +
-      '  SELECT {{ range_source }}.{{ min_name }} AS date_from,\n' +
-      '         {{ range_source }}.{{ max_name }} AS max_date\n' +
-      '  FROM {{ range_source }}\n' +
-      '  UNION ALL\n' +
-      '  SELECT DATE_ADD(date_from, INTERVAL {{ granularity }}), max_date\n' +
-      '  FROM date_series\n' +
-      '  WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= max_date\n' +
-      ')\n' +
-      'SELECT CAST(date_from AS DATETIME) AS date_from,\n' +
-      '       CAST(DATE_SUB(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME) AS date_to\n' +
-      'FROM date_series';
+    // Generated (recursive CTE based) time series. Recursive CTEs require
+    // MySQL 8.0+, so they are only registered when explicitly enabled via
+    // CUBEJS_DB_MYSQL_USE_GENERATED_TIME_SERIES. When absent, the Tesseract
+    // planner falls back to the portable VALUES/UNION ALL `time_series_select`
+    // template, which also works on MySQL 5.6/5.7.
+    //
+    // The template body becomes the content of `time_series AS (...)` CTE, so
+    // it self-references `time_series` for recursion (no nested WITH). The
+    // outer WITH is emitted as `WITH RECURSIVE` because the
+    // `generated_time_series_recursive` marker below is present.
+    if (this.useGeneratedTimeSeries) {
+      templates.statements.generated_time_series_select =
+        'SELECT CAST(TIMESTAMP({{ start }}) AS DATETIME(6)) AS date_from,\n' +
+        '       CAST(DATE_SUB(DATE_ADD(TIMESTAMP({{ start }}), INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6)) AS date_to\n' +
+        'UNION ALL\n' +
+        'SELECT DATE_ADD(date_from, INTERVAL {{ granularity }}),\n' +
+        '       CAST(DATE_SUB(DATE_ADD(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6))\n' +
+        'FROM time_series\n' +
+        'WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= TIMESTAMP({{ end }})';
+
+      templates.statements.generated_time_series_with_cte_range_source =
+        'SELECT CAST({{ range_source }}.{{ min_name }} AS DATETIME(6)) AS date_from,\n' +
+        '       CAST(DATE_SUB(DATE_ADD({{ range_source }}.{{ min_name }}, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6)) AS date_to,\n' +
+        '       {{ range_source }}.{{ max_name }} AS max_date\n' +
+        'FROM {{ range_source }}\n' +
+        'UNION ALL\n' +
+        'SELECT DATE_ADD(date_from, INTERVAL {{ granularity }}),\n' +
+        '       CAST(DATE_SUB(DATE_ADD(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME(6)),\n' +
+        '       max_date\n' +
+        'FROM time_series\n' +
+        'WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= max_date';
+
+      // Marker (presence-only) telling the Tesseract planner that the generated
+      // time series is a self-referencing recursive CTE and the outer WITH must
+      // be emitted as `WITH RECURSIVE`.
+      templates.statements.generated_time_series_recursive = 'true';
+    }
     templates.expressions.wrap_segment_select = 'IF({{ expr }}, 1, 0)';
     templates.expressions.wrap_segment_filter = '{{ expr }} = 1';
 

packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts

@@ -123,6 +123,7 @@ export class ClickHouseDbRunner extends BaseDbRunner {
     } else {
       if (!this.container) {
         this.container = await new GenericContainer(`clickhouse/clickhouse-server:${this.clickHouseVersion}`)
+          .withEnvironment({ CLICKHOUSE_SKIP_USER_SETUP: '1' })
           .withExposedPorts(this.port())
           .start();
       }

packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-dataschema-compiler.test.ts

@@ -1,3 +1,5 @@
+import { getEnv } from '@cubejs-backend/shared';
+
 import { CompileError } from '../../../src/compiler/CompileError';
 import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery';
 import { prepareCompiler } from '../../../src/compiler/PrepareCompiler';
@@ -347,7 +349,12 @@ describe('ClickHouse DataSchemaCompiler', () => {
         { visitors__created_at: '2017-01-04T16:00:00.000' },
         { visitors__created_at: '2017-01-05T16:00:00.000' }
       ],
-      [{ visitors__created_at: '2017-01-06T16:00:00.000' }]
+      // afterDate boundary differs by planner: Tesseract uses the symmetric model
+      // (`> endOfDay`, so the 2017-01-06 16:00 row is excluded → empty), whereas the
+      // legacy planner used `> startOfDay` and returned that row.
+      getEnv('nativeSqlPlanner')
+        ? []
+        : [{ visitors__created_at: '2017-01-06T16:00:00.000' }]
     ];
     ['in_date_range', 'not_in_date_range', 'on_the_date', 'before_date', 'after_date'].map((operator, index) => {
       const filterValues = index < 2 ? ['2017-01-01', '2017-01-03'] : ['2017-01-06', '2017-01-06'];

packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-graph-builder.test.ts

@@ -1,3 +1,4 @@
+import { getEnv } from '@cubejs-backend/shared';
 import { UserError } from '../../../src/compiler/UserError';
 import { prepareJsCompiler } from '../../unit/PrepareCompiler';
 import { ClickHouseDbRunner } from './ClickHouseDbRunner';
@@ -1157,7 +1158,7 @@ describe('ClickHouse JoinGraph', () => {
     return dbRunner.testQuery(query.buildSqlAndParams()).then(res => {
       debugLog(JSON.stringify(res));
       expect(res).toEqual(
-        [{ visitor_checkins__revenue_per_checkin: '60' }]
+        [{ visitor_checkins__revenue_per_checkin: getEnv('nativeSqlPlanner') ? '50' : '60' }]
       );
     });
   }));