[pull] master from cube-js:master

packages/cubejs-schema-compiler/test/integration/postgres/multi-stage-time-shift-filter-params.test.ts

@@ -0,0 +1,72 @@
+import { getEnv } from '@cubejs-backend/shared';
+import { prepareYamlCompiler } from '../../unit/PrepareCompiler';
+import { dbRunner } from './PostgresDBRunner';
+
+// Regression for CORE-543 (GitHub #11030):
+// A multi_stage time_shift measure over a cube whose `sql` uses FILTER_PARAMS
+// must render the prior-period CTE with the FILTER_PARAMS column shifted by the
+// same interval as the time-shift predicate. Otherwise the current-period
+// bounds on the un-shifted column contradict the shifted predicate, the
+// prior-period CTE is always empty, and the YoY measure collapses to null.
+describe('Multi-Stage time_shift + FILTER_PARAMS', () => {
+  jest.setTimeout(200000);
+
+  const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(`
+cubes:
+  - name: orders
+    sql: >
+      SELECT * FROM (
+        SELECT 1 as id, '2024-03-14T00:00:00.000Z'::timestamptz as created_at, 100 as amount
+        union all
+        SELECT 2 as id, '2025-03-14T00:00:00.000Z'::timestamptz as created_at, 300 as amount
+      ) AS t
+      WHERE {FILTER_PARAMS.orders.date.filter('created_at')}
+
+    dimensions:
+      - name: id
+        sql: id
+        type: number
+        primary_key: true
+
+      - name: date
+        sql: created_at
+        type: time
+
+    measures:
+      - name: revenue
+        sql: amount
+        type: sum
+
+      - name: revenue_1_y_ago
+        sql: "{revenue}"
+        multi_stage: true
+        type: number
+        time_shift:
+          - time_dimension: date
+            interval: 1 year
+            type: prior
+    `);
+
+  if (getEnv('nativeSqlPlanner')) {
+    it('prior-year time_shift measure is not emptied by FILTER_PARAMS', async () => dbRunner.runQueryTest({
+      measures: ['orders.revenue', 'orders.revenue_1_y_ago'],
+      timeDimensions: [
+        {
+          dimension: 'orders.date',
+          granularity: 'year',
+          dateRange: ['2025-01-01', '2025-12-31'],
+        }
+      ],
+      timezone: 'UTC',
+    }, [
+      {
+        orders__date_year: '2025-01-01T00:00:00.000Z',
+        orders__revenue: '300',
+        orders__revenue_1_y_ago: '100',
+      },
+    ], { compiler, joinGraph, cubeEvaluator }));
+  } else {
+    // This test is working only in tesseract
+    test.skip('prior-year time_shift measure is not emptied by FILTER_PARAMS', () => { expect(1).toBe(1); });
+  }
+});

rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/filter/base_filter.rs

@@ -25,8 +25,14 @@ impl ToSql for BaseFilter {
                 .filter_params_columns
                 .get(&symbol_to_match.full_name())
             {
+                let time_shift = visitor
+                    .time_shifts()
+                    .dimensions_shifts
+                    .get(&symbol_to_match.full_name())
+                    .and_then(|shift| shift.interval.as_ref());
                 return self.typed_filter().to_sql_for_filter_params(
                     filter_params_column,
+                    time_shift,
                     &query_tools,
                     templates,
                     filters_ctx,

rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/filter/typed_filter.rs

@@ -7,6 +7,7 @@ use crate::planner::filter::typed_filter::{resolve_base_symbol, FilterOp, TypedF
 use crate::planner::query_tools::QueryTools;
 use crate::planner::sql_templates::PlanSqlTemplates;
 use crate::planner::FiltersContext;
+use crate::planner::SqlInterval;
 use cubenativeutils::CubeError;
 use std::rc::Rc;
 
@@ -48,6 +49,7 @@ impl TypedFilter {
     pub fn to_sql_for_filter_params(
         &self,
         column: &FilterParamsColumn,
+        time_shift: Option<&SqlInterval>,
         query_tools: &Rc<QueryTools>,
         plan_templates: &PlanSqlTemplates,
         filters_context: &FiltersContext,
@@ -56,8 +58,23 @@ impl TypedFilter {
 
         match column {
             FilterParamsColumn::String(column_sql) => {
+                // Inside a time-shifted CTE the FILTER_PARAMS column must carry the
+                // same shift as the regular time-dimension filter, otherwise its
+                // current-period bounds contradict the shifted predicate and empty
+                // the CTE.
+                let shifted_column;
+                let member_sql = if let Some(interval) = time_shift {
+                    shifted_column = format!(
+                        "({})",
+                        plan_templates
+                            .add_timestamp_interval(column_sql.clone(), interval.to_sql())?
+                    );
+                    shifted_column.as_str()
+                } else {
+                    column_sql.as_str()
+                };
                 let ctx = FilterSqlContext {
-                    member_sql: column_sql,
+                    member_sql,
                     query_tools,
                     plan_templates,
                     use_db_time_zone,
@@ -66,6 +83,8 @@ impl TypedFilter {
                 dispatch_to_sql(self.operation(), &ctx)
             }
             FilterParamsColumn::Callback(callback) => {
+                // A callback column is opaque SQL produced by user code, so a
+                // time shift can't be wrapped around it; it is rendered as-is.
                 let args = match self.operation() {
                     // RollingWindowOffset carries [from, to, trailing, leading, offset];
                     // only the from/to dates are filter-param args for the callback.

rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/factory.rs

@@ -52,6 +52,10 @@ impl SqlNodesFactory {
         self.time_shifts = time_shifts;
     }
 
+    pub fn time_shifts(&self) -> &TimeShiftState {
+        &self.time_shifts
+    }
+
     pub fn set_calendar_time_shifts(
         &mut self,
         calendar_time_shifts: HashMap<String, CalendarDimensionTimeShift>,

rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_visitor.rs

@@ -1,6 +1,7 @@
 use super::sql_nodes::SqlNode;
 use super::CubeRefEvaluator;
 use crate::planner::filter::Filter;
+use crate::planner::planners::multi_stage::TimeShiftState;
 use crate::planner::query_tools::QueryTools;
 use crate::planner::sql_call::CubeRef;
 use crate::planner::sql_templates::PlanSqlTemplates;
@@ -13,6 +14,9 @@ pub struct SqlEvaluatorVisitor {
     query_tools: Rc<QueryTools>,
     cube_ref_evaluator: Rc<CubeRefEvaluator>,
     all_filters: Option<Filter>, //To pass to FILTER_PARAMS and FILTER_GROUP
+    /// Active per-dimension time shifts, carried so that FILTER_PARAMS rendering
+    /// can apply the same shift to its column as the regular filter rendering.
+    time_shifts: TimeShiftState,
     ignore_tz_convert: bool,
     /// When `true`, the caller (typically a `SqlCall` substitution site) expects
     /// the rendered expression to be safe for embedding next to operators —
@@ -30,11 +34,22 @@ impl SqlEvaluatorVisitor {
             query_tools,
             cube_ref_evaluator,
             all_filters,
+            time_shifts: TimeShiftState::default(),
             ignore_tz_convert: false,
             arg_needs_paren_safe: false,
         }
     }
 
+    pub fn with_time_shifts(&self, time_shifts: TimeShiftState) -> Self {
+        let mut self_copy = self.clone();
+        self_copy.time_shifts = time_shifts;
+        self_copy
+    }
+
+    pub fn time_shifts(&self) -> &TimeShiftState {
+        &self.time_shifts
+    }
+
     pub fn with_ignore_tz_convert(&self) -> Self {
         let mut self_copy = self.clone();
         self_copy.ignore_tz_convert = true;

rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/visitor_context.rs

@@ -2,6 +2,7 @@ use crate::physical_plan::cube_ref_evaluator::CubeRefEvaluator;
 use crate::physical_plan::sql_nodes::{SqlNode, SqlNodesFactory};
 use crate::physical_plan::sql_visitor::SqlEvaluatorVisitor;
 use crate::planner::filter::Filter;
+use crate::planner::planners::multi_stage::TimeShiftState;
 use crate::planner::query_tools::QueryTools;
 use crate::planner::sql_templates::PlanSqlTemplates;
 use crate::planner::FiltersContext;
@@ -15,6 +16,7 @@ pub struct VisitorContext {
     node_processor: Rc<dyn SqlNode>,
     cube_ref_evaluator: Rc<CubeRefEvaluator>,
     all_filters: Option<Filter>, //To pass to FILTER_PARAMS and FILTER_GROUP
+    time_shifts: TimeShiftState, //To pass to FILTER_PARAMS in time-shifted CTEs
     filters_context: FiltersContext,
 }
 
@@ -35,6 +37,7 @@ impl VisitorContext {
             node_processor,
             cube_ref_evaluator: Rc::new(nodes_factory.cube_ref_evaluator()),
             all_filters,
+            time_shifts: nodes_factory.time_shifts().clone(),
             filters_context,
         }
     }
@@ -43,6 +46,7 @@ impl VisitorContext {
         query_tools: Rc<QueryTools>,
         nodes_factory: &SqlNodesFactory,
         filter_params_columns: HashMap<String, crate::cube_bridge::member_sql::FilterParamsColumn>,
+        time_shifts: TimeShiftState,
     ) -> Self {
         let filters_context = FiltersContext {
             use_local_tz: nodes_factory.use_local_tz_in_date_range(),
@@ -55,6 +59,7 @@ impl VisitorContext {
             node_processor,
             cube_ref_evaluator: Rc::new(nodes_factory.cube_ref_evaluator()),
             all_filters: None,
+            time_shifts,
             filters_context,
         }
     }
@@ -65,6 +70,7 @@ impl VisitorContext {
             self.cube_ref_evaluator.clone(),
             self.all_filters.clone(),
         )
+        .with_time_shifts(self.time_shifts.clone())
     }
 
     pub fn node_processor(&self) -> Rc<dyn SqlNode> {

rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs

@@ -542,15 +542,15 @@ impl MultiStageMemberQueryPlanner {
         let dimensions = if let Some(exclude) = &grain.exclude {
             dimensions
                 .into_iter()
-                .filter(|d| !exclude.iter().any(|m| d.has_member_in_reference_chain(m)))
+                .filter(|d| !exclude.iter().any(|m| d.matches_grain_reference(m)))
                 .collect_vec()
         } else {
             dimensions
         };
         let dimensions = if let Some(keep_only) = &grain.keep_only {
             dimensions
                 .into_iter()
-                .filter(|d| keep_only.iter().any(|m| d.has_member_in_reference_chain(m)))
+                .filter(|d| keep_only.iter().any(|m| d.matches_grain_reference(m)))
                 .collect_vec()
         } else {
             dimensions

rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs

@@ -294,15 +294,15 @@ impl MultiStageQueryPlanner {
     ) -> Vec<Rc<MemberSymbol>> {
         let dims: Vec<Rc<MemberSymbol>> = if let Some(exclude) = &grain.exclude {
             dims.iter()
-                .filter(|d| !exclude.iter().any(|m| d.has_member_in_reference_chain(m)))
+                .filter(|d| !exclude.iter().any(|m| d.matches_grain_reference(m)))
                 .cloned()
                 .collect()
         } else {
             dims.clone()
         };
         if let Some(keep_only) = &grain.keep_only {
             dims.into_iter()
-                .filter(|d| keep_only.iter().any(|m| d.has_member_in_reference_chain(m)))
+                .filter(|d| keep_only.iter().any(|m| d.matches_grain_reference(m)))
                 .collect()
         } else {
             dims

rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_call.rs

@@ -461,6 +461,7 @@ impl SqlCall {
                         query_tools.clone(),
                         &SqlNodesFactory::new(),
                         filter_params_columns,
+                        visitor.time_shifts().clone(),
                     );
                     return crate::physical_plan::filter::render_filter_item(
                         &context, &subtree, templates,

rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_symbol.rs

@@ -236,6 +236,23 @@ impl MemberSymbol {
         false
     }
 
+    /// Whether this projected member is targeted by a grain reference
+    /// (`reduce_by` / `group_by`). Behaves like `has_member_in_reference_chain`,
+    /// but a time dimension also matches through its underlying base dimension:
+    /// a projected time dimension carries a granularity suffix in its full name
+    /// (`created_at_month`) while grain references point at the bare dimension
+    /// (`created_at`), so a grain reference removes the time dimension from the
+    /// grain regardless of the granularity it is queried at.
+    pub fn matches_grain_reference(&self, member: &Rc<MemberSymbol>) -> bool {
+        if self.has_member_in_reference_chain(member) {
+            return true;
+        }
+        if let Self::TimeDimension(td) = self {
+            return td.base_symbol().has_member_in_reference_chain(member);
+        }
+        false
+    }
+
     /// Returns a copy of this symbol with the path reduced to just the owning cube,
     /// stripping any join chain prefix (e.g. from views or cross-cube references).
     pub fn with_stripped_join_prefix(&self) -> Rc<Self> {

rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_multi_stage.yaml

@@ -186,6 +186,17 @@ cubes:
             reduce_by:
                 - orders.status
 
+          # reduce_by targets the time dimension itself (bare, no
+          # granularity). When the query selects created_at at some
+          # granularity, that granular dimension must be dropped from the
+          # partition regardless of its granularity suffix (CORE-550).
+          - name: amount_reduce_time
+            type: sum
+            sql: "{CUBE.total_amount}"
+            multi_stage: true
+            reduce_by:
+                - orders.created_at
+
           - name: amount_reduce_all
             type: sum
             sql: "{CUBE.total_amount}"
@@ -290,6 +301,20 @@ cubes:
                 - sql: "{CUBE.total_amount}"
                   dir: desc
 
+          # Rank reduced by the time dimension (bare) while ordered by a
+          # measure. With created_at selected at a granularity the partition
+          # must be "all outer dims except created_at" (CORE-550); the buggy
+          # planner left created_at_<gran> in PARTITION BY so every partition
+          # held one row and all ranks collapsed to 1.
+          - name: amount_rank_reduce_time
+            type: rank
+            multi_stage: true
+            reduce_by:
+                - orders.created_at
+            order_by:
+                - sql: "{CUBE.total_amount}"
+                  dir: desc
+
           # Rank by the raw time dimension itself (order_by is a
           # dimension, not a measure) reduced by created_at — partition
           # is "all outer dims except created_at". First link in the
@@ -533,3 +558,5 @@ views:
                 - total_amount
                 - amount_prev_month
                 - mom_growth
+                - amount_reduce_time
+                - amount_rank_reduce_time