Merge branch 'fix/freshness-gated-volume-eventspace' into 'enterprise'

fix(monitors): event-space forecasting for freshness-gated volume/metric prediction

See merge request dkinternal/testgen/dataops-testgen!574

Author: ci bot

testgen/commands/test_thresholds_prediction.py

@@ -299,9 +299,12 @@ def compute_sarimax_threshold(
     exclude_weekends: bool = False,
     holiday_codes: list[str] | None = None,
     schedule_tz: str | None = None,
+    event_space: bool = False,
 ) -> tuple[float | None, float | None, str | None]:
     """Compute SARIMAX-based thresholds for the next forecast point.
 
+    `event_space=True` fits in event-space (one step per observation, no interpolation).
+
     Returns (lower, upper, forecast_json) or (None, None, None) if insufficient data.
     """
     try:
@@ -311,6 +314,7 @@ def compute_sarimax_threshold(
             exclude_weekends=exclude_weekends,
             holiday_codes=holiday_codes,
             tz=schedule_tz,
+            event_space=event_space,
         )
 
         num_points = len(history)
@@ -359,13 +363,17 @@ def compute_volume_or_metric_threshold(
     Freshness_Trend detected a fingerprint change.
     """
     filtered_history = history.loc[history["test_run_id"].astype(str).isin(freshness_updates)]
+    # Event-space fit: the filtered series has one point per refresh and is irregularly spaced.
+    # Calendar resampling + interpolation would smooth the refresh jumps into small uniform
+    # increments and collapse the jump variance, biasing the forecast low and the band too tight.
     lower, upper, prediction = compute_sarimax_threshold(
         filtered_history,
         sensitivity=sensitivity,
         num_forecast=num_forecast,
         exclude_weekends=exclude_weekends,
         holiday_codes=holiday_codes,
         schedule_tz=schedule_tz,
+        event_space=True,
     )
     if prediction is not None:
         # Pull the baseline value from the most-recent filtered row.

testgen/common/time_series_service.py

@@ -23,6 +23,7 @@ def get_sarimax_forecast(
     exclude_weekends: bool = False,
     holiday_codes: list[str] | None = None,
     tz: str | None = None,
+    event_space: bool = False,
 ) -> pd.DataFrame:
     """
     # Parameters
@@ -33,30 +34,48 @@ def get_sarimax_forecast(
     :param exclude_weekends: Whether weekends should be considered exogenous when training the model and forecasting.
     :param holiday_codes: List of country or financial market codes defining holidays to be considered exogenous when training the model and forecasting.
     :param tz: IANA timezone (e.g. "America/New_York") for day-of-week/holiday checks. Naive timestamps are treated as UTC and converted to this timezone before determining weekday/holiday status.
+    :param event_space: When False (default), resample the series onto a regular calendar grid and
+                    linearly interpolate gaps before fitting — appropriate for series sampled at a
+                    steady cadence, and where weekend/holiday exogenous flags apply. When True, fit
+                    in event-space: one model step per observation, no interpolation, no calendar
+                    exog. Required for irregularly-spaced series (e.g. the freshness-update points of
+                    a refresh-driven table) where interpolation would smooth multi-period jumps into
+                    small uniform increments and collapse the very jump variance the forecast must
+                    capture.
 
     # Return value
     Returns a Pandas dataframe with forecast DatetimeIndex, "mean" column, and "se" (standard error) column.
     """
     if len(history) < MIN_TRAIN_VALUES:
         raise NotEnoughData("Not enough data points in history.")
 
-    # statsmodels requires DatetimeIndex with a regular frequency
-    # Resample the data to get a regular time series
-    datetimes = history.index.to_series()
-    frequency = infer_frequency(datetimes)
-    resampled_history = history.resample(frequency).mean().interpolate(method="linear")
-
-    if len(resampled_history) < MIN_TRAIN_VALUES:
-        raise NotEnoughData("Not enough data points after resampling.")
+    if event_space:
+        # Event-space: one step per observation. Map the values onto a synthetic regular grid
+        # stepped by the median observed interval, so forecast timestamps remain realistic while
+        # the model sees each observation as a single step (no interpolated points between them).
+        median_step = history.index.to_series().diff().median()
+        if pd.isna(median_step) or median_step <= pd.Timedelta(0):
+            median_step = pd.Timedelta(days=1)
+        step = median_step
+        synthetic_index = pd.date_range(end=history.index[-1], periods=len(history), freq=step)
+        train_history = pd.DataFrame(history.iloc[:, 0].values, index=synthetic_index, columns=[history.columns[0]])
+    else:
+        # statsmodels requires DatetimeIndex with a regular frequency
+        # Resample the data to get a regular time series
+        datetimes = history.index.to_series()
+        frequency = infer_frequency(datetimes)
+        train_history = history.resample(frequency).mean().interpolate(method="linear")
+        if len(train_history) < MIN_TRAIN_VALUES:
+            raise NotEnoughData("Not enough data points after resampling.")
+        step = pd.to_timedelta(frequency)
 
     # Generate DatetimeIndex with future dates
-    forecast_start = resampled_history.index[-1] + pd.to_timedelta(frequency)
-    forecast_index = pd.date_range(start=forecast_start, periods=num_forecast, freq=frequency)
+    forecast_index = pd.date_range(start=train_history.index[-1] + step, periods=num_forecast, freq=step)
 
-    # Detect holidays in entire date range
+    # Detect holidays in entire date range (calendar-aware path only)
     holiday_dates = None
-    if holiday_codes:
-        all_dates_index = resampled_history.index.append(forecast_index)
+    if not event_space and holiday_codes:
+        all_dates_index = train_history.index.append(forecast_index)
         holiday_dates = get_holiday_dates(holiday_codes, all_dates_index)
 
     def get_exog_flags(index: pd.DatetimeIndex) -> pd.DataFrame:
@@ -71,11 +90,13 @@ def get_exog_flags(index: pd.DatetimeIndex) -> pd.DataFrame:
             exog.loc[pd.Index(check_index.date).isin(holiday_dates), "is_excluded"] = 1
         return exog
 
-    exog_train = get_exog_flags(resampled_history.index)
+    # Calendar exogenous flags only apply when fitting against real calendar time.
+    exog_train = None if event_space else get_exog_flags(train_history.index)
+    exog_forecast = None if event_space else get_exog_flags(forecast_index)
 
     # When seasonal_order is not specified, this is effectively the ARIMAX model
     model = SARIMAX(
-        resampled_history.iloc[:, 0],
+        train_history.iloc[:, 0],
         exog=exog_train,
         # This is a good starting point according to Gemini - tune if needed
         order=(1, 1, 1),
@@ -85,12 +106,6 @@ def get_exog_flags(index: pd.DatetimeIndex) -> pd.DataFrame:
     )
     fitted_model = model.fit(disp=False)
 
-    forecast_index = pd.date_range(
-        start=resampled_history.index[-1] + pd.to_timedelta(frequency),
-        periods=num_forecast,
-        freq=frequency
-    )
-    exog_forecast = get_exog_flags(forecast_index)
     forecast = fitted_model.get_forecast(steps=num_forecast, exog=exog_forecast)
 
     results = pd.DataFrame(index=forecast_index)

testgen/ui/components/frontend/js/pages/table_monitoring_trends.js

@@ -123,18 +123,6 @@ const TableMonitoringTrend = (props) => {
       },
       div(
         { class: '', style: 'width: 100%;' },
-        () => {
-          const extendedHistory = getValue(props.extended_history) ?? false;
-          return div(
-            { class: 'extended-history-toggle' },
-            Button({
-              label: extendedHistory ? 'Show default view' : 'Show more history',
-              icon: extendedHistory ? 'history_toggle_off' : 'history',
-              width: 'auto',
-              onclick: () => emit('ToggleExtendedHistory', { payload: {} }),
-            }),
-          );
-        },
         () => {
           if (!getValue(props.dialog)?.open) return div();
           return ChartsSection(props, { schemaChartSelection, getDataStructureLogs });
@@ -229,12 +217,22 @@ const TableMonitoringTrend = (props) => {
   );
 
   const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? '');
+  const historyToggle = () => {
+    const extendedHistory = getValue(props.extended_history) ?? false;
+    return Button({
+      label: extendedHistory ? 'Show default view' : 'Show more history',
+      icon: extendedHistory ? 'history_toggle_off' : 'history',
+      width: 'auto',
+      onclick: () => emit('ToggleExtendedHistory', { payload: {} }),
+    });
+  };
   return Dialog(
     {
       title: dialogTitle,
       open: dialogOpen,
       onClose: () => { dialogOpen.val = false; emit('CloseTrendsDialog', {}); },
       width: '75rem',
+      headerActions: historyToggle,
     },
     content,
   );
@@ -825,13 +823,6 @@ stylesheet.replace(`
     position: relative;
   }
 
-  .extended-history-toggle {
-    position: absolute;
-    top: -70px;
-    right: 48px;
-    z-index: 1;
-  }
-
   .table-monitoring-trend-wrapper:not(.has-sidebar) > .tg-dualpane-divider {
     display: none;
   }

testgen/ui/static/js/components/dialog.js

@@ -5,6 +5,7 @@
  * @property {import('../van.min.js').State<boolean>} open - Reactive open state
  * @property {Function} onClose - Called when the dialog is closed (backdrop click or X button)
  * @property {string} [width] - CSS width value, default '30rem'
+ * @property {(Element | Function)} [headerActions] - Optional node rendered right-aligned in header row
  */
 import van from '../van.min.js';
 import { getValue, loadStylesheet } from '../utils.js';
@@ -28,7 +29,7 @@ const { button, div, i, span } = van.tags;
  * @param {DialogProps} props
  * @param {...(Element | string)} children - Content rendered in the dialog body
  */
-const Dialog = ({ title, open, onClose, width = '30rem' }, ...children) => {
+const Dialog = ({ title, open, onClose, width = '30rem', headerActions }, ...children) => {
     loadStylesheet('dialog', stylesheet);
 
     const overlay = div(
@@ -50,6 +51,7 @@ const Dialog = ({ title, open, onClose, width = '30rem' }, ...children) => {
             div(
                 { class: 'tg-dialog-header' },
                 span({ 'data-testid': 'dialog-title', class: 'tg-dialog-title' }, title),
+                headerActions ? div({ class: 'tg-dialog-header-actions' }, headerActions) : null,
             ),
             div({ 'data-testid': 'dialog-content', class: 'tg-dialog-content' }, ...children),
             button(
@@ -117,6 +119,12 @@ stylesheet.replace(`
     flex-shrink: 0;
 }
 
+.tg-dialog-header-actions {
+    margin-left: auto;
+    font-size: initial;
+    font-weight: initial;
+}
+
 .tg-dialog-content {
     padding: 0.75rem 1.5rem 1.5rem;
     overflow-y: auto;