From c5237112a13cceee924d37cd9398e71400f7c4e1 Mon Sep 17 00:00:00 2001 From: Andras Toth <4157749+tothandras@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:11:31 +0200 Subject: [PATCH 1/3] feat(streaming): query cache --- api/v3/handlers/meters/query.go | 2 + api/v3/handlers/meters/query_csv.go | 2 + app/common/streaming.go | 14 + app/config/aggregation.go | 69 + app/config/config_test.go | 41 + app/config/testdata/complete.yaml | 5 + cmd/balance-worker/wire.go | 1 + cmd/balance-worker/wire_gen.go | 7 +- cmd/billing-worker/wire_gen.go | 10 +- cmd/jobs/internal/wire_gen.go | 10 +- cmd/notification-service/wire.go | 19 +- cmd/notification-service/wire_gen.go | 7 +- cmd/server/wire_gen.go | 10 +- cmd/sink-worker/wire.go | 19 +- cmd/sink-worker/wire_gen.go | 7 +- config.example.yaml | 31 + .../usagebased/service/rating/service_test.go | 42 + openmeter/cost/adapter/adapter.go | 2 + openmeter/credit/balance/usage.go | 2 + openmeter/entitlement/metered/balance.go | 2 + openmeter/entitlement/metered/balance_test.go | 30 + openmeter/meter/httphandler/query.go | 4 + openmeter/meter/httphandler/query_csv.go | 4 + openmeter/streaming/clickhouse/connector.go | 142 +- openmeter/streaming/clickhouse/meter_query.go | 39 +- .../clickhouse/meterqueryrow_cache.go | 320 ++++ .../meterqueryrow_cache_coverage.go | 287 +++ .../meterqueryrow_cache_coverage_test.go | 131 ++ .../clickhouse/meterqueryrow_cache_exec.go | 365 ++++ .../clickhouse/meterqueryrow_cache_gate.go | 235 +++ .../meterqueryrow_cache_integration_test.go | 1606 +++++++++++++++++ .../clickhouse/meterqueryrow_cache_monitor.go | 326 ++++ .../meterqueryrow_cache_parity_test.go | 285 +++ .../clickhouse/meterqueryrow_cache_query.go | 543 ++++++ .../clickhouse/meterqueryrow_cache_test.go | 113 ++ .../streaming/clickhouse/zz_audit_test.go | 254 +++ openmeter/streaming/query_params.go | 3 + openmeter/streaming/testutils/streaming.go | 24 + 38 files changed, 4961 insertions(+), 52 deletions(-) create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_coverage.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_coverage_test.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_query.go create mode 100644 openmeter/streaming/clickhouse/meterqueryrow_cache_test.go create mode 100644 openmeter/streaming/clickhouse/zz_audit_test.go diff --git a/api/v3/handlers/meters/query.go b/api/v3/handlers/meters/query.go index b0a5649c4b..61c5261891 100644 --- a/api/v3/handlers/meters/query.go +++ b/api/v3/handlers/meters/query.go @@ -59,6 +59,8 @@ func (h *handler) QueryMeter() QueryMeterHandler { return QueryMeterResponse{}, err } + params.Cachable = true + rows, err := h.streaming.QueryMeter(ctx, req.Namespace, m, params) if err != nil { return QueryMeterResponse{}, err diff --git a/api/v3/handlers/meters/query_csv.go b/api/v3/handlers/meters/query_csv.go index 96d286430a..68ca9d868f 100644 --- a/api/v3/handlers/meters/query_csv.go +++ b/api/v3/handlers/meters/query_csv.go @@ -72,6 +72,8 @@ func (h *handler) QueryMeterCSV() QueryMeterCSVHandler { return nil, err } + params.Cachable = true + rows, err := h.streaming.QueryMeter(ctx, req.Namespace, m, params) if err != nil { return nil, err diff --git a/app/common/streaming.go b/app/common/streaming.go index 19480defd1..705a72b73b 100644 --- a/app/common/streaming.go +++ b/app/common/streaming.go @@ -7,6 +7,8 @@ import ( "github.com/ClickHouse/clickhouse-go/v2" "github.com/google/wire" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" "github.com/openmeterio/openmeter/app/config" "github.com/openmeterio/openmeter/openmeter/namespace" @@ -14,6 +16,7 @@ import ( "github.com/openmeterio/openmeter/openmeter/streaming" clickhouseconnector "github.com/openmeterio/openmeter/openmeter/streaming/clickhouse" streamingretry "github.com/openmeterio/openmeter/openmeter/streaming/retry" + "github.com/openmeterio/openmeter/pkg/featuregate" ) var Streaming = wire.NewSet( @@ -27,6 +30,9 @@ func NewStreamingConnector( logger *slog.Logger, progressmanager progressmanager.Service, namespaceManager *namespace.Manager, + featureGate *featuregate.FeatureGateChecker, + meter metric.Meter, + tracer trace.Tracer, ) (streaming.Connector, error) { var connector streaming.Connector var err error @@ -43,6 +49,14 @@ func NewStreamingConnector( EnablePrewhere: conf.EnablePrewhere, EnableDecimalPrecision: conf.EnableDecimalPrecision, ProgressManager: progressmanager, + + QueryCacheEnabled: conf.QueryCache.Enabled, + QueryCacheMinimumCacheableQueryPeriod: conf.QueryCache.MinimumCacheableQueryPeriod, + QueryCacheMinimumCacheableUsageAge: conf.QueryCache.MinimumCacheableUsageAge, + QueryCacheParityCheckSampleRate: conf.QueryCache.ParityCheckSampleRate, + Meter: meter, + Tracer: tracer, + FeatureGate: featureGate, }) if err != nil { return nil, fmt.Errorf("init clickhouse connector: %w", err) diff --git a/app/config/aggregation.go b/app/config/aggregation.go index da77ccbb73..a4a4a507a9 100644 --- a/app/config/aggregation.go +++ b/app/config/aggregation.go @@ -41,6 +41,9 @@ type AggregationConfiguration struct { // When enabled, values are calculated using Decimal128 instead of Float64, providing // higher precision for financial calculations at the cost of some performance. EnableDecimalPrecision bool + + // QueryCache configures the optional meter query-result cache. + QueryCache AggregationQueryCacheConfiguration } // Validate validates the configuration. @@ -57,9 +60,69 @@ func (c AggregationConfiguration) Validate() error { return errors.New("async insert wait is set but async insert is not") } + if err := c.QueryCache.Validate(); err != nil { + return fmt.Errorf("query cache: %w", err) + } + + if c.QueryCache.Enabled && !c.EnableDecimalPrecision { + return errors.New("query cache requires enableDecimalPrecision to be true") + } + + // The cache's late-event invalidation must run after the event is visible + // to SELECTs; async insert without wait acks before the buffer flush, which + // would let a read cache rollups missing the event with no invalidation + // ever coming for it. + if c.QueryCache.Enabled && c.AsyncInsert && !c.AsyncInsertWait { + return errors.New("query cache requires asyncInsertWait when asyncInsert is enabled") + } + return nil } +// AggregationQueryCacheConfiguration configures the optional meter query-result +// cache: an hourly pre-aggregated rollup table that serves the settled history +// of cacheable queries while the fresh tail is scanned live. Off by default. +type AggregationQueryCacheConfiguration struct { + // Enabled turns the cache on. When false the live query path is unchanged and + // no cache table is created. + Enabled bool + // MinimumCacheableQueryPeriod is the minimum queried span for a query to be + // worth caching. + MinimumCacheableQueryPeriod time.Duration + // MinimumCacheableUsageAge is the freshness horizon: only windows entirely + // older than now minus this age are cached; the fresher tail is served live. + MinimumCacheableUsageAge time.Duration + // ParityCheckSampleRate (0..1) samples cache-served queries for shadow + // verification against the live query. A detected mismatch is logged, exposed + // as streaming.query_cache.parity_checks{outcome=mismatch}, and self-heals by + // invalidating the namespace's cache. Each sampled query re-runs the full + // live query, so keep the rate low (e.g. 0.01). 0 disables. + ParityCheckSampleRate float64 +} + +// Validate validates the configuration. +func (c AggregationQueryCacheConfiguration) Validate() error { + if !c.Enabled { + return nil + } + + var errs []error + + if c.MinimumCacheableQueryPeriod <= 0 { + errs = append(errs, errors.New("minimum cacheable query period is required")) + } + + if c.MinimumCacheableUsageAge <= 0 { + errs = append(errs, errors.New("minimum cacheable usage age is required")) + } + + if c.ParityCheckSampleRate < 0 || c.ParityCheckSampleRate > 1 { + errs = append(errs, errors.New("parity check sample rate must be between 0 and 1")) + } + + return errors.Join(errs...) +} + // ClickHouseAggregationConfiguration is the configuration for the ClickHouse aggregation engine type ClickHouseAggregationConfiguration struct { Address string @@ -225,4 +288,10 @@ func ConfigureAggregation(v *viper.Viper) { // Decimal precision v.SetDefault("aggregation.enableDecimalPrecision", false) + + // Query cache (off by default) + v.SetDefault("aggregation.queryCache.enabled", false) + v.SetDefault("aggregation.queryCache.minimumCacheableQueryPeriod", "168h") + v.SetDefault("aggregation.queryCache.minimumCacheableUsageAge", "24h") + v.SetDefault("aggregation.queryCache.parityCheckSampleRate", 0.0) } diff --git a/app/config/config_test.go b/app/config/config_test.go index df32b9e657..f6c6c105e0 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -160,6 +160,11 @@ func TestComplete(t *testing.T) { EventsTableName: "om_events", AsyncInsert: false, AsyncInsertWait: false, + QueryCache: AggregationQueryCacheConfiguration{ + Enabled: false, + MinimumCacheableQueryPeriod: 72 * time.Hour, + MinimumCacheableUsageAge: 24 * time.Hour, + }, }, Entitlements: EntitlementsConfiguration{ GracePeriod: datetime.ISODurationString("P1D"), @@ -494,3 +499,39 @@ func TestComplete(t *testing.T) { assert.Equal(t, expected, actual) } + +// TestAggregationQueryCacheValidation pins the cross-field rule: the query +// cache stores exact Decimal128 rollups, so enabling it without decimal +// precision would ship a silently dead cache (table created, invalidation +// running, zero queries served from it). Validate must reject the combination. +func TestAggregationQueryCacheValidation(t *testing.T) { + base := AggregationConfiguration{ + ClickHouse: ClickHouseAggregationConfiguration{ + Address: "127.0.0.1:9440", + DialTimeout: 10 * time.Second, + MaxOpenConns: 5, + MaxIdleConns: 5, + ConnMaxLifetime: 10 * time.Minute, + BlockBufferSize: 10, + }, + EventsTableName: "om_events", + QueryCache: AggregationQueryCacheConfiguration{ + Enabled: true, + MinimumCacheableQueryPeriod: 72 * time.Hour, + MinimumCacheableUsageAge: 24 * time.Hour, + }, + } + + withoutDecimal := base + withoutDecimal.EnableDecimalPrecision = false + assert.ErrorContains(t, withoutDecimal.Validate(), "enableDecimalPrecision") + + withDecimal := base + withDecimal.EnableDecimalPrecision = true + assert.NoError(t, withDecimal.Validate()) + + disabled := base + disabled.QueryCache.Enabled = false + disabled.EnableDecimalPrecision = false + assert.NoError(t, disabled.Validate(), "the rule only applies when the cache is enabled") +} diff --git a/app/config/testdata/complete.yaml b/app/config/testdata/complete.yaml index 00d90341ee..73f3cb5b3e 100644 --- a/app/config/testdata/complete.yaml +++ b/app/config/testdata/complete.yaml @@ -66,6 +66,11 @@ aggregation: retry: enabled: true + queryCache: + enabled: false + minimumCacheableQueryPeriod: 72h + minimumCacheableUsageAge: 24h + sink: groupId: openmeter-sink-worker minCommitCount: 500 diff --git a/cmd/balance-worker/wire.go b/cmd/balance-worker/wire.go index 725f5090c3..eaaf1ea7a0 100644 --- a/cmd/balance-worker/wire.go +++ b/cmd/balance-worker/wire.go @@ -36,6 +36,7 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl common.Customer, common.Database, common.Feature, + common.FeatureGateChecker, common.Framework, common.Meter, common.Namespace, diff --git a/cmd/balance-worker/wire_gen.go b/cmd/balance-worker/wire_gen.go index 0169415055..b2accf921a 100644 --- a/cmd/balance-worker/wire_gen.go +++ b/cmd/balance-worker/wire_gen.go @@ -12,6 +12,7 @@ import ( "github.com/openmeterio/openmeter/app/config" "github.com/openmeterio/openmeter/openmeter/watermill/driver/kafka" "github.com/openmeterio/openmeter/openmeter/watermill/router" + "github.com/openmeterio/openmeter/pkg/featuregate" "log/slog" ) @@ -162,7 +163,11 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, service, manager) + gate := featuregate.NewNoop() + featureGateConfiguration := conf.FeatureGate + creditsConfiguration := conf.Credits + featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) + connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, service, manager, featureGateChecker, meter, tracer) if err != nil { cleanup7() cleanup6() diff --git a/cmd/billing-worker/wire_gen.go b/cmd/billing-worker/wire_gen.go index dcd11d4390..2cdbed29bf 100644 --- a/cmd/billing-worker/wire_gen.go +++ b/cmd/billing-worker/wire_gen.go @@ -212,7 +212,11 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, progressmanagerService, manager) + gate := featuregate.NewNoop() + featureGateConfiguration := conf.FeatureGate + creditsConfiguration := conf.Credits + featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) + connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, progressmanagerService, manager, featureGateChecker, meter, tracer) if err != nil { cleanup7() cleanup6() @@ -325,7 +329,6 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl return Application{}, nil, err } billingFeatureSwitchesConfiguration := billingConfiguration.FeatureSwitches - creditsConfiguration := conf.Credits repo := common.NewLedgerHistoricalRepo(client) accountRepo := common.NewLedgerAccountRepo(client) accountService := common.NewLedgerAccountService(creditsConfiguration, accountRepo, locker) @@ -349,9 +352,6 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - gate := featuregate.NewNoop() - featureGateConfiguration := conf.FeatureGate - featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) billingRegistry, err := common.NewBillingRegistry(logger, service, adapter, ratingService, customerService, featureConnector, meterService, connector, eventbusPublisher, billingConfiguration, subscriptionServiceWithWorkflow, client, billingFeatureSwitchesConfiguration, creditsConfiguration, tracer, taxcodeService, locker, ledger, balanceQuerier, accountResolver, accountService, breakageService, featureGateChecker) if err != nil { cleanup7() diff --git a/cmd/jobs/internal/wire_gen.go b/cmd/jobs/internal/wire_gen.go index c5b19bf0af..71af893196 100644 --- a/cmd/jobs/internal/wire_gen.go +++ b/cmd/jobs/internal/wire_gen.go @@ -220,7 +220,11 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, progressmanagerService, manager) + gate := featuregate.NewNoop() + featureGateConfiguration := conf.FeatureGate + creditsConfiguration := conf.Credits + featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) + connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, progressmanagerService, manager, featureGateChecker, meter, tracer) if err != nil { cleanup7() cleanup6() @@ -334,7 +338,6 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl return Application{}, nil, err } billingFeatureSwitchesConfiguration := billingConfiguration.FeatureSwitches - creditsConfiguration := conf.Credits repo := common.NewLedgerHistoricalRepo(client) accountRepo := common.NewLedgerAccountRepo(client) accountService := common.NewLedgerAccountService(creditsConfiguration, accountRepo, locker) @@ -358,9 +361,6 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - gate := featuregate.NewNoop() - featureGateConfiguration := conf.FeatureGate - featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) billingRegistry, err := common.NewBillingRegistry(logger, service, adapter, ratingService, customerService, featureConnector, meterService, connector, eventbusPublisher, billingConfiguration, subscriptionServiceWithWorkflow, client, billingFeatureSwitchesConfiguration, creditsConfiguration, tracer, taxcodeService, locker, ledger, balanceQuerier, accountResolver, accountService, breakageService, featureGateChecker) if err != nil { cleanup7() diff --git a/cmd/notification-service/wire.go b/cmd/notification-service/wire.go index d709ac6b12..cfbdc67be8 100644 --- a/cmd/notification-service/wire.go +++ b/cmd/notification-service/wire.go @@ -27,15 +27,15 @@ type Application struct { common.GlobalInitializer common.Migrator - BrokerOptions watermillkafka.BrokerOptions - EventPublisher eventbus.Publisher - EntClient *db.Client - FeatureConnector feature.FeatureConnector - Logger *slog.Logger - MessagePublisher message.Publisher - Meter metric.Meter - Tracer trace.Tracer - Metadata common.Metadata + BrokerOptions watermillkafka.BrokerOptions + EventPublisher eventbus.Publisher + EntClient *db.Client + FeatureConnector feature.FeatureConnector + Logger *slog.Logger + MessagePublisher message.Publisher + Meter metric.Meter + Tracer trace.Tracer + Metadata common.Metadata MeterService meter.Service Notification notification.Service RuntimeMetricsCollector common.RuntimeMetricsCollector @@ -50,6 +50,7 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl common.Config, common.Database, common.Feature, + common.FeatureGateChecker, common.Framework, common.Meter, common.Namespace, diff --git a/cmd/notification-service/wire_gen.go b/cmd/notification-service/wire_gen.go index ec56d42785..e6be1ccbbe 100644 --- a/cmd/notification-service/wire_gen.go +++ b/cmd/notification-service/wire_gen.go @@ -18,6 +18,7 @@ import ( "github.com/openmeterio/openmeter/openmeter/streaming" "github.com/openmeterio/openmeter/openmeter/watermill/driver/kafka" "github.com/openmeterio/openmeter/openmeter/watermill/eventbus" + "github.com/openmeterio/openmeter/pkg/featuregate" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "log/slog" @@ -215,7 +216,11 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v4, logger, progressmanagerService, manager) + gate := featuregate.NewNoop() + featureGateConfiguration := conf.FeatureGate + creditsConfiguration := conf.Credits + featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) + connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v4, logger, progressmanagerService, manager, featureGateChecker, meter, tracer) if err != nil { cleanup7() cleanup6() diff --git a/cmd/server/wire_gen.go b/cmd/server/wire_gen.go index 9975d2d12d..3b63d868bb 100644 --- a/cmd/server/wire_gen.go +++ b/cmd/server/wire_gen.go @@ -259,7 +259,11 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, progressmanagerService, manager) + gate := featuregate.NewNoop() + featureGateConfiguration := conf.FeatureGate + creditsConfiguration := conf.Credits + featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) + connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v3, logger, progressmanagerService, manager, featureGateChecker, meter, tracer) if err != nil { cleanup7() cleanup6() @@ -329,7 +333,6 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl return Application{}, nil, err } billingFeatureSwitchesConfiguration := billingConfiguration.FeatureSwitches - creditsConfiguration := conf.Credits repo := common.NewLedgerHistoricalRepo(client) accountRepo := common.NewLedgerAccountRepo(client) accountService := common.NewLedgerAccountService(creditsConfiguration, accountRepo, locker) @@ -353,9 +356,6 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - gate := featuregate.NewNoop() - featureGateConfiguration := conf.FeatureGate - featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) billingRegistry, err := common.NewBillingRegistry(logger, appService, billingAdapter, ratingService, customerService, featureConnector, service, connector, eventbusPublisher, billingConfiguration, subscriptionServiceWithWorkflow, client, billingFeatureSwitchesConfiguration, creditsConfiguration, tracer, taxcodeService, locker, ledger, balanceQuerier, accountResolver, accountService, breakageService, featureGateChecker) if err != nil { cleanup7() diff --git a/cmd/sink-worker/wire.go b/cmd/sink-worker/wire.go index 90f5a52a24..baf08c38c2 100644 --- a/cmd/sink-worker/wire.go +++ b/cmd/sink-worker/wire.go @@ -28,15 +28,15 @@ import ( type Application struct { common.GlobalInitializer - FlushHandler flushhandler.FlushEventHandler - Logger *slog.Logger - Metadata common.Metadata - Meter metric.Meter - Streaming streaming.Connector - TelemetryServer common.TelemetryServer - TopicProvisioner pkgkafka.TopicProvisioner - TopicResolver *topicresolver.NamespacedTopicResolver - Tracer trace.Tracer + FlushHandler flushhandler.FlushEventHandler + Logger *slog.Logger + Metadata common.Metadata + Meter metric.Meter + Streaming streaming.Connector + TelemetryServer common.TelemetryServer + TopicProvisioner pkgkafka.TopicProvisioner + TopicResolver *topicresolver.NamespacedTopicResolver + Tracer trace.Tracer MeterService meter.Service RuntimeMetricsCollector common.RuntimeMetricsCollector Sink *sink.Sink @@ -50,6 +50,7 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl common.ClickHouse, common.Config, common.Database, + common.FeatureGateChecker, common.Framework, common.KafkaNamespaceResolver, common.NewKafkaTopicProvisioner, diff --git a/cmd/sink-worker/wire_gen.go b/cmd/sink-worker/wire_gen.go index e74cb2152f..2b1745ae02 100644 --- a/cmd/sink-worker/wire_gen.go +++ b/cmd/sink-worker/wire_gen.go @@ -17,6 +17,7 @@ import ( "github.com/openmeterio/openmeter/openmeter/sink/flushhandler" "github.com/openmeterio/openmeter/openmeter/streaming" "github.com/openmeterio/openmeter/openmeter/watermill/driver/kafka" + "github.com/openmeterio/openmeter/pkg/featuregate" kafka2 "github.com/openmeterio/openmeter/pkg/kafka" "github.com/samber/slog-multi" "go.opentelemetry.io/otel/metric" @@ -146,7 +147,11 @@ func initializeApplication(ctx context.Context, conf config.Configuration) (Appl cleanup() return Application{}, nil, err } - connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v4, logger, service, manager) + gate := featuregate.NewNoop() + featureGateConfiguration := conf.FeatureGate + creditsConfiguration := conf.Credits + featureGateChecker := common.NewFeatureGateChecker(gate, featureGateConfiguration, creditsConfiguration) + connector, err := common.NewStreamingConnector(ctx, aggregationConfiguration, v4, logger, service, manager, featureGateChecker, meter, tracer) if err != nil { cleanup6() cleanup5() diff --git a/config.example.yaml b/config.example.yaml index c3d00302c9..c397f8a7ed 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -142,6 +142,37 @@ postgres: url: postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable autoMigrate: ent # Runs migrations as part of the service startup, valid values are: ent, migration, false +# aggregation configures the ClickHouse-backed meter aggregation engine. +# aggregation: +# # Required by queryCache: the cache stores exact Decimal128 rollups, so only +# # decimal-precision queries can be served from it. +# enableDecimalPrecision: true +# # queryCache is an optional meter query-result cache. When enabled, the settled +# # (older than minimumCacheableUsageAge) history of a meter query is served from +# # an hourly pre-aggregated rollup table, while the fresh tail is scanned live and +# # merged in a single query. It is OFF by default; enabling it never changes query +# # results (only settled, provably-identical windows are cached) and creates one +# # extra rollup table. Requires a ClickHouse topology where a synchronous INSERT +# # is immediately visible to the next SELECT (single node, sticky replica, or +# # quorum writes + sequential-consistency reads). +# queryCache: +# enabled: false +# # Per-namespace enablement is handled by the feature gate (flag +# # om_ff_query_cache_enabled): it is enabled for every namespace by default, +# # and a configured feature-gate backend can restrict it. +# # minimumCacheableQueryPeriod: only queries spanning at least this long are cached. +# minimumCacheableQueryPeriod: 168h +# # minimumCacheableUsageAge: freshness horizon. Only windows entirely older than +# # now minus this age are cached; the fresher tail is always served live. Late +# # events older than this horizon invalidate the affected namespace's cache. +# minimumCacheableUsageAge: 24h +# # parityCheckSampleRate (0..1): shadow-verify this fraction of cache-served +# # queries against the live query, off the request path. A mismatch is logged, +# # counted (streaming.query_cache.parity_checks{outcome=mismatch} — alert on > 0) +# # and self-heals by invalidating the namespace's cache. Each sampled query +# # re-runs the full live query; keep the rate low. 0 disables. +# parityCheckSampleRate: 0.01 + meters: # Sample meter to count API requests - slug: api_requests_total # Unique identifier for the meter diff --git a/openmeter/billing/charges/usagebased/service/rating/service_test.go b/openmeter/billing/charges/usagebased/service/rating/service_test.go index 28b272b566..c6c11089bf 100644 --- a/openmeter/billing/charges/usagebased/service/rating/service_test.go +++ b/openmeter/billing/charges/usagebased/service/rating/service_test.go @@ -357,6 +357,48 @@ func TestGetDetailedRatingForUsageFiltersQuantityByServicePeriodToAndStoredAtLT( require.Equal(t, float64(2), out.Quantity.InexactFloat64()) } +// TestGetDetailedRatingForUsageAlwaysReadsRawMeterData is the billing-safety +// guard: invoicing must never read the meter query-result cache — invoiced +// quantities have to reflect the exact stored events. Billing relies on +// QueryParams.Cachable defaulting to false, so this asserts every meter query +// the rating path issues leaves Cachable false (which routes it to the live +// path). It must fail if a billing call site ever sets Cachable=true. +func TestGetDetailedRatingForUsageAlwaysReadsRawMeterData(t *testing.T) { + t.Parallel() + + servicePeriod := timeutil.ClosedPeriod{ + From: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), + To: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC), + } + servicePeriodTo := servicePeriod.From.Add(24 * time.Hour) + storedAtLT := servicePeriod.From.Add(48 * time.Hour) + + streamingConnector := streamingtestutils.NewMockStreamingConnector(t) + streamingConnector.AddSimpleEvent("meter-1", 2, servicePeriod.From.Add(time.Hour)) + + svc, err := New(Config{ + StreamingConnector: streamingConnector, + RatingService: &stubRatingService{}, + DetailedLinesFetcher: passthroughDetailedLinesFetcher, + }) + require.NoError(t, err) + + _, err = svc.GetDetailedRatingForUsage(t.Context(), GetDetailedRatingForUsageInput{ + Charge: newDetailedRatingTestCharge(servicePeriod, usagebased.RealizationRuns{}), + ServicePeriodTo: servicePeriodTo, + StoredAtLT: storedAtLT, + Customer: newDetailedRatingTestCustomer(), + FeatureMeter: newDetailedRatingTestFeatureMeter(), + }) + require.NoError(t, err) + + recorded := streamingConnector.RecordedQueryParams() + require.NotEmpty(t, recorded, "expected at least one meter query") + for i, params := range recorded { + require.False(t, params.Cachable, "billing meter query[%d] must not opt into the cache", i) + } +} + func TestGetTotalsForUsageMinimumCommitment(t *testing.T) { t.Parallel() diff --git a/openmeter/cost/adapter/adapter.go b/openmeter/cost/adapter/adapter.go index 533feb4a6d..e51fd5a3c4 100644 --- a/openmeter/cost/adapter/adapter.go +++ b/openmeter/cost/adapter/adapter.go @@ -105,6 +105,8 @@ func (a *adapter) QueryFeatureCost(ctx context.Context, input cost.QueryFeatureC // Track which ones were added internally so we can aggregate across them if the user didn't request them. internalGroupByKeys := addLLMGroupByKeys(feat, ¶ms) + params.Cachable = true + // Query usage data rows, err := a.streamingConnector.QueryMeter(ctx, input.Namespace, m, params) if err != nil { diff --git a/openmeter/credit/balance/usage.go b/openmeter/credit/balance/usage.go index a5b4f90f19..1e67a65896 100644 --- a/openmeter/credit/balance/usage.go +++ b/openmeter/credit/balance/usage.go @@ -44,6 +44,8 @@ func (u *usageQuerier) QueryUsage(ctx context.Context, ownerID models.Namespaced return 0.0, err } + params.Cachable = true + owner, err := u.DescribeOwner(ctx, ownerID) if err != nil { return 0.0, err diff --git a/openmeter/entitlement/metered/balance.go b/openmeter/entitlement/metered/balance.go index 904305b63f..130eefc8be 100644 --- a/openmeter/entitlement/metered/balance.go +++ b/openmeter/entitlement/metered/balance.go @@ -377,5 +377,7 @@ func (e *connector) queryMeter(ctx context.Context, namespace string, m meter.Me }, nil } + params.Cachable = true + return e.streamingConnector.QueryMeter(ctx, namespace, m, params) } diff --git a/openmeter/entitlement/metered/balance_test.go b/openmeter/entitlement/metered/balance_test.go index d07bb00814..d0dd516369 100644 --- a/openmeter/entitlement/metered/balance_test.go +++ b/openmeter/entitlement/metered/balance_test.go @@ -836,6 +836,36 @@ func TestGetEntitlementBalance(t *testing.T) { assert.Len(t, snaps, 2) // one for the initial and one we made last time }, }, + { + // Entitlement balance queries opt into the query-result cache: every + // meter query the balance calculation issues must set Cachable=true so + // the connector's cacheability gate can serve them from the cache when + // enabled. (Whether it is actually cached is the gate's decision.) + name: "Should opt entitlement meter queries into the cache", + run: func(t *testing.T, connector meteredentitlement.Connector, deps *dependencies) { + ctx := t.Context() + startTime := getAnchor(t) + + randName := testutils.NameGenerator.Generate() + cust := createCustomerAndSubject(t, deps.subjectService, deps.customerService, namespace, randName.Key, randName.Name) + + inp := getEntitlement(t, feat, cust.GetUsageAttribution()) + inp.MeasureUsageFrom = &startTime + entitlement, err := deps.entitlementRepo.CreateEntitlement(ctx, inp) + require.NoError(t, err) + + deps.streamingConnector.AddSimpleEvent(meterSlug, 100, startTime.Add(time.Minute)) + + _, err = connector.GetEntitlementBalance(ctx, models.NamespacedID{Namespace: namespace, ID: entitlement.ID}, startTime.Add(time.Hour)) + require.NoError(t, err) + + recorded := deps.streamingConnector.RecordedQueryParams() + require.NotEmpty(t, recorded, "expected at least one meter query") + for i, params := range recorded { + require.True(t, params.Cachable, "entitlement meter query[%d] must opt into the cache", i) + } + }, + }, } for _, tc := range tt { diff --git a/openmeter/meter/httphandler/query.go b/openmeter/meter/httphandler/query.go index 92ec612423..a17c24d57c 100644 --- a/openmeter/meter/httphandler/query.go +++ b/openmeter/meter/httphandler/query.go @@ -128,6 +128,8 @@ func (h *handler) QueryMeter() QueryMeterHandler { return nil, fmt.Errorf("failed to construct query meter params: %w", err) } + params.Cachable = true + rows, err := h.streaming.QueryMeter(ctx, request.namespace, meter, params) if err != nil { return nil, fmt.Errorf("failed to query meter: %w", err) @@ -191,6 +193,8 @@ func (h *handler) QueryMeterPost() QueryMeterPostHandler { return nil, fmt.Errorf("failed to construct query meter params: %w", err) } + params.Cachable = true + rows, err := h.streaming.QueryMeter(ctx, request.namespace, meter, params) if err != nil { return nil, fmt.Errorf("failed to query meter: %w", err) diff --git a/openmeter/meter/httphandler/query_csv.go b/openmeter/meter/httphandler/query_csv.go index 3fb14af463..7f60956230 100644 --- a/openmeter/meter/httphandler/query_csv.go +++ b/openmeter/meter/httphandler/query_csv.go @@ -68,6 +68,8 @@ func (h *handler) QueryMeterCSV() QueryMeterCSVHandler { return nil, fmt.Errorf("failed to construct query meter params: %w", err) } + params.Cachable = true + rows, err := h.streaming.QueryMeter(ctx, request.namespace, meter, params) if err != nil { return nil, fmt.Errorf("failed to query meter: %w", err) @@ -140,6 +142,8 @@ func (h *handler) QueryMeterPostCSV() QueryMeterPostCSVHandler { return nil, fmt.Errorf("failed to construct query meter params: %w", err) } + params.Cachable = true + rows, err := h.streaming.QueryMeter(ctx, request.namespace, meter, params) if err != nil { return nil, fmt.Errorf("failed to query meter: %w", err) diff --git a/openmeter/streaming/clickhouse/connector.go b/openmeter/streaming/clickhouse/connector.go index 837ba886f2..ec844e7ec8 100644 --- a/openmeter/streaming/clickhouse/connector.go +++ b/openmeter/streaming/clickhouse/connector.go @@ -10,11 +10,15 @@ import ( "time" "github.com/ClickHouse/clickhouse-go/v2" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + tracenoop "go.opentelemetry.io/otel/trace/noop" meterpkg "github.com/openmeterio/openmeter/openmeter/meter" "github.com/openmeterio/openmeter/openmeter/progressmanager" progressmanagerentity "github.com/openmeterio/openmeter/openmeter/progressmanager/entity" "github.com/openmeterio/openmeter/openmeter/streaming" + "github.com/openmeterio/openmeter/pkg/featuregate" "github.com/openmeterio/openmeter/pkg/models" ) @@ -23,6 +27,10 @@ var _ streaming.Connector = (*Connector)(nil) // Connector implements `ingest.Connector" and `namespace.Handler interfaces. type Connector struct { config Config + // queryCacheMetrics is nil when Config.Meter is unset. + queryCacheMetrics *queryCacheMetrics + // tracer is Config.Tracer or a noop. + tracer trace.Tracer } type Config struct { @@ -38,6 +46,45 @@ type Config struct { EnableDecimalPrecision bool ProgressManager progressmanager.Service SkipCreateTables bool + + // QueryCacheEnabled turns on the optional meter query-result cache: an + // hourly pre-aggregated rollup table that serves the settled history of + // cacheable queries while the fresh tail is scanned live. + // + // Topology requirement: the cache populates lazily on read (synchronous + // INSERT...SELECT, deliberately NOT using InsertQuerySettings so async_insert + // cannot be applied to it) and the merge SELECT that follows must see that + // insert. On a single node this always holds; behind a load-balanced + // multi-replica endpoint without quorum writes the merge can read a replica + // the insert has not replicated to yet and silently undercount the settled + // range. Point the connector at a single node / sticky replica, or configure + // quorum + sequential-consistency on the deployment, before enabling. + QueryCacheEnabled bool + // QueryCacheMinimumCacheableQueryPeriod is the minimum queried span for a + // query to be worth caching. + QueryCacheMinimumCacheableQueryPeriod time.Duration + // QueryCacheMinimumCacheableUsageAge is the freshness horizon: only windows + // entirely older than now-this-age are cached. The + // fresher tail is always served live. + QueryCacheMinimumCacheableUsageAge time.Duration + // QueryCacheParityCheckSampleRate (0..1) samples cache-served queries for + // shadow verification: the live query is re-run off the request path and + // compared to the cached result; a mismatch is logged, counted + // (streaming.query_cache.parity_checks{outcome=mismatch}) and self-heals by + // invalidating the namespace's cache. 0 disables. Each sampled query costs + // one extra full live query. + QueryCacheParityCheckSampleRate float64 + // Meter, when set, exposes the query cache's health counters (queries, + // populate errors, invalidations, parity checks) and duration histograms. + // Optional: nil disables metrics without affecting behavior. + Meter metric.Meter + // Tracer, when set, emits spans for the query cache operations (cached + // query, populate, invalidate, parity check). Optional: nil falls back to a + // noop tracer. The live query path is deliberately NOT instrumented here so + // it stays byte-identical to the pre-cache behavior. + Tracer trace.Tracer + // FeatureGate gates the query cache per namespace. + FeatureGate *featuregate.FeatureGateChecker } func (c Config) Validate() error { @@ -61,6 +108,33 @@ func (c Config) Validate() error { return fmt.Errorf("progress manager is required") } + if c.QueryCacheEnabled { + if !c.EnableDecimalPrecision { + return fmt.Errorf("query cache requires decimal precision to be enabled") + } + + // The late-event invalidation must run AFTER the event is visible to + // SELECTs. Async insert without wait acks before the buffer flush, so a + // read repopulating inside the flush gap would store a coverage claim + // over rollups missing that event — and no later invalidation ever + // comes for it. + if c.AsyncInsert && !c.AsyncInsertWait { + return fmt.Errorf("query cache requires synchronous event visibility: enable async insert wait or disable async insert") + } + + if c.QueryCacheMinimumCacheableQueryPeriod <= 0 { + return fmt.Errorf("minimum cacheable query period is required") + } + + if c.QueryCacheMinimumCacheableUsageAge <= 0 { + return fmt.Errorf("minimum cacheable usage age is required") + } + + if c.QueryCacheParityCheckSampleRate < 0 || c.QueryCacheParityCheckSampleRate > 1 { + return fmt.Errorf("query cache parity check sample rate must be between 0 and 1") + } + } + return nil } @@ -73,6 +147,19 @@ func New(ctx context.Context, config Config) (*Connector, error) { // Create the connector connector := &Connector{ config: config, + tracer: config.Tracer, + } + + if connector.tracer == nil { + connector.tracer = tracenoop.NewTracerProvider().Tracer("openmeter.streaming.clickhouse") + } + + if config.Meter != nil { + var err error + connector.queryCacheMetrics, err = newQueryCacheMetrics(config.Meter) + if err != nil { + return nil, fmt.Errorf("create query cache metrics: %w", err) + } } if !config.SkipCreateTables { @@ -92,6 +179,38 @@ func (c *Connector) createTable(ctx context.Context) error { return fmt.Errorf("create events table in clickhouse: %w", err) } + // Create the query-result cache rollup table only when the cache is enabled, + // so a disabled cache leaves the schema byte-identical to today. + if c.config.QueryCacheEnabled { + if err := c.createMeterQueryRowCacheTable(ctx); err != nil { + return fmt.Errorf("create meter query cache table in clickhouse: %w", err) + } + } + + return nil +} + +// createMeterQueryRowCacheTable creates the rollup table and its coverage +// companion; they always exist (and are invalidated) together. +func (c *Connector) createMeterQueryRowCacheTable(ctx context.Context) error { + table := createMeterQueryRowCacheTable{ + Database: c.config.Database, + TableName: meterQueryRowCacheTable, + } + + if err := c.config.ClickHouse.Exec(ctx, table.toSQL()); err != nil { + return fmt.Errorf("create meter query cache table: %w", err) + } + + coverageTable := createMeterQueryRowCacheCoverageTable{ + Database: c.config.Database, + TableName: meterQueryRowCacheCoverageTable, + } + + if err := c.config.ClickHouse.Exec(ctx, coverageTable.toSQL()); err != nil { + return fmt.Errorf("create meter query cache coverage table: %w", err) + } + return nil } @@ -163,12 +282,19 @@ func (c *Connector) QueryMeter(ctx context.Context, namespace string, meter mete EnableDecimalPrecision: c.config.EnableDecimalPrecision, } - // Load cached rows if any var err error var values []meterpkg.MeterQueryRow - // If the client ID is set, we track track the progress of the query - if params.ClientID != nil { + // If the query is provably reproducible from the hourly rollup, serve it from + // the cache (settled history from the rollup + fresh tail scanned live, + // merged in one SQL statement). Otherwise the live path runs unchanged. + if c.canQueryBeCached(namespace, meter, params) { + values, err = c.queryMeterCached(ctx, query) + if err != nil { + return values, fmt.Errorf("query meter cached: %w", err) + } + } else if params.ClientID != nil { + // If the client ID is set, we track track the progress of the query values, err = c.queryMeterWithProgress(ctx, namespace, *params.ClientID, query) if err != nil { return values, fmt.Errorf("query meter with progress: %w", err) @@ -284,6 +410,16 @@ func (c *Connector) BatchInsert(ctx context.Context, rawEvents []streaming.RawEv return fmt.Errorf("failed to batch insert raw events: %w", err) } + // Late events mutate already-settled windows: wipe the affected namespaces' cached rollups. + namespacesToInvalidate := c.findNamespacesToInvalidateCache(rawEvents) + if err := c.invalidateMeterQueryRowCache(ctx, namespacesToInvalidate); err != nil { + // Do not fail the insert on invalidation error. + // A stale cache is corrected on the next late event or by re-population. + c.config.Logger.Error("failed to invalidate meter query cache", "error", err, "namespaces", namespacesToInvalidate) + } else if len(namespacesToInvalidate) > 0 { + c.queryCacheMetrics.recordInvalidation(ctx, "late_event", len(namespacesToInvalidate)) + } + return nil } diff --git a/openmeter/streaming/clickhouse/meter_query.go b/openmeter/streaming/clickhouse/meter_query.go index 7f1694cf95..a5f753f981 100644 --- a/openmeter/streaming/clickhouse/meter_query.go +++ b/openmeter/streaming/clickhouse/meter_query.go @@ -87,6 +87,29 @@ func (d *queryMeter) from() *time.Time { return d.Meter.EventFrom } +// rawValueExpr returns the per-event value extraction expression (WITHOUT the +// aggregate wrapper) for the meter's value property. +func (d *queryMeter) rawValueExpr() string { + getColumn := columnFactory(d.EventsTableName) + valueProperty := escapeJSONPathLiteral(*d.Meter.ValueProperty) + + if d.EnableDecimalPrecision { + return fmt.Sprintf("toDecimal128OrNull(nullIf(JSON_VALUE(%s, '%s'), 'null'), 19)", getColumn("data"), valueProperty) + } + + // JSON_VALUE returns an empty string if the JSON Path is not found. With toFloat64OrNull we convert it to NULL so the aggregation function can handle it properly. + return fmt.Sprintf("ifNotFinite(toFloat64OrNull(JSON_VALUE(%s, '%s')), null)", getColumn("data"), valueProperty) +} + +// groupByValueExpr returns the SQL expression that extracts one meter group-by +// dimension's value from the raw event. +func (d *queryMeter) groupByValueExpr(groupByKey string) string { + getColumn := columnFactory(d.EventsTableName) + groupByJSONPath := escapeJSONPathLiteral(d.Meter.GroupBy[groupByKey]) + + return fmt.Sprintf("JSON_VALUE(%s, '%s')", getColumn("data"), groupByJSONPath) +} + // toCountRowSQL returns the SQL query for the estimated number of rows. // This estimate is useful for query progress tracking. // We only filter by columns that are in the ClickHouse table order. @@ -209,18 +232,9 @@ func (d *queryMeter) toSQL() (string, []interface{}, error) { case meterpkg.MeterAggregationUniqueCount: selectColumns = append(selectColumns, fmt.Sprintf("%s(nullIf(JSON_VALUE(%s, '%s'), 'null')) AS value", sqlAggregation, getColumn("data"), escapeJSONPathLiteral(*d.Meter.ValueProperty))) case meterpkg.MeterAggregationLatest: - if d.EnableDecimalPrecision { - selectColumns = append(selectColumns, fmt.Sprintf("%s(toDecimal128OrNull(nullIf(JSON_VALUE(%s, '%s'), 'null'), 19), %s) AS value", sqlAggregation, getColumn("data"), escapeJSONPathLiteral(*d.Meter.ValueProperty), timeColumn)) - } else { - selectColumns = append(selectColumns, fmt.Sprintf("%s(ifNotFinite(toFloat64OrNull(JSON_VALUE(%s, '%s')), null), %s) AS value", sqlAggregation, getColumn("data"), escapeJSONPathLiteral(*d.Meter.ValueProperty), timeColumn)) - } + selectColumns = append(selectColumns, fmt.Sprintf("%s(%s, %s) AS value", sqlAggregation, d.rawValueExpr(), timeColumn)) default: - if d.EnableDecimalPrecision { - selectColumns = append(selectColumns, fmt.Sprintf("%s(toDecimal128OrNull(nullIf(JSON_VALUE(%s, '%s'), 'null'), 19)) AS value", sqlAggregation, getColumn("data"), escapeJSONPathLiteral(*d.Meter.ValueProperty))) - } else { - // JSON_VALUE returns an empty string if the JSON Path is not found. With toFloat64OrNull we convert it to NULL so the aggregation function can handle it properly. - selectColumns = append(selectColumns, fmt.Sprintf("%s(ifNotFinite(toFloat64OrNull(JSON_VALUE(%s, '%s')), null)) AS value", sqlAggregation, getColumn("data"), escapeJSONPathLiteral(*d.Meter.ValueProperty))) - } + selectColumns = append(selectColumns, fmt.Sprintf("%s(%s) AS value", sqlAggregation, d.rawValueExpr())) } for _, groupByKey := range d.GroupBy { @@ -239,8 +253,7 @@ func (d *queryMeter) toSQL() (string, []interface{}, error) { // Group by columns need to be parsed from the JSON data groupByColumn := sqlbuilder.Escape(groupByKey) - groupByJSONPath := escapeJSONPathLiteral(d.Meter.GroupBy[groupByKey]) - selectColumn := fmt.Sprintf("JSON_VALUE(%s, '%s') as %s", getColumn("data"), groupByJSONPath, groupByColumn) + selectColumn := fmt.Sprintf("%s as %s", d.groupByValueExpr(groupByKey), groupByColumn) selectColumns = append(selectColumns, selectColumn) groupByColumns = append(groupByColumns, groupByColumn) diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache.go b/openmeter/streaming/clickhouse/meterqueryrow_cache.go new file mode 100644 index 0000000000..75ca494ec0 --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache.go @@ -0,0 +1,320 @@ +package clickhouse + +import ( + "fmt" + "hash/fnv" + "sort" + "strings" + "time" + + "github.com/alpacahq/alpacadecimal" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" +) + +// meterQueryRowCacheTable is the name of the pre-aggregated rollup table that +// backs the optional meter query-result cache. +const meterQueryRowCacheTable = "meterqueryrow_cache" + +// cacheGrainInterval is the tumble interval of the cache's hourly grain. +const cacheGrainInterval = "toIntervalHour(1)" + +// createMeterQueryRowCacheTable builds the DDL for the rollup table. +// +// Storage decisions: +// - ReplacingMergeTree(created_at): combined with the settled-window +// write-once invariant, any duplicate produced by concurrent lazy +// population is byte-identical, so replacing by the sort key is safe and +// collapses them. Plain MergeTree would let the read-time UNION double-count +// SUM/COUNT. +// - ORDER BY includes group_by so distinct group-by combos in the same window +// are distinct rows, not replacement collisions. +// - sum_value/min_value/max_value are Nullable: a settled window whose events +// all have a null value property yields a NULL sum/min/max in the live query +// (ClickHouse sum/min/max over all-NULL is NULL, not 0), and the live scan +// skips that row. A non-nullable column would store 0 there and both emit a +// spurious row and corrupt the enclosing day's aggregate. count_value stays +// non-nullable: count(*) counts rows regardless of value nullness, and a +// COUNT meter emits that window as a row just as live does. +// - meter_slug (the per-namespace unique meter key) is in the ORDER BY key: +// the (namespace, event_type) pair does NOT identify a meter — many meters +// can share an event type (uniqueness is on (namespace, key)). Two such +// meters (e.g. SUM of tokens vs SUM of latency) would otherwise share a sort +// key and one would clobber the other, so a query would read the other +// meter's value. type is kept for locality/pruning but meter_slug is the +// discriminator that makes rows meter-specific. +// - meter_hash pins the meter's EXTRACTION SHAPE (aggregation, value property +// and sorted group-by map). Meter definitions are mutable (UpdateMeter can add +// or remove group-by dimensions), and the group_by array is aligned to the +// CURRENT sorted paths — without the hash, rows written before and after a +// definition change coexist under different sort keys and the cache leg +// double-counts every settled hour. With the hash in the key and the read +// filter, a shape change simply orphans the old rows (never read again). +// - The query's WindowTimeZone is deliberately NOT part of the key: rows are +// timezone-agnostic hourly-UTC partials (populate hard-codes UTC), and the +// timezone only re-windows them at read time. The gate admits only +// whole-hour-offset zones, where every tz-local window boundary — including +// DST transition instants — falls on a UTC hour boundary, so one stored row +// set serves every admitted timezone exactly (pinned by +// TestQueryCacheCrossTimezoneSharing). Keying by timezone would store +// byte-identical copies per zone for no correctness gain. +// - created_at is DateTime64(3): it is both the ReplacingMergeTree version and +// the argMax pick in the read-time collapse; second granularity would make +// newest-wins ties far more likely under racing populates. +// - TTL garbage-collects rows. Gap tracking means covered ranges are NOT +// rewritten on read, so actively queried rows DO age: the coverage claim's +// trust window (cacheCoverageTrustWindow, one day shorter than this TTL) +// expires first and forces a full re-populate that rewrites the rows and +// resets their clock before any can expire. Rows orphaned by a meter shape +// change or an idle meter age out unreferenced. +type createMeterQueryRowCacheTable struct { + Database string + TableName string +} + +func (t createMeterQueryRowCacheTable) toSQL() string { + tableName := getTableName(t.Database, t.TableName) + + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + namespace String, + type LowCardinality(String), + meter_slug String, + meter_hash String, + windowstart DateTime, + subject String, + group_by Array(String), + sum_value Nullable(Decimal128(19)), + count_value UInt64, + min_value Nullable(Decimal128(19)), + max_value Nullable(Decimal128(19)), + created_at DateTime64(3) DEFAULT now64(3) +) +ENGINE = ReplacingMergeTree(created_at) +PARTITION BY toYYYYMM(windowstart) +ORDER BY (namespace, type, meter_slug, meter_hash, windowstart, subject, group_by) +TTL toDateTime(created_at) + toIntervalDay(90)`, tableName) +} + +// meterShapeHash fingerprints everything that determines what a cache row's +// value and group_by array MEAN for a meter: the event type, the aggregation, +// the value property, and the sorted group-by map (keys and JSON paths). It +// deliberately excludes fields that don't change row semantics (name, +// description, EventFrom). Rows are written and read pinned to this hash, so +// any meter definition change makes old rows unreachable instead of silently +// combining two incompatible shapes into one aggregate. +// +// EventType is included even though rollup rows also carry a type column: +// the coverage claim's identity is only (namespace, slug, hash), and a meter +// deleted and recreated under the same slug with a different event type (the +// slug unique index is scoped to non-deleted meters) but identical +// aggregation/value/group-by would otherwise inherit the old claim while the +// cache leg's type filter matches none of the claimed rows — the whole +// settled range would read as empty. +func meterShapeHash(meter meterpkg.Meter) string { + h := fnv.New64a() + + _, _ = h.Write([]byte(meter.EventType)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(meter.Aggregation)) + _, _ = h.Write([]byte{0}) + if meter.ValueProperty != nil { + _, _ = h.Write([]byte(*meter.ValueProperty)) + } + _, _ = h.Write([]byte{0}) + + for _, key := range cacheGroupByPaths(meter) { + _, _ = h.Write([]byte(key)) + _, _ = h.Write([]byte{1}) + _, _ = h.Write([]byte(meter.GroupBy[key])) + _, _ = h.Write([]byte{2}) + } + + return fmt.Sprintf("%016x", h.Sum64()) +} + +// cacheGroupByPaths returns the meter's group-by JSON-path dimension keys in +// sorted order. This is the order the group_by Array(String) is aligned to, +// both when populating the cache and when reading it back, so array index i +// always corresponds to the same meter dimension. Subject (a top-level column) +// and customer_id (derived from subject) are NOT stored in the array — subject +// has its own column and customer_id queries are not cached. +func cacheGroupByPaths(meter meterpkg.Meter) []string { + paths := make([]string, 0, len(meter.GroupBy)) + for key := range meter.GroupBy { + paths = append(paths, key) + } + sort.Strings(paths) + return paths +} + +// aggCacheColumn maps a meter aggregation to the cache column that stores its +// per-window partial and the SQL function that recombines partials across +// windows/groups. Only the four mergeable aggregations are supported; callers +// must gate on canQueryBeCached before reaching here. +func aggCacheColumn(agg meterpkg.MeterAggregation) (column string, recombine string, ok bool) { + switch agg { + case meterpkg.MeterAggregationSum: + return "sum_value", "sum", true + case meterpkg.MeterAggregationCount: + // Counts recombine by summing the per-window counts. + return "count_value", "sum", true + case meterpkg.MeterAggregationMin: + return "min_value", "min", true + case meterpkg.MeterAggregationMax: + return "max_value", "max", true + default: + return "", "", false + } +} + +// populateMeterQueryRowCache rolls up hourly windows in [From, Cutoff) into the +// cache table for a single meter, storing one row per (hourly window, subject, +// full group-by combo). From MUST be hour-aligned (the caller passes the +// head-ceiled boundary) so every stored window is COMPLETE: an incomplete window +// would store a different value for the same key depending on the query's `from`, +// and the read-time collapse would then serve the wrong total. +// +// The filter is (namespace, type, time range) ONLY — deliberately NOT the +// query's subject/group-by filters — so the rollup is subject- and group- +// complete for the meter and can serve any later subject/group subset. The +// value/group extraction reuses queryMeter.rawValueExpr / groupByValueExpr so +// stored values are byte-identical to the live query. +// +// Only the aggregate column matching the meter's aggregation carries a real +// value; the others are filler (NULL, or 0 for the non-nullable count_value). +// A meter has exactly one aggregation, so every query over this meter's type +// reads the same column and the filler columns are never read back. This also +// avoids dereferencing a nil ValueProperty for COUNT meters. +type populateMeterQueryRowCache struct { + Database string + CacheTableName string + EventsTableName string + query queryMeter + From time.Time + Cutoff time.Time +} + +func (p populateMeterQueryRowCache) toSQL() (string, []interface{}) { + cacheTable := getTableName(p.Database, p.CacheTableName) + eventsTable := getTableName(p.Database, p.EventsTableName) + getColumn := columnFactory(p.EventsTableName) + timeColumn := getColumn("time") + + paths := cacheGroupByPaths(p.query.Meter) + + groupExprs := make([]string, len(paths)) + for i, path := range paths { + groupExprs[i] = p.query.groupByValueExpr(path) + } + groupArray := "[]" + if len(groupExprs) > 0 { + groupArray = "[" + strings.Join(groupExprs, ", ") + "]" + } + + windowStartExpr := fmt.Sprintf("tumbleStart(%s, %s, 'UTC')", timeColumn, cacheGrainInterval) + + // Only the meter's own aggregation column gets a real value; the others are + // filler — NULL for the Nullable sum/min/max, 0 for the non-nullable + // count_value. Filler is constant per meter so it compresses to almost + // nothing, whereas storing the real count(*) on SUM/MIN/MAX rows costs + // roughly a fifth of the table for data no read ever touches + // (aggCacheColumn routes each meter's reads to its owning column only). + // COUNT meters have no ValueProperty, so we must not emit a value + // aggregate for them. + sumExpr, countExpr, minExpr, maxExpr := "CAST(NULL AS Nullable(Decimal128(19)))", "0", "CAST(NULL AS Nullable(Decimal128(19)))", "CAST(NULL AS Nullable(Decimal128(19)))" + if p.query.Meter.Aggregation == meterpkg.MeterAggregationCount { + countExpr = "count(*)" + } + if p.query.Meter.ValueProperty != nil { + rawValue := p.query.rawValueExpr() + switch p.query.Meter.Aggregation { + case meterpkg.MeterAggregationSum: + sumExpr = fmt.Sprintf("sum(%s)", rawValue) + case meterpkg.MeterAggregationMin: + minExpr = fmt.Sprintf("min(%s)", rawValue) + case meterpkg.MeterAggregationMax: + maxExpr = fmt.Sprintf("max(%s)", rawValue) + } + } + + groupByTerms := append([]string{"namespace", "type", windowStartExpr, "subject"}, groupExprs...) + + sql := fmt.Sprintf(`INSERT INTO %s (namespace, type, meter_slug, meter_hash, windowstart, subject, group_by, sum_value, count_value, min_value, max_value) +SELECT + namespace, + type, + ? AS meter_slug, + ? AS meter_hash, + %s AS windowstart, + subject, + %s AS group_by, + %s AS sum_value, + %s AS count_value, + %s AS min_value, + %s AS max_value +FROM %s +WHERE namespace = ? AND type = ? AND %s >= ? AND %s < ? +GROUP BY %s`, + cacheTable, + windowStartExpr, + groupArray, + sumExpr, + countExpr, + minExpr, + maxExpr, + eventsTable, + timeColumn, timeColumn, + strings.Join(groupByTerms, ", "), + ) + + args := []interface{}{ + p.query.Meter.Key, + meterShapeHash(p.query.Meter), + p.query.Namespace, + p.query.Meter.EventType, + p.From.Unix(), + p.Cutoff.Unix(), + } + + return sql, args +} + +// cachedMeterRow is one merged result row as returned from the cache/live merge, +// carried as exact decimals end-to-end (never float) until the final +// MeterQueryRow conversion. Duplicate keys cannot occur in the merged output: +// the outer GROUP BY collapses every (window, subject, group-by) tuple to one +// row, and the cache leg's argMax-over-created_at collapse deterministically +// picks the newest stored rollup for each key before recombination. +type cachedMeterRow struct { + WindowStart time.Time + WindowEnd time.Time + Subject string + GroupBy []string + Value alpacadecimal.Decimal + // ValueValid is false when the recombined value is NULL + ValueValid bool +} + +// deleteMeterQueryRowCacheForNamespaces builds a DELETE that wipes all cached +// rows for the given namespaces. +type deleteMeterQueryRowCacheForNamespaces struct { + Database string + TableName string + Namespaces []string +} + +func (d deleteMeterQueryRowCacheForNamespaces) toSQL() (string, []interface{}) { + tableName := getTableName(d.Database, d.TableName) + + placeholders := strings.Repeat("?, ", len(d.Namespaces)) + placeholders = strings.TrimSuffix(placeholders, ", ") + + sql := fmt.Sprintf("DELETE FROM %s WHERE namespace IN (%s)", tableName, placeholders) + + args := make([]any, 0, len(d.Namespaces)) + for _, ns := range d.Namespaces { + args = append(args, ns) + } + + return sql, args +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_coverage.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_coverage.go new file mode 100644 index 0000000000..6bca84aafd --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_coverage.go @@ -0,0 +1,287 @@ +package clickhouse + +import ( + "fmt" + "strings" + "time" +) + +// meterQueryRowCacheCoverageTable records, per meter shape, the contiguous +// settled range whose hourly rollups are already present in the cache table, +// so a read can populate only the missing prefix/suffix of its own range +// instead of re-scanning raw events for hours that are already rolled up. +const meterQueryRowCacheCoverageTable = "meterqueryrow_cache_coverage" + +// cacheCoverageMarkerSlug is the reserved meter_slug of per-namespace +// invalidation marker rows in the coverage table. It can never collide with a +// real claim: meter keys are validated non-empty. +const cacheCoverageMarkerSlug = "" + +// cacheCoverageClockSkewMargin pads the claim-vs-marker comparison. A claim's +// populated_at is stamped with the app server's clock while the marker's +// created_at is stamped by ClickHouse, so honoring a claim requires +// populated_at to beat the marker by more than any realistic clock skew. +// Over-distrusting costs one redundant (idempotent) populate; under-distrusting +// would serve a wiped range as empty. +const cacheCoverageClockSkewMargin = 5 * time.Second + +// cacheCoverageTrustWindow bounds how long a coverage claim is honored, +// measured from the FIRST populate that established the interval. The cache +// table's TTL expires rollup rows 90 days after their created_at, and gap +// tracking means covered rows are no longer rewritten (and thus refreshed) on +// every read — so the oldest rows under a claim expire at +// first_written_at + 90d while the claim would still assert coverage, silently +// undercounting the settled range. Distrusting the claim one day earlier +// forces a full re-populate (which rewrites the rows and resets the clock) +// before any row can expire. If this window did not exist, a meter queried for +// longer than 90 days over the same coverage interval would lose its oldest +// windows. +const cacheCoverageTrustWindow = 89 * 24 * time.Hour + +// createMeterQueryRowCacheCoverageTable builds the DDL for the coverage table. +// +// Storage decisions: +// - One row per (namespace, meter_slug, meter_hash), same identity as the +// rollup rows it describes: a meter shape change orphans both the rollups +// and their claim together. +// - ReplacingMergeTree(created_at) + read-time argMax: extensions simply +// insert a new row and the newest interval wins; racing readers can only +// UNDER-report coverage (each claims what it verified populated), which +// costs a redundant populate, never a wrong result. +// - first_written_at is carried forward on extension (it tracks the OLDEST +// rollup rows under the claim, which expire first — see +// cacheCoverageTrustWindow). +// - TTL matches the rollup table: an expired claim is just absent coverage +// and the next read re-populates, so expiry here is always safe. +type createMeterQueryRowCacheCoverageTable struct { + Database string + TableName string +} + +func (t createMeterQueryRowCacheCoverageTable) toSQL() string { + tableName := getTableName(t.Database, t.TableName) + + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + namespace String, + meter_slug String, + meter_hash String, + covered_from DateTime, + covered_until DateTime, + first_written_at DateTime, + populated_at DateTime64(3), + created_at DateTime64(3) DEFAULT now64(3) +) +ENGINE = ReplacingMergeTree(created_at) +ORDER BY (namespace, meter_slug, meter_hash) +TTL toDateTime(created_at) + toIntervalDay(90)`, tableName) +} + +// cacheCoverage is the newest stored coverage claim for one meter shape: +// hourly rollups for [From, Until) are present in the cache table, the oldest +// of them written at FirstWrittenAt. PopulatedAt is the claiming read's +// PLAN-START time (app-server clock, captured before it read the previous +// claim and ran its populates): a claim is honored only when PopulatedAt +// postdates the namespace's newest invalidation marker, which is what lets an +// invalidation win against a claim INSERT that lands after its deletes. +type cacheCoverage struct { + From time.Time + Until time.Time + FirstWrittenAt time.Time + PopulatedAt time.Time +} + +// timeRange is a half-open [From, To) hour-aligned populate target. +type timeRange struct { + From time.Time + To time.Time +} + +// cachePlan is the outcome of planning a cached read against the stored +// coverage: the sub-ranges of the query's settled window that must be +// populated, and the coverage claim to store once they succeed (nil = leave +// the stored claim untouched). +type cachePlan struct { + Populate []timeRange + Store *cacheCoverage +} + +// planCachePopulation decides which parts of the settled range [lo, hi) must +// be rolled up given the stored coverage claim (nil if none) and the +// namespace's newest invalidation marker (zero if none). `now` is the calling +// read's plan-start time and becomes the stored claim's PopulatedAt. Every +// branch preserves the invariant that a stored claim only ever describes rows +// whose populate has committed: +// +// - No claim, a claim older than the trust window, or a claim whose +// PopulatedAt does not clearly postdate the invalidation marker (it may +// describe rows the invalidation wiped): populate the whole range and +// start a fresh claim (re-populating over surviving rows is harmless — +// newest created_at wins the read-time collapse — and resets their TTL +// clock). +// - Claim disjoint from the queried range: populate the whole range but keep +// the stored claim, because a single contiguous interval cannot describe +// both. The read is still correct (its rows exist after this populate); +// it just isn't remembered, which is the v1 behavior for that range. +// - Overlapping or adjacent claim: populate only the missing prefix and/or +// suffix and extend the claim to the union. A fully covered range +// populates nothing and stores nothing (no write amplification on the hot +// path). +func planCachePopulation(lo, hi time.Time, cov *cacheCoverage, invalidatedAt time.Time, now time.Time) cachePlan { + if cov != nil && now.After(cov.FirstWrittenAt.Add(cacheCoverageTrustWindow)) { + cov = nil + } + if cov != nil && !invalidatedAt.IsZero() && !cov.PopulatedAt.After(invalidatedAt.Add(cacheCoverageClockSkewMargin)) { + cov = nil + } + + if cov == nil { + return cachePlan{ + Populate: []timeRange{{From: lo, To: hi}}, + Store: &cacheCoverage{From: lo, Until: hi, FirstWrittenAt: now, PopulatedAt: now}, + } + } + + // Disjoint (not even adjacent): [lo,hi) ends before the claim starts or + // starts after it ends. + if hi.Before(cov.From) || lo.After(cov.Until) { + return cachePlan{ + Populate: []timeRange{{From: lo, To: hi}}, + } + } + + plan := cachePlan{} + if lo.Before(cov.From) { + plan.Populate = append(plan.Populate, timeRange{From: lo, To: cov.From}) + } + if hi.After(cov.Until) { + plan.Populate = append(plan.Populate, timeRange{From: cov.Until, To: hi}) + } + + if len(plan.Populate) > 0 { + merged := &cacheCoverage{From: cov.From, Until: cov.Until, FirstWrittenAt: cov.FirstWrittenAt, PopulatedAt: now} + if lo.Before(merged.From) { + merged.From = lo + } + if hi.After(merged.Until) { + merged.Until = hi + } + plan.Store = merged + } + + return plan +} + +// getMeterQueryRowCacheCoverage builds the newest-wins lookup of one meter +// shape's coverage claim AND the namespace's invalidation marker in a single +// round trip (grouped by meter_slug; the marker lives under the reserved empty +// slug). Absence of a group means "none stored". +// +// The claim fields are picked through ONE argMax over a tuple, never +// independent argMax calls: with per-column argMax, a created_at tie between +// two racing claims (e.g. disjoint first-claims landing in the same +// millisecond) could stitch covered_from from one row and covered_until from +// the other, asserting coverage over the gap between them that no populate +// ever wrote — a silent undercount. The tuple makes the pick atomic: whichever +// row wins the tie, the claim describes rows that writer actually populated. +type getMeterQueryRowCacheCoverage struct { + Database string + TableName string + Namespace string + Meter string + Hash string +} + +func (g getMeterQueryRowCacheCoverage) toSQL() (string, []interface{}) { + tableName := getTableName(g.Database, g.TableName) + + sql := fmt.Sprintf(`SELECT + meter_slug, + tupleElement(argMax(tuple(covered_from, covered_until, first_written_at, populated_at), created_at), 1), + tupleElement(argMax(tuple(covered_from, covered_until, first_written_at, populated_at), created_at), 2), + tupleElement(argMax(tuple(covered_from, covered_until, first_written_at, populated_at), created_at), 3), + tupleElement(argMax(tuple(covered_from, covered_until, first_written_at, populated_at), created_at), 4), + max(created_at) +FROM %s +WHERE namespace = ? AND ((meter_slug = ? AND meter_hash = ?) OR (meter_slug = '' AND meter_hash = '')) +GROUP BY meter_slug`, tableName) + + return sql, []interface{}{g.Namespace, g.Meter, g.Hash} +} + +// insertMeterQueryRowCacheCoverage builds the claim upsert (a plain insert; +// ReplacingMergeTree + argMax reads make the newest row the effective claim). +type insertMeterQueryRowCacheCoverage struct { + Database string + TableName string + Namespace string + Meter string + Hash string + Coverage cacheCoverage +} + +func (i insertMeterQueryRowCacheCoverage) toSQL() (string, []interface{}) { + tableName := getTableName(i.Database, i.TableName) + + sql := fmt.Sprintf(`INSERT INTO %s (namespace, meter_slug, meter_hash, covered_from, covered_until, first_written_at, populated_at) +VALUES (?, ?, ?, ?, ?, ?, ?)`, tableName) + + return sql, []interface{}{ + i.Namespace, i.Meter, i.Hash, + i.Coverage.From.Unix(), i.Coverage.Until.Unix(), i.Coverage.FirstWrittenAt.Unix(), + i.Coverage.PopulatedAt.UTC(), + } +} + +// insertMeterQueryRowCacheInvalidationMarkers builds the per-namespace +// invalidation marker inserts. The marker (not the claim DELETE) is what makes +// invalidation win against in-flight claim writers: a racing claim INSERT can +// always land after any DELETE, but its populated_at was captured at plan +// start and therefore predates the marker's created_at, so every read +// distrusts it. Markers live under the reserved empty slug and must survive +// claim deletion; the table TTL garbage-collects them long after any claim +// that could predate them has aged out of the trust window. +type insertMeterQueryRowCacheInvalidationMarkers struct { + Database string + TableName string + Namespaces []string +} + +func (i insertMeterQueryRowCacheInvalidationMarkers) toSQL() (string, []interface{}) { + tableName := getTableName(i.Database, i.TableName) + + var values []string + args := make([]interface{}, 0, len(i.Namespaces)) + for _, ns := range i.Namespaces { + values = append(values, "(?, '', '', 0, 0, 0, 0)") + args = append(args, ns) + } + + sql := fmt.Sprintf(`INSERT INTO %s (namespace, meter_slug, meter_hash, covered_from, covered_until, first_written_at, populated_at) +VALUES %s`, tableName, strings.Join(values, ", ")) + + return sql, args +} + +// deleteMeterQueryRowCacheCoverageClaims builds the claim cleanup for the +// given namespaces. Markers (the reserved empty slug) are deliberately +// excluded: they must outlive the deletion to kill racing claim writers. +type deleteMeterQueryRowCacheCoverageClaims struct { + Database string + TableName string + Namespaces []string +} + +func (d deleteMeterQueryRowCacheCoverageClaims) toSQL() (string, []interface{}) { + tableName := getTableName(d.Database, d.TableName) + + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(d.Namespaces)), ", ") + + sql := fmt.Sprintf("DELETE FROM %s WHERE namespace IN (%s) AND meter_slug != ''", tableName, placeholders) + + args := make([]interface{}, 0, len(d.Namespaces)) + for _, ns := range d.Namespaces { + args = append(args, ns) + } + + return sql, args +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_coverage_test.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_coverage_test.go new file mode 100644 index 0000000000..7853a11b57 --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_coverage_test.go @@ -0,0 +1,131 @@ +package clickhouse + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestPlanCachePopulation(t *testing.T) { + base := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + now := base.Add(30 * 24 * time.Hour) + hour := func(h int) time.Time { return base.Add(time.Duration(h) * time.Hour) } + + freshClaim := func(from, until time.Time) *cacheCoverage { + return &cacheCoverage{From: from, Until: until, FirstWrittenAt: now.Add(-time.Hour), PopulatedAt: now.Add(-time.Hour)} + } + + tests := []struct { + name string + lo, hi time.Time + cov *cacheCoverage + invalidatedAt time.Time + wantPopulate []timeRange + wantStore *cacheCoverage + }{ + { + name: "no claim populates everything and starts a claim", + lo: hour(0), hi: hour(10), + wantPopulate: []timeRange{{From: hour(0), To: hour(10)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now, PopulatedAt: now}, + }, + { + name: "fully covered populates and stores nothing", + lo: hour(2), hi: hour(8), + cov: freshClaim(hour(0), hour(10)), + }, + { + name: "exactly covered populates and stores nothing", + lo: hour(0), hi: hour(10), + cov: freshClaim(hour(0), hour(10)), + }, + { + name: "missing prefix populates only the prefix and extends the claim", + lo: hour(0), hi: hour(8), + cov: freshClaim(hour(4), hour(10)), + wantPopulate: []timeRange{{From: hour(0), To: hour(4)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now.Add(-time.Hour), PopulatedAt: now}, + }, + { + name: "missing suffix populates only the suffix and extends the claim", + lo: hour(2), hi: hour(15), + cov: freshClaim(hour(0), hour(10)), + wantPopulate: []timeRange{{From: hour(10), To: hour(15)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(15), FirstWrittenAt: now.Add(-time.Hour), PopulatedAt: now}, + }, + { + name: "missing both sides populates prefix and suffix and extends both ways", + lo: hour(0), hi: hour(15), + cov: freshClaim(hour(4), hour(10)), + wantPopulate: []timeRange{{From: hour(0), To: hour(4)}, {From: hour(10), To: hour(15)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(15), FirstWrittenAt: now.Add(-time.Hour), PopulatedAt: now}, + }, + { + name: "adjacent below is an extension, not disjoint", + lo: hour(0), hi: hour(4), + cov: freshClaim(hour(4), hour(10)), + wantPopulate: []timeRange{{From: hour(0), To: hour(4)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now.Add(-time.Hour), PopulatedAt: now}, + }, + { + name: "disjoint below populates everything but keeps the stored claim", + lo: hour(0), hi: hour(3), + cov: freshClaim(hour(4), hour(10)), + wantPopulate: []timeRange{{From: hour(0), To: hour(3)}}, + }, + { + name: "disjoint above populates everything but keeps the stored claim", + lo: hour(12), hi: hour(15), + cov: freshClaim(hour(4), hour(10)), + wantPopulate: []timeRange{{From: hour(12), To: hour(15)}}, + }, + { + name: "expired claim is ignored: full populate and a fresh claim", + lo: hour(0), hi: hour(10), + cov: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now.Add(-cacheCoverageTrustWindow - time.Minute), PopulatedAt: now.Add(-time.Hour)}, + wantPopulate: []timeRange{{From: hour(0), To: hour(10)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now, PopulatedAt: now}, + }, + { + name: "claim at exactly the trust window boundary is still honored", + lo: hour(0), hi: hour(10), + cov: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now.Add(-cacheCoverageTrustWindow), PopulatedAt: now.Add(-time.Hour)}, + }, + { + // The blocker class: a claim planned BEFORE an invalidation marker + // may describe rows the invalidation wiped — it must be ignored even + // though its row landed (created_at) after the marker. + name: "claim planned before the invalidation marker is distrusted", + lo: hour(0), hi: hour(10), + cov: freshClaim(hour(0), hour(10)), // PopulatedAt = now-1h + invalidatedAt: now.Add(-30 * time.Minute), // marker after the plan + wantPopulate: []timeRange{{From: hour(0), To: hour(10)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now, PopulatedAt: now}, + }, + { + name: "claim planned safely after the marker is honored", + lo: hour(0), hi: hour(10), + cov: freshClaim(hour(0), hour(10)), // PopulatedAt = now-1h + invalidatedAt: now.Add(-time.Hour - cacheCoverageClockSkewMargin - 2*time.Second), // marker clearly before the plan + }, + { + // Within the skew margin the ordering is ambiguous (app clock vs + // ClickHouse clock) — distrust, at the cost of one redundant populate. + name: "claim inside the clock-skew margin of the marker is distrusted", + lo: hour(0), hi: hour(10), + cov: freshClaim(hour(0), hour(10)), // PopulatedAt = now-1h + invalidatedAt: now.Add(-time.Hour - cacheCoverageClockSkewMargin/2), // marker just before the plan + wantPopulate: []timeRange{{From: hour(0), To: hour(10)}}, + wantStore: &cacheCoverage{From: hour(0), Until: hour(10), FirstWrittenAt: now, PopulatedAt: now}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plan := planCachePopulation(tt.lo, tt.hi, tt.cov, tt.invalidatedAt, now) + assert.Equal(t, tt.wantPopulate, plan.Populate) + assert.Equal(t, tt.wantStore, plan.Store) + }) + } +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go new file mode 100644 index 0000000000..c3e1c6624b --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go @@ -0,0 +1,365 @@ +package clickhouse + +import ( + "context" + "fmt" + "strings" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" + "github.com/openmeterio/openmeter/openmeter/streaming" +) + +// queryMeterCached serves a cacheable meter query. It lazily rolls up the +// settled whole-hour range into the cache table, then runs the merge query that +// UNIONs the settled rollup with live scans of the sub-hour head and fresh tail. +// +// Population is lazy, idempotent, and gap-tracked: the coverage table records +// the contiguous settled interval already rolled up per meter shape, so a read +// populates only the missing prefix/suffix of its own range (nothing at all on +// a fully covered repeat read). Idempotence is what makes the tracking safe +// rather than merely fast: because the rollup table is a ReplacingMergeTree +// and the cache leg is collapsed newest-wins at read time, a redundant +// populate — from a racing reader, a lost coverage claim, or an expired trust +// window — is always harmless. +func (c *Connector) queryMeterCached(ctx context.Context, query queryMeter) ([]meterpkg.MeterQueryRow, error) { + cutoff := c.prepareCacheableCutoff(query) + + from := query.from() + if from == nil { + return nil, fmt.Errorf("cached query requires a from time") + } + + merge := queryCachedMeter{ + Database: c.config.Database, + CacheTableName: meterQueryRowCacheTable, + EventsTableName: c.config.EventsTableName, + query: query, + Cutoff: cutoff, + } + + // The cache stores ONLY complete hour windows [cacheLo, cacheHi): the sub-hour + // head [from, cacheLo) and the partial last hour [cacheHi, to) are served live. + // Populate exactly the settled whole-hour range the cache leg reads. + cacheLo := merge.headCeil() + cacheHi := merge.cacheHi() + + ctx, span := c.tracer.Start(ctx, "streaming.query_cache.query", trace.WithAttributes( + attribute.String("namespace", query.Namespace), + attribute.String("meter_slug", query.Meter.Key), + attribute.String("cache_lo", cacheLo.UTC().Format(time.RFC3339)), + attribute.String("cache_hi", cacheHi.UTC().Format(time.RFC3339)), + )) + defer span.End() + + logger := c.config.Logger.With( + "namespace", query.Namespace, + "type", query.Meter.EventType, + "from", from, + "cacheLo", cacheLo, + "cacheHi", cacheHi, + "to", query.To, + "cutoff", cutoff, + ) + + if cacheLo.Before(cacheHi) { + plan := c.planCachedRangePopulation(ctx, query, cacheLo, cacheHi) + span.SetAttributes(attribute.Int("populate_ranges", len(plan.Populate))) + + for _, r := range plan.Populate { + if err := c.populateMeterQueryRowCache(ctx, query, r.From, r.To); err != nil { + logger.Error("failed to populate meter query cache, falling back to live query", "error", err) + c.queryCacheMetrics.recordPopulateError(ctx) + c.queryCacheMetrics.recordQuery(ctx, "live_fallback") + span.RecordError(err) + span.SetAttributes(attribute.Bool("live_fallback", true)) + + return c.queryMeter(ctx, query) + } + } + + // The claim is stored strictly AFTER every populate above committed, so + // a claim can never describe rows that were not written. A failed claim + // write is not a query failure: the rollups exist and serve this read; + // the next read just re-populates redundantly (idempotent). + if plan.Store != nil { + if err := c.storeMeterQueryRowCacheCoverage(ctx, query, *plan.Store); err != nil { + logger.Warn("failed to store meter query cache coverage", "error", err) + span.RecordError(err) + } + } + } + + sql, args, err := merge.toSQL() + if err != nil { + return nil, fmt.Errorf("build cached query sql: %w", err) + } + + start := time.Now() + + rows, err := c.config.ClickHouse.Query(ctx, sql, args...) + if err != nil { + span.SetStatus(codes.Error, "cached query failed") + span.RecordError(err) + + return nil, fmt.Errorf("clickhouse cached query: %w", err) + } + defer rows.Close() + + elapsed := time.Since(start) + c.queryCacheMetrics.recordQueryDuration(ctx, elapsed) + logger.Debug("clickhouse cached query executed", "elapsed", elapsed.String(), "sql", sql, "args", args) + + values, err := merge.scanRows(rows) + if err != nil { + span.SetStatus(codes.Error, "cached query scan failed") + span.RecordError(err) + + return nil, fmt.Errorf("scan cached query rows: %w", err) + } + + c.queryCacheMetrics.recordQuery(ctx, "cached") + span.SetAttributes(attribute.Int("rows", len(values))) + + c.maybeShadowVerifyCachedResult(ctx, query, values) + + return values, nil +} + +// planCachedRangePopulation reads the stored coverage claim and the +// namespace's invalidation marker for the query's meter shape and plans which +// sub-ranges of [cacheLo, cacheHi) still need rolling up. The plan-start time +// is captured BEFORE the coverage read: it becomes the stored claim's +// PopulatedAt, so an invalidation marker landing at any later instant — +// including between our populates and our claim INSERT — makes the claim +// distrusted. A coverage read failure degrades to "no claim" (populate the +// whole range), which is always safe — never fail or under-populate the query +// because the metadata was unreadable. +func (c *Connector) planCachedRangePopulation(ctx context.Context, query queryMeter, cacheLo, cacheHi time.Time) cachePlan { + planStart := time.Now().UTC() + + coverage, invalidatedAt, err := c.readMeterQueryRowCacheCoverage(ctx, query) + if err != nil { + c.config.Logger.Warn("failed to read meter query cache coverage, repopulating the full range", "error", err, "namespace", query.Namespace, "meter", query.Meter.Key) + coverage = nil + } + + return planCachePopulation(cacheLo, cacheHi, coverage, invalidatedAt, planStart) +} + +// readMeterQueryRowCacheCoverage returns the meter shape's newest claim (nil +// if none) and the namespace's newest invalidation marker time (zero if none). +func (c *Connector) readMeterQueryRowCacheCoverage(ctx context.Context, query queryMeter) (*cacheCoverage, time.Time, error) { + get := getMeterQueryRowCacheCoverage{ + Database: c.config.Database, + TableName: meterQueryRowCacheCoverageTable, + Namespace: query.Namespace, + Meter: query.Meter.Key, + Hash: meterShapeHash(query.Meter), + } + + sql, args := get.toSQL() + + rows, err := c.config.ClickHouse.Query(ctx, sql, args...) + if err != nil { + return nil, time.Time{}, fmt.Errorf("read cache coverage: %w", err) + } + defer rows.Close() + + var coverage *cacheCoverage + var invalidatedAt time.Time + + for rows.Next() { + var slug string + var from, until, firstWrittenAt, populatedAt, newestCreatedAt time.Time + if err := rows.Scan(&slug, &from, &until, &firstWrittenAt, &populatedAt, &newestCreatedAt); err != nil { + return nil, time.Time{}, fmt.Errorf("scan cache coverage: %w", err) + } + + if slug == cacheCoverageMarkerSlug { + invalidatedAt = newestCreatedAt.UTC() + continue + } + + coverage = &cacheCoverage{ + From: from.UTC(), + Until: until.UTC(), + FirstWrittenAt: firstWrittenAt.UTC(), + PopulatedAt: populatedAt.UTC(), + } + } + + if err := rows.Err(); err != nil { + return nil, time.Time{}, fmt.Errorf("read cache coverage rows: %w", err) + } + + return coverage, invalidatedAt, nil +} + +func (c *Connector) storeMeterQueryRowCacheCoverage(ctx context.Context, query queryMeter, coverage cacheCoverage) error { + insert := insertMeterQueryRowCacheCoverage{ + Database: c.config.Database, + TableName: meterQueryRowCacheCoverageTable, + Namespace: query.Namespace, + Meter: query.Meter.Key, + Hash: meterShapeHash(query.Meter), + Coverage: coverage, + } + + sql, args := insert.toSQL() + if err := c.config.ClickHouse.Exec(ctx, sql, args...); err != nil { + return fmt.Errorf("store cache coverage: %w", err) + } + + return nil +} + +func (c *Connector) populateMeterQueryRowCache(ctx context.Context, query queryMeter, from, cutoff time.Time) error { + ctx, span := c.tracer.Start(ctx, "streaming.query_cache.populate", trace.WithAttributes( + attribute.String("namespace", query.Namespace), + attribute.String("meter_slug", query.Meter.Key), + attribute.String("from", from.UTC().Format(time.RFC3339)), + attribute.String("to", cutoff.UTC().Format(time.RFC3339)), + )) + defer span.End() + + populate := populateMeterQueryRowCache{ + Database: c.config.Database, + CacheTableName: meterQueryRowCacheTable, + EventsTableName: c.config.EventsTableName, + query: query, + From: from, + Cutoff: cutoff, + } + + start := time.Now() + + sql, args := populate.toSQL() + if err := c.config.ClickHouse.Exec(ctx, sql, args...); err != nil { + span.SetStatus(codes.Error, "populate insert failed") + span.RecordError(err) + + return fmt.Errorf("populate cache insert: %w", err) + } + + c.queryCacheMetrics.recordPopulateDuration(ctx, time.Since(start)) + + return nil +} + +// findNamespacesToInvalidateCache returns the distinct namespaces of any events +// whose time is older than the freshness horizon. Such an event mutates an +// already-settled window, so that namespace's cached rollup is now stale. +func (c *Connector) findNamespacesToInvalidateCache(rawEvents []streaming.RawEvent) []string { + if c.config.QueryCacheMinimumCacheableUsageAge <= 0 { + return nil + } + + horizon := time.Now().UTC().Add(-c.config.QueryCacheMinimumCacheableUsageAge) + + seen := map[string]struct{}{} + namespaces := []string{} + + for _, event := range rawEvents { + if event.Time.Before(horizon) { + if _, ok := seen[event.Namespace]; !ok { + seen[event.Namespace] = struct{}{} + namespaces = append(namespaces, event.Namespace) + } + } + } + + return namespaces +} + +// invalidateMeterQueryRowCache deletes all cached rows for the given namespaces. +func (c *Connector) invalidateMeterQueryRowCache(ctx context.Context, namespaces []string) error { + if len(namespaces) == 0 { + return nil + } + + ctx, span := c.tracer.Start(ctx, "streaming.query_cache.invalidate", trace.WithAttributes( + attribute.Int("namespaces", len(namespaces)), + )) + defer span.End() + + // Step 1: insert the per-namespace invalidation markers FIRST. The marker, + // not the claim DELETE, is what beats in-flight claim writers: a racing + // claim INSERT can land after any DELETE, but its populated_at (captured at + // plan start) predates the marker, so every read distrusts it. Once the + // marker commits, all past and racing claims are dead even if the cleanup + // deletes below fail — the next read re-populates, and its rows win the + // newest-wins collapse over any stale survivors (this restores the + // self-healing v1 had from populating on every read). + markers := insertMeterQueryRowCacheInvalidationMarkers{ + Database: c.config.Database, + TableName: meterQueryRowCacheCoverageTable, + Namespaces: namespaces, + } + + markerSQL, markerArgs := markers.toSQL() + if err := c.config.ClickHouse.Exec(ctx, markerSQL, markerArgs...); err != nil { + // A missing table (code 60) means nothing is cached: invalidation is + // always-on even with the cache disabled. + if !strings.Contains(err.Error(), "code: 60") { + // Fall back to deleting the claims outright: that still kills every + // COMMITTED claim (only a claim racing this exact invalidation could + // survive, which is the pre-marker exposure). If the delete fails + // too, report the error — stale claims then persist until the next + // invalidation or trust-window expiry. + delClaims := deleteMeterQueryRowCacheCoverageClaims{ + Database: c.config.Database, + TableName: meterQueryRowCacheCoverageTable, + Namespaces: namespaces, + } + + delSQL, delArgs := delClaims.toSQL() + if delErr := c.config.ClickHouse.Exec(ctx, delSQL, delArgs...); delErr != nil { + span.SetStatus(codes.Error, "invalidation marker and claim delete failed") + span.RecordError(err) + + return fmt.Errorf("insert invalidation markers: %w (claim delete fallback: %v)", err, delErr) + } + + c.config.Logger.Warn("invalidation marker insert failed, fell back to deleting coverage claims", "error", err, "namespaces", namespaces) + } + } + + // Step 2 (cleanup, correctness no longer depends on it): drop the dead + // claims and the rollup rows. Failures degrade to storage waste plus one + // redundant repopulate, so they are logged but not propagated. + claimsSQL, claimsArgs := deleteMeterQueryRowCacheCoverageClaims{ + Database: c.config.Database, + TableName: meterQueryRowCacheCoverageTable, + Namespaces: namespaces, + }.toSQL() + + rollupSQL, rollupArgs := deleteMeterQueryRowCacheForNamespaces{ + Database: c.config.Database, + TableName: meterQueryRowCacheTable, + Namespaces: namespaces, + }.toSQL() + + for _, cleanup := range []struct { + sql string + args []interface{} + }{ + {claimsSQL, claimsArgs}, + {rollupSQL, rollupArgs}, + } { + if err := c.config.ClickHouse.Exec(ctx, cleanup.sql, cleanup.args...); err != nil { + if strings.Contains(err.Error(), "code: 60") { + continue + } + + span.RecordError(err) + c.config.Logger.Warn("invalidation cleanup delete failed (markers already invalidate the claims)", "error", err, "namespaces", namespaces) + } + } + + return nil +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go new file mode 100644 index 0000000000..f2dbae61a8 --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go @@ -0,0 +1,235 @@ +package clickhouse + +import ( + "slices" + "time" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" + "github.com/openmeterio/openmeter/openmeter/streaming" +) + +// cacheGrain is the fixed hourly grain the rollup table is stored at. The cache +// can only serve window sizes that re-aggregate cleanly from hourly rows. +const cacheGrain = time.Hour + +// queryCacheFeatureFlag is the feature-gate flag that permits the meter +// query-result cache for a namespace. It follows the `om_ff_` convention. By +// default (the Noop gate) it is enabled for every namespace; a configured +// feature-gate backend evaluates this flag and can restrict it per namespace. +const queryCacheFeatureFlag = "om_ff_query_cache_enabled" + +// canQueryBeCached reports whether a meter query is provably reproducible from +// the hourly-UTC rollup, i.e. the cached result is byte-identical to the live +// query. Anything not admitted falls through to the live path unchanged — that +// is what makes the cache safely optional. The guards, all of which must hold: +// +// - The cache is enabled (the master switch, checked first). +// - Decimal precision is enabled: the rollup stores exact Decimal128(19) +// values, and only the decimal live path is provably byte-identical to a +// recombination of them (Float64 sums at billing magnitudes can diverge at +// the sixth decimal; the UNION would also hit a Decimal/Float64 supertype +// error). +// - The query opted in (params.Cachable) — the HTTP meter-query handlers set +// this; internal callers do not, so they never hit the cache. +// - No ClientID: progress tracking only exists on the live path — the cached +// path creates no progress record, so a client polling the progress endpoint +// would spin on a permanently missing entry. +// - Both bounds are set. Without From the settled range is unbounded; without +// To the live path applies no upper time bound (future-dated events count) +// while the merge would cap at now() — and for the total (nil) window the +// live path derives the single window end from tumbleEnd(max(event time)), +// which the cache cannot reproduce. +// - The effective span is at least MinimumCacheableQueryPeriod. +// - The aggregation is one of the four mergeable ones (SUM/COUNT/MIN/MAX). +// AVG and UNIQUE_COUNT are a permanent design boundary: per-window averages +// don't compose into a global average and exact distinct-counts can't be +// unioned from partial scalars. +// - The window size is hour/day/month/total (nil). Minute windows can't be +// reconstructed from an hourly rollup. +// - The window timezone offset from UTC is a whole number of hours across the +// queried range (fractional-hour zones like +05:30 can't be composed from +// hourly-UTC windows; DST transitions are handled conservatively). +// - No FilterStoredAt: the rollup has no stored_at dimension, so a stored-at +// filter can't be reproduced. +// - No customer_id involvement: customer_id is derived from subject via a +// dictionary and is not stored in the rollup, so any customer_id group-by or +// customer filter routes to live in v1. +// - Every FilterGroupBy key is a meter dimension, mirroring the live path's +// validation (unknown keys — including "subject" — are a 400 on the live +// path; routing them live preserves that behavior instead of silently +// accepting them or turning the 400 into a 500). customer_id stays excluded +// even if a meter defines it as a dimension key, because it aliases the +// derived top-level column on the live path. +// - The feature gate permits the namespace (checked LAST — EvaluateBool may be +// a remote lookup, LRU-cached per (flag, namespace), so only otherwise- +// cacheable queries pay for it). By default the gate is enabled for every +// namespace; a configured feature-gate backend can restrict it. An +// evaluation error routes to live — never cache on a gate failure. +func (c *Connector) canQueryBeCached(namespace string, meter meterpkg.Meter, params streaming.QueryParams) bool { + if !c.config.QueryCacheEnabled { + return false + } + + if !c.config.EnableDecimalPrecision { + return false + } + + if !params.Cachable { + return false + } + + if params.ClientID != nil { + return false + } + + if params.From == nil { + return false + } + + if params.To == nil { + return false + } + + // Effective from merges the meter's EventFrom (matches queryMeter.from()). + from := *params.From + if meter.EventFrom != nil && meter.EventFrom.After(from) { + from = *meter.EventFrom + } + + to := *params.To + + // The queried range must be old enough to have settled windows to serve. + // If from is younger than the freshness horizon there is nothing to cache. + horizonCutoff := time.Now().UTC().Add(-c.config.QueryCacheMinimumCacheableUsageAge) + if !from.Before(horizonCutoff) { + return false + } + + if to.Sub(from) < c.config.QueryCacheMinimumCacheableQueryPeriod { + return false + } + + switch meter.Aggregation { + case meterpkg.MeterAggregationSum, + meterpkg.MeterAggregationCount, + meterpkg.MeterAggregationMin, + meterpkg.MeterAggregationMax: + // mergeable + default: + return false + } + + if !isCacheableWindowSize(params.WindowSize) { + return false + } + + if !isWholeHourTimeZone(params.WindowTimeZone, from, to) { + return false + } + + if params.FilterStoredAt != nil && !params.FilterStoredAt.IsEmpty() { + return false + } + + if len(params.FilterCustomer) > 0 { + return false + } + if slices.Contains(params.GroupBy, "customer_id") { + return false + } + + for key := range params.FilterGroupBy { + if _, ok := meter.GroupBy[key]; !ok { + return false + } + if key == "customer_id" { + return false + } + } + + enabled, err := c.config.FeatureGate.Enabled(namespace, queryCacheFeatureFlag) + if err != nil { + c.config.Logger.Warn("query cache feature gate evaluation failed, serving live", "error", err, "namespace", namespace) + return false + } + + return enabled +} + +// isCacheableWindowSize reports whether the requested window size re-aggregates +// from the hourly rollup. Minute is the only excluded size (finer than the +// grain); nil (total) and hour/day/month all compose from hourly rows. +func isCacheableWindowSize(ws *meterpkg.WindowSize) bool { + if ws == nil { + return true + } + switch *ws { + case meterpkg.WindowSizeHour, meterpkg.WindowSizeDay, meterpkg.WindowSizeMonth: + return true + default: + return false + } +} + +// isWholeHourTimeZone reports whether the zone's offset from UTC is a whole +// number of hours for the entire [from, to) range. Hourly-UTC windows can only +// be re-tumbled into day/month windows in a target zone when the zone boundary +// falls on a UTC hour boundary; fractional-hour zones (India +05:30, Nepal +// +05:45) break this. +// +// DST is handled conservatively: we sample the offset at from, at to, and at +// every day boundary in between, and require every sampled offset to be a whole +// number of hours. A zone that is whole-hour year-round (e.g. Europe/* is always +// a whole hour even across DST) passes; a zone that is ever fractional in the +// range fails. UTC and a nil zone always pass. +func isWholeHourTimeZone(loc *time.Location, from, to time.Time) bool { + if loc == nil || loc == time.UTC || loc.String() == "UTC" { + return true + } + + isWholeHour := func(t time.Time) bool { + _, offsetSeconds := t.In(loc).Zone() + return offsetSeconds%3600 == 0 + } + + if !isWholeHour(from) || !isWholeHour(to) { + return false + } + + // Sample each day boundary to catch a DST transition into a fractional + // offset within the range. Bounded by the query span; cacheable queries are + // finite. Step by 24h from from up to to. + for t := from; t.Before(to); t = t.Add(24 * time.Hour) { + if !isWholeHour(t) { + return false + } + } + + return true +} + +// prepareCacheableCutoff returns the freshness boundary that splits a cacheable +// query into settled history [from, cutoff) served from the cache and a fresh +// tail [cutoff, to) served live. The cutoff is now - MinimumCacheableUsageAge, +// floored to the hour grid so it aligns with stored hourly windows and the tail +// leg's tumble boundaries. +// +// Clamped to [from, to]: if the whole range is older than the horizon the tail +// is empty (all-cached); if the whole range is fresher than the horizon the +// cache leg is empty (all-live) — though canQueryBeCached already rejects a +// wholly-fresh range, so in practice cutoff > from here. +func (c *Connector) prepareCacheableCutoff(query queryMeter) time.Time { + now := time.Now().UTC() + cutoff := now.Add(-c.config.QueryCacheMinimumCacheableUsageAge).Truncate(cacheGrain) + + from := query.from() + if from != nil && cutoff.Before(*from) { + cutoff = *from + } + + if query.To != nil && cutoff.After(*query.To) { + cutoff = *query.To + } + + return cutoff +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go new file mode 100644 index 0000000000..a9f451742c --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go @@ -0,0 +1,1606 @@ +package clickhouse + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/oklog/ulid/v2" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + tracenoop "go.opentelemetry.io/otel/trace/noop" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" + progressmanager "github.com/openmeterio/openmeter/openmeter/progressmanager/adapter" + "github.com/openmeterio/openmeter/openmeter/streaming" + "github.com/openmeterio/openmeter/pkg/featuregate" + "github.com/openmeterio/openmeter/pkg/filter" + "github.com/openmeterio/openmeter/pkg/models" +) + +// enableCacheOnConnector turns on the query cache on the suite connector (small +// horizon so the parity extremes are reachable) and ensures the rollup table +// exists. +func (s *ConnectorTestSuite) enableCacheOnConnector() { + // Mutate the existing connector's config in place: the events table and DB + // are already created; we only need the cache table + flags. + s.Connector.config.QueryCacheEnabled = true + s.Connector.config.QueryCacheMinimumCacheableQueryPeriod = time.Hour + s.Connector.config.QueryCacheMinimumCacheableUsageAge = time.Hour + // The cache only serves decimal-precision queries (see canQueryBeCached): + // the rollup stores Decimal128 and recombines exactly. Configure the test to + // that mode, which is the only one the cache supports. + s.Connector.config.EnableDecimalPrecision = true + s.NoError(s.Connector.createMeterQueryRowCacheTable(s.T().Context())) +} + +// TestQueryCacheParity is the billing-safety gate: for every admitted +// (meter, window size, filter subset, aggregation), and at the two extremes +// (all-fresh cutoff==from, all-cached cutoff==to) plus interior cutoffs, the +// cached result equals the live result. +func (s *ConnectorTestSuite) TestQueryCacheParity() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + // Seed a deterministic multi-day window with grouped, multi-subject data, + // including an all-null value-property event (min/max Nullable coverage). + base := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + eventType := "parity_event" + + // Events spread across ~3 days and 2 subjects, 2 group-by dimensions. + events := []parityEvent{ + {subject: "s1", at: base.Add(30 * time.Minute), data: `{"value": 10, "region": "us", "tier": "free"}`}, + {subject: "s1", at: base.Add(90 * time.Minute), data: `{"value": 5, "region": "us", "tier": "free"}`}, + {subject: "s2", at: base.Add(2 * time.Hour), data: `{"value": 20, "region": "eu", "tier": "pro"}`}, + {subject: "s1", at: base.Add(26 * time.Hour), data: `{"value": 7, "region": "us", "tier": "pro"}`}, + {subject: "s2", at: base.Add(27 * time.Hour), data: `{"value": 3, "region": "eu", "tier": "free"}`}, + {subject: "s2", at: base.Add(50 * time.Hour), data: `{"value": 100, "region": "eu", "tier": "pro"}`}, + {subject: "s1", at: base.Add(51 * time.Hour), data: `{"value": 1, "region": "us", "tier": "free"}`}, + // All-null value property in its own settled hour window: exercises the + // Nullable min/max path (live returns no row; SUM/COUNT still do). + {subject: "s1", at: base.Add(70 * time.Hour), data: `{"region": "us", "tier": "free"}`}, + } + s.seedEvents(ctx, eventType, events) + + groupByMap := map[string]string{"region": "$.region", "tier": "$.tier"} + + // Bound sets: hour-aligned, and OFF the hour grid on BOTH ends. The off-grid + // set exercises the partial head [from, ceil(from)) and partial tail + // [floor(cutoff), to) live legs simultaneously across every agg/window/cutoff, + // closing the boundary class the aligned matrix structurally cannot reach. + type bounds struct { + name string + from time.Time + to time.Time + } + boundSets := []bounds{ + {name: "aligned", from: base, to: base.Add(72 * time.Hour)}, + {name: "offgrid", from: base.Add(15 * time.Minute), to: base.Add(72*time.Hour - 20*time.Minute)}, + } + + meters := []parityMeter{ + {name: "parity_sum", eventType: eventType, valueProperty: ptr("$.value"), aggregation: meterpkg.MeterAggregationSum, groupBy: groupByMap}, + {name: "parity_count", eventType: eventType, aggregation: meterpkg.MeterAggregationCount, groupBy: groupByMap}, + {name: "parity_min", eventType: eventType, valueProperty: ptr("$.value"), aggregation: meterpkg.MeterAggregationMin, groupBy: groupByMap}, + {name: "parity_max", eventType: eventType, valueProperty: ptr("$.value"), aggregation: meterpkg.MeterAggregationMax, groupBy: groupByMap}, + } + + // Window sizes handled by the direct-merge comparison. nil (total) is included: + // with To set, QueryMeter overwrites windowstart/end with From/To on both the + // live and cached path, so they align. (Total with To==nil is routed to live by + // the gate and thus never reaches the cache.) + windowSizes := []*meterpkg.WindowSize{ + nil, + ptr(meterpkg.WindowSizeHour), + ptr(meterpkg.WindowSizeDay), + ptr(meterpkg.WindowSizeMonth), + } + + // Filter/group-by subsets: no grouping, group by one/both dims, subject + // group-by, subject filter, group-by value filter. + type filterCase struct { + name string + groupBy []string + filterSubject []string + filterGroupBy map[string]filter.FilterString + } + filterCases := []filterCase{ + {name: "total-nofilter"}, + {name: "group-region", groupBy: []string{"region"}}, + {name: "group-region-tier", groupBy: []string{"region", "tier"}}, + {name: "group-subject", groupBy: []string{"subject"}}, + {name: "group-subject-region", groupBy: []string{"subject", "region"}}, + {name: "filter-subject-s1", filterSubject: []string{"s1"}}, + {name: "filter-region-us", groupBy: []string{"region"}, filterGroupBy: map[string]filter.FilterString{"region": filterEq("us")}}, + // Non-Eq operators exercise filterStringWhere's fragment splicing (the + // cached path renders these on group_by[i] while live renders them on the + // raw JSON extraction — they must select identical row sets). + {name: "filter-region-in", groupBy: []string{"region"}, filterGroupBy: map[string]filter.FilterString{"region": {In: &[]string{"us", "eu"}}}}, + {name: "filter-tier-ne", groupBy: []string{"region", "tier"}, filterGroupBy: map[string]filter.FilterString{"tier": {Ne: ptr("pro")}}}, + {name: "filter-region-like", groupBy: []string{"region"}, filterGroupBy: map[string]filter.FilterString{"region": {Like: ptr("u%")}}}, + } + + for _, bs := range boundSets { + from := bs.from + to := bs.to + + // Cutoffs: extremes (from = all-fresh, to = all-cached) plus interior hour + // boundaries. For the off-grid set, `from` and `to` are mid-hour, so the + // all-fresh (cutoff=from) and all-cached (cutoff=to) extremes also exercise + // the partial head and partial tail live legs. + cutoffs := []time.Time{ + from, // all-fresh: cache empty (or head-only), tail is the whole range + from.Add(24 * time.Hour), + from.Add(48 * time.Hour), + to, // all-cached: fresh tail empty (partial last hour still live) + } + + for _, m := range meters { + meter := s.newMeter(m) + for _, ws := range windowSizes { + for _, fc := range filterCases { + params := streaming.QueryParams{ + From: &from, + To: &to, + WindowSize: ws, + GroupBy: fc.groupBy, + FilterSubject: fc.filterSubject, + FilterGroupBy: fc.filterGroupBy, + } + live := s.liveRows(ctx, meter, params) + + for _, cutoff := range cutoffs { + name := fmt.Sprintf("%s/%s/%s/%s/cutoff=%s", bs.name, m.name, windowSizeName(ws), fc.name, cutoff.Sub(from)) + s.Run(name, func() { + // Fresh cache table state per case to isolate populate. + s.truncateCacheTable(ctx) + cached := s.cachedRows(ctx, meter, params, cutoff) + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("parity mismatch", "%s:\n%s", name, diff) + } + }) + } + } + } + } + } +} + +// TestQueryCachePartialFirstWindow locks in that a mid-hour `from` produces the +// same partial first window in the cache as in the live query. Both the populate +// and the live query filter `time >= from`, so the hour window straddling `from` +// aggregates only the events at/after `from` on both sides; the cache leg's +// floored lower bound (windowstart >= tumbleStart(from)) includes that partial +// window without re-including the excluded earlier events. +func (s *ConnectorTestSuite) TestQueryCachePartialFirstWindow() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "partial_event" + hour0 := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + from := hour0.Add(30 * time.Minute) // mid-hour lower bound + to := hour0.Add(3 * time.Hour) + + // One event BEFORE from (excluded), one after (included), plus later windows. + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: hour0.Add(15 * time.Minute), data: `{"value": 10}`}, // before from -> excluded + {subject: "s1", at: hour0.Add(45 * time.Minute), data: `{"value": 20}`}, // in hour 0, after from + {subject: "s1", at: hour0.Add(90 * time.Minute), data: `{"value": 5}`}, // hour 1 + }) + + meter := s.newMeter(parityMeter{ + name: "partial_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + + live := s.liveRows(ctx, meter, params) + // cutoff mid-range so both legs contribute. + cached := s.cachedRows(ctx, meter, params, hour0.Add(2*time.Hour)) + + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("partial-first-window parity mismatch", "%s", diff) + } + // The straddling hour must be 20 (only the post-from event), never 30. + for _, r := range cached { + if r.WindowStart.Equal(hour0) { + s.Equal(float64(20), r.Value, "partial first window must exclude pre-from events") + } + } +} + +// TestQueryCacheCrossQueryPartialWindow is the discriminating test for the +// shared-cache hazard: two queries over the same meter with DIFFERENT mid-hour +// `from`s that floor to the SAME hour must both match their live result. If +// populate stored partial (incomplete) hour windows, query A would write one +// partial for hour 00 and query B a different partial for the same key, and the +// read-time any() collapse would serve one query the other's partial — a silent +// wrong billing total. Storing only COMPLETE windows (head served live) makes +// every stored window identical regardless of `from`. +func (s *ConnectorTestSuite) TestQueryCacheCrossQueryPartialWindow() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "cross_event" + hour0 := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + to := hour0.Add(3 * time.Hour) + + // Three events inside hour 0 at :10, :20, :50, plus a later window. + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: hour0.Add(10 * time.Minute), data: `{"value": 1}`}, + {subject: "s1", at: hour0.Add(20 * time.Minute), data: `{"value": 2}`}, + {subject: "s1", at: hour0.Add(50 * time.Minute), data: `{"value": 4}`}, + {subject: "s1", at: hour0.Add(150 * time.Minute), data: `{"value": 8}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "cross_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + cutoff := hour0.Add(2 * time.Hour) + + // Query A: from = 00:15 (floors to hour 0). Live hour 0 = 2+4 = 6. + fromA := hour0.Add(15 * time.Minute) + paramsA := streaming.QueryParams{From: &fromA, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + liveA := s.liveRows(ctx, meter, paramsA) + cachedA := s.cachedRows(ctx, meter, paramsA, cutoff) + if diff := compareMeterRows(liveA, cachedA); diff != "" { + s.Failf("query A parity mismatch", "%s", diff) + } + + // Query B: from = 00:45 (also floors to hour 0), same wall-clock hour so it + // would collide on the same stored key. Live hour 0 = 4 (only the :50 event). + fromB := hour0.Add(45 * time.Minute) + paramsB := streaming.QueryParams{From: &fromB, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + liveB := s.liveRows(ctx, meter, paramsB) + cachedB := s.cachedRows(ctx, meter, paramsB, cutoff) + if diff := compareMeterRows(liveB, cachedB); diff != "" { + s.Failf("query B parity mismatch (cross-query partial-window pollution)", "%s", diff) + } + + // Re-run A after B populated: A must STILL be correct (B must not have + // overwritten hour 0 with its partial). + cachedA2 := s.cachedRows(ctx, meter, paramsA, cutoff) + if diff := compareMeterRows(liveA, cachedA2); diff != "" { + s.Failf("query A re-run mismatch after B populated", "%s", diff) + } +} + +// TestQueryCacheMidHourFromLowCutoff exercises the boundary where the cutoff +// falls at or below the head-ceiling: mid-hour `from` with a cutoff <= headCeil. +// The cache range [headCeil, cutoff) is then empty, and the head [from, headCeil) +// and tail [cutoff, to) legs must NOT overlap (which would double-count events in +// [cutoff, headCeil)). Covers the all-fresh extreme (cutoff==from) with a +// mid-hour from, which the hour-aligned parity matrix cannot reach. +func (s *ConnectorTestSuite) TestQueryCacheMidHourFromLowCutoff() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "midhour_event" + hour0 := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + from := hour0.Add(15 * time.Minute) // headCeil = 01:00 + to := hour0.Add(3 * time.Hour) + + // Event at 00:30 sits in [from, headCeil) — the overlap window if head and + // tail both cover it. + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: hour0.Add(30 * time.Minute), data: `{"value": 10}`}, + {subject: "s1", at: hour0.Add(90 * time.Minute), data: `{"value": 20}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "midhour_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + live := s.liveRows(ctx, meter, params) + + // Cutoffs at/below headCeil (01:00): the cache range is empty and head+tail + // must partition the range without overlap. + for _, cutoff := range []time.Time{from, hour0.Add(30 * time.Minute), hour0.Add(time.Hour)} { + s.Run(cutoff.Sub(from).String(), func() { + s.truncateCacheTable(ctx) + cached := s.cachedRows(ctx, meter, params, cutoff) + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("mid-hour from / low cutoff parity mismatch", "%s", diff) + } + // The 00:30 event's hour must be 10, never 20 (doubled). + for _, r := range cached { + if r.WindowStart.Equal(hour0) { + s.Equal(float64(10), r.Value, "head/tail overlap must not double-count hour 0") + } + } + }) + } +} + +// TestQueryCacheTwoMetersSameType covers the meter axis of the cache key: two +// distinct meters in the same namespace CAN share an event type (the unique +// constraint is on (namespace, key), not (namespace, event_type)). They must not +// collide in the cache — populating one must not corrupt the other's rows. +// +// The sharp case is two SUM meters over the SAME type with DIFFERENT value +// properties (e.g. sum of tokens vs sum of latency): both write sum_value under +// the same (ns,type,window,subject,group_by) sort key, so without a meter +// discriminator the read-time any(sum_value) serves one meter the other's total. +// (SUM-vs-MIN accidentally survives because any() skips the NULL non-owned +// column, so it is NOT a sufficient test of the collision.) +func (s *ConnectorTestSuite) TestQueryCacheTwoMetersSameType() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "shared_type" + base := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + from := base + to := base.Add(3 * time.Hour) + cutoff := base.Add(2 * time.Hour) + + // Distinct magnitudes for tokens vs latency so a collision is obvious. + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: base.Add(15 * time.Minute), data: `{"tokens": 100, "latency": 3}`}, + {subject: "s1", at: base.Add(45 * time.Minute), data: `{"tokens": 200, "latency": 4}`}, + {subject: "s1", at: base.Add(90 * time.Minute), data: `{"tokens": 50, "latency": 1}`}, + }) + + tokensMeter := s.newMeter(parityMeter{name: "shared_tokens", eventType: eventType, valueProperty: ptr("$.tokens"), aggregation: meterpkg.MeterAggregationSum}) + latencyMeter := s.newMeter(parityMeter{name: "shared_latency", eventType: eventType, valueProperty: ptr("$.latency"), aggregation: meterpkg.MeterAggregationSum}) + + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + + liveTokens := s.liveRows(ctx, tokensMeter, params) + liveLatency := s.liveRows(ctx, latencyMeter, params) + + // Populate BOTH meters into the SAME cache table FIRST, then read each — with + // NO re-population between populate and read. Re-populating right before the + // read masks the collision (the just-inserted row wins the order-dependent + // any() pick), so population and read are deliberately separated here. + s.populateCache(ctx, tokensMeter, params, cutoff) + s.populateCache(ctx, latencyMeter, params, cutoff) + + cachedTokens := s.readCached(ctx, tokensMeter, params, cutoff) + cachedLatency := s.readCached(ctx, latencyMeter, params, cutoff) + + if diff := compareMeterRows(liveTokens, cachedTokens); diff != "" { + s.Failf("tokens meter corrupted by latency meter sharing the event type", "%s", diff) + } + if diff := compareMeterRows(liveLatency, cachedLatency); diff != "" { + s.Failf("latency meter corrupted by tokens meter sharing the event type", "%s", diff) + } +} + +func (s *ConnectorTestSuite) truncateCacheTable(ctx context.Context) { + // The coverage claims must go with the rows: a surviving claim over a + // truncated table would make the next read skip population and serve the + // settled range as empty. + for _, name := range []string{parityCacheTable, meterQueryRowCacheCoverageTable} { + table := getTableName(s.Connector.config.Database, name) + s.NoError(s.Connector.config.ClickHouse.Exec(ctx, "TRUNCATE TABLE IF EXISTS "+table)) + } +} + +func windowSizeName(ws *meterpkg.WindowSize) string { + if ws == nil { + return "total" + } + return string(*ws) +} + +// TestQueryCacheStoreRereadWithGap populates a settled range that has a GAP (an +// hour window with no events), then re-queries an OVERLAPPING range and asserts +// parity. This is the shape the old suite never had — where the two removed bugs +// met. It proves the rollup + merge is correct across separate populate/read +// calls with a real gap. +func (s *ConnectorTestSuite) TestQueryCacheStoreRereadWithGap() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + base := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + eventType := "gap_event" + + // Hours 0 and 3 have events; hours 1,2 are a gap; hour 5 more events. + events := []parityEvent{ + {subject: "s1", at: base.Add(15 * time.Minute), data: `{"value": 10, "region": "us"}`}, + {subject: "s1", at: base.Add(3*time.Hour + 15*time.Minute), data: `{"value": 20, "region": "us"}`}, + {subject: "s2", at: base.Add(5*time.Hour + 15*time.Minute), data: `{"value": 30, "region": "eu"}`}, + } + s.seedEvents(ctx, eventType, events) + + meter := s.newMeter(parityMeter{ + name: "gap_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, groupBy: map[string]string{"region": "$.region"}, + }) + + from := base + to := base.Add(6 * time.Hour) + + // First: populate a sub-range [from, from+4h) (covers the gap), separate call. + firstParams := streaming.QueryParams{From: &from, To: ptr(base.Add(4 * time.Hour)), WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"region"}} + q := s.buildQueryMeter(meter, firstParams, []string{"region"}) + s.NoError(s.Connector.populateMeterQueryRowCache(ctx, q, from, base.Add(4*time.Hour))) + + // Now re-query the OVERLAPPING wider range with a cutoff inside the cached + // portion, so the cache leg serves the gap+events and the tail serves hour 5. + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"region"}} + live := s.liveRows(ctx, meter, params) + cached := s.cachedRows(ctx, meter, params, base.Add(4*time.Hour)) + + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("store-reread parity mismatch", "%s", diff) + } +} + +// TestQueryCacheConcurrentPopulate proves §4.2: two concurrent populates of the +// SAME settled window must not double-count SUM/COUNT after the read-time cache +// leg collapse. +func (s *ConnectorTestSuite) TestQueryCacheConcurrentPopulate() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + base := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + eventType := "concurrent_event" + events := []parityEvent{ + {subject: "s1", at: base.Add(15 * time.Minute), data: `{"value": 10}`}, + {subject: "s1", at: base.Add(45 * time.Minute), data: `{"value": 5}`}, + } + s.seedEvents(ctx, eventType, events) + + meter := s.newMeter(parityMeter{ + name: "concurrent_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + from := base + to := base.Add(2 * time.Hour) + cutoff := base.Add(1 * time.Hour) // hour 0 is settled + + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + q := s.buildQueryMeter(meter, params, nil) + + // Populate the SAME settled window twice (simulating a race). With plain + // MergeTree + no read-time collapse this would double the SUM. + s.NoError(s.Connector.populateMeterQueryRowCache(ctx, q, from, cutoff)) + s.NoError(s.Connector.populateMeterQueryRowCache(ctx, q, from, cutoff)) + + live := s.liveRows(ctx, meter, params) + cached := s.cachedRows(ctx, meter, params, cutoff) + + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("concurrent populate double-count", "%s", diff) + } + // Explicit: the settled hour's SUM must be 15, not 30. + var settledValue float64 = -1 + for _, r := range cached { + if r.WindowStart.Equal(from) { + settledValue = r.Value + } + } + s.Equal(float64(15), settledValue, "settled window SUM must not be doubled by concurrent populate") +} + +// TestQueryCacheGateRouting asserts that queries the gate must NOT admit route +// to the live path (canQueryBeCached == false): minute windows, fractional-hour +// timezones, AVG, UNIQUE_COUNT, customer_id, stored-at filter, too-short span, +// and too-fresh range. +func (s *ConnectorTestSuite) TestQueryCacheGateRouting() { + if s.T().Skipped() { + return + } + s.enableCacheOnConnector() + + sumMeter := s.newMeter(parityMeter{name: "gate_sum", eventType: "gate_event", valueProperty: ptr("$.value"), aggregation: meterpkg.MeterAggregationSum}) + avgMeter := s.newMeter(parityMeter{name: "gate_avg", eventType: "gate_event", valueProperty: ptr("$.value"), aggregation: meterpkg.MeterAggregationAvg}) + uniqueMeter := s.newMeter(parityMeter{name: "gate_unique", eventType: "gate_event", valueProperty: ptr("$.value"), aggregation: meterpkg.MeterAggregationUniqueCount}) + dimMeter := s.newMeter(parityMeter{name: "gate_dim", eventType: "gate_event", valueProperty: ptr("$.value"), aggregation: meterpkg.MeterAggregationSum, groupBy: map[string]string{"region": "$.region"}}) + + oldFrom := time.Now().UTC().Add(-30 * 24 * time.Hour) + oldTo := time.Now().UTC().Add(-10 * 24 * time.Hour) + + kolkata, err := time.LoadLocation("Asia/Kolkata") + s.NoError(err) + + cases := []struct { + name string + meter meterpkg.Meter + params streaming.QueryParams + want bool + }{ + { + name: "sum hour whole range -> cacheable", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: true, + }, + { + name: "total with To set -> cacheable", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: nil}, + want: true, + }, + { + name: "total with To nil -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: nil, WindowSize: nil}, + want: false, + }, + { + name: "minute window -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeMinute)}, + want: false, + }, + { + name: "fractional-hour tz -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeDay), WindowTimeZone: kolkata}, + want: false, + }, + { + name: "avg -> live", + meter: avgMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + name: "unique_count -> live", + meter: uniqueMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + name: "not cachable flag -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: false, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + name: "no from -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + name: "stored-at filter -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), FilterStoredAt: &filter.FilterTimeUnix{FilterTime: filter.FilterTime{Gte: ptr(oldFrom)}}}, + want: false, + }, + { + name: "customer_id group by -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"customer_id"}}, + want: false, + }, + { + name: "too-fresh range -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: ptr(time.Now().UTC().Add(-30 * time.Minute)), To: ptr(time.Now().UTC()), WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + // Span (30m) below MinimumCacheableQueryPeriod (1h) while the range IS + // settled — isolates the period guard from the age guard, so deleting + // either one on its own turns some case red. + name: "settled but too-short span -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: ptr(time.Now().UTC().Add(-3 * time.Hour)), To: ptr(time.Now().UTC().Add(-150 * time.Minute)), WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + // Span (70m) above the minimum period while from is fresher than the + // horizon — isolates the age guard from the period guard. + name: "long span but fresh from -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: ptr(time.Now().UTC().Add(-30 * time.Minute)), To: ptr(time.Now().UTC().Add(40 * time.Minute)), WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + name: "windowed with To nil -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: nil, WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + name: "client id (progress tracking) -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, ClientID: ptr("client-1"), From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)}, + want: false, + }, + { + name: "customer filter -> live", + meter: sumMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), FilterCustomer: []streaming.Customer{gateTestCustomer{}}}, + want: false, + }, + { + name: "filter on meter dimension -> cacheable", + meter: dimMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), FilterGroupBy: map[string]filter.FilterString{"region": filterEq("us")}}, + want: true, + }, + { + // The live path rejects filter keys that are not meter dimensions with a + // validation error (including "subject"); routing to live preserves that. + name: "filter on unknown key -> live", + meter: dimMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), FilterGroupBy: map[string]filter.FilterString{"subject": filterEq("s1")}}, + want: false, + }, + } + + for _, tc := range cases { + s.Run(tc.name, func() { + got := s.Connector.canQueryBeCached(namespace, tc.meter, tc.params) + s.Equal(tc.want, got, tc.name) + }) + } + + // Decimal precision is required: the cache only serves the provably-exact + // decimal mode. Toggle it off and confirm an otherwise-cacheable query routes + // to live. + s.Run("decimal precision off -> live", func() { + s.Connector.config.EnableDecimalPrecision = false + defer func() { s.Connector.config.EnableDecimalPrecision = true }() + + got := s.Connector.canQueryBeCached(namespace, sumMeter, streaming.QueryParams{ + Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), + }) + s.False(got, "decimal precision off must route to live") + }) + + // Feature gate: enabled for every namespace by default (nil checker or Noop + // gate -> true); a gate backend that denies the namespace routes to live. + cacheable := streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)} + + s.Run("feature gate nil (default) -> cacheable", func() { + s.Connector.config.FeatureGate = nil + s.True(s.Connector.canQueryBeCached(namespace, sumMeter, cacheable), "nil feature gate must default to enabled") + }) + + s.Run("feature gate noop -> cacheable", func() { + s.Connector.config.FeatureGate = featuregate.NewFeatureGateChecker(featuregate.NewNoop(), nil, nil) + defer func() { s.Connector.config.FeatureGate = nil }() + s.True(s.Connector.canQueryBeCached(namespace, sumMeter, cacheable), "noop gate must default to enabled") + }) + + s.Run("feature gate denies namespace -> live", func() { + s.Connector.config.FeatureGate = featuregate.NewFeatureGateChecker(alwaysFalseGate{}, nil, nil) + defer func() { s.Connector.config.FeatureGate = nil }() + s.False(s.Connector.canQueryBeCached(namespace, sumMeter, cacheable), "a denying feature gate must route to live") + }) +} + +// alwaysFalseGate denies every namespace/flag — used to prove the feature gate +// can restrict the cache per namespace (the cloud posture). +type alwaysFalseGate struct{} + +func (alwaysFalseGate) EvaluateBool(_, _ string, _ bool) (bool, error) { return false, nil } + +// TestQueryCacheEndToEndPublicPath drives the fully-wired public QueryMeter with +// the cache enabled, against a query the gate admits (old range, hour window, +// decimal precision). It proves the whole path — canQueryBeCached → lazy +// populate-on-read → merge → scan — produces the same rows as the same query on +// a cache-disabled connector. This is the coverage the direct-merge harness +// (which bypasses the gate and QueryMeter) cannot give. +func (s *ConnectorTestSuite) TestQueryCacheEndToEndPublicPath() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "e2e_event" + // Range must be older than the 1h horizon and span >= 1h; use a settled range + // well in the past so cutoff (now-1h) is after `to` (all-cached). + to := time.Now().UTC().Add(-25 * time.Hour).Truncate(time.Hour) + from := to.Add(-6 * time.Hour) + + events := []parityEvent{ + {subject: "s1", at: from.Add(30 * time.Minute), data: `{"value": 10, "region": "us"}`}, + {subject: "s2", at: from.Add(2*time.Hour + 15*time.Minute), data: `{"value": 20, "region": "eu"}`}, + {subject: "s1", at: from.Add(4*time.Hour + 5*time.Minute), data: `{"value": 5, "region": "us"}`}, + } + s.seedEvents(ctx, eventType, events) + + meter := s.newMeter(parityMeter{ + name: "e2e_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, groupBy: map[string]string{"region": "$.region"}, + }) + + // Sanity: the gate must admit this query. + params := streaming.QueryParams{ + Cachable: true, From: &from, To: &to, + WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"region"}, + } + s.True(s.Connector.canQueryBeCached(namespace, meter, params), "query must be admitted by the gate") + + // Cache-enabled public path. + cached, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + + // Cache-disabled comparison: same connector with the flag off (live path). + s.Connector.config.QueryCacheEnabled = false + live, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + s.Connector.config.QueryCacheEnabled = true + + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("end-to-end public path parity mismatch", "%s", diff) + } + s.NotEmpty(cached, "expected non-empty result") +} + +// TestQueryCacheInvalidationOnLateEvent proves §4.3: inserting an event older +// than the freshness horizon invalidates (wipes) the affected namespace's cached +// rows, while a fresh event does not. +func (s *ConnectorTestSuite) TestQueryCacheInvalidationOnLateEvent() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "invalidation_event" + from := time.Now().UTC().Add(-10 * 24 * time.Hour).Truncate(time.Hour) + cutoff := from.Add(4 * time.Hour) + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: from.Add(30 * time.Minute), data: `{"value": 10}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "invalidation_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + params := streaming.QueryParams{From: &from, To: ptr(from.Add(4 * time.Hour)), WindowSize: ptr(meterpkg.WindowSizeHour)} + q := s.buildQueryMeter(meter, params, nil) + s.NoError(s.Connector.populateMeterQueryRowCache(ctx, q, from, cutoff)) + s.Positive(s.countCacheRows(ctx), "cache should be populated") + + // A fresh event (now) must NOT invalidate (it does not mutate a settled window). + s.NoError(s.Connector.BatchInsert(ctx, []streaming.RawEvent{{ + Namespace: namespace, ID: ulid.Make().String(), Time: time.Now().UTC(), Type: eventType, + Source: "test", Subject: "s1", Data: `{"value": 1}`, IngestedAt: time.Now().UTC(), StoredAt: time.Now().UTC(), + }})) + s.Positive(s.countCacheRows(ctx), "fresh event must not invalidate the cache") + + // A late event (older than the horizon) MUST invalidate the namespace. + s.NoError(s.Connector.BatchInsert(ctx, []streaming.RawEvent{{ + Namespace: namespace, ID: ulid.Make().String(), Time: from.Add(45 * time.Minute), Type: eventType, + Source: "test", Subject: "s1", Data: `{"value": 99}`, IngestedAt: time.Now().UTC(), StoredAt: time.Now().UTC(), + }})) + // Lightweight DELETE is async-applied but mutations block until done by + // default; give the count a moment via a direct read. + s.Zero(s.countCacheRows(ctx), "late event must wipe the namespace's cache") +} + +func (s *ConnectorTestSuite) countCacheRows(ctx context.Context) int { + table := getTableName(s.Connector.config.Database, meterQueryRowCacheTable) + rows, err := s.Connector.config.ClickHouse.Query(ctx, "SELECT count() FROM "+table+" WHERE namespace = ?", namespace) + s.NoError(err) + defer rows.Close() + var count uint64 + for rows.Next() { + s.NoError(rows.Scan(&count)) + } + return int(count) +} + +// TestQueryCacheOffByDefault asserts that with the cache disabled, QueryMeter +// takes the live path and no cache table is created. +func (s *ConnectorTestSuite) TestQueryCacheOffByDefault() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + + // Fresh connector with cache OFF (the suite default). + connector, err := New(ctx, Config{ + Logger: slog.Default(), + ClickHouse: s.ClickHouse, + Database: s.Database, + EventsTableName: "off_events", + ProgressManager: progressmanager.NewMockProgressManager(), + }) + s.NoError(err) + + // The cache table must NOT exist. + table := getTableName(s.Database, meterQueryRowCacheTable) + rows, err := s.ClickHouse.Query(ctx, "EXISTS TABLE "+table) + s.NoError(err) + var exists uint8 + for rows.Next() { + s.NoError(rows.Scan(&exists)) + } + rows.Close() + s.Equal(uint8(0), exists, "cache table must not exist when cache is disabled") + + // canQueryBeCached must be false regardless of params. + oldFrom := time.Now().UTC().Add(-30 * 24 * time.Hour) + oldTo := time.Now().UTC().Add(-10 * 24 * time.Hour) + meter := meterpkg.Meter{ + ManagedResource: models.ManagedResource{ID: ulid.Make().String(), NamespacedModel: models.NamespacedModel{Namespace: namespace}}, + Key: "off_sum", EventType: "off_event", ValueProperty: ptr("$.value"), Aggregation: meterpkg.MeterAggregationSum, + } + s.False(connector.canQueryBeCached(namespace, meter, streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour)})) +} + +// gateTestCustomer is a minimal streaming.Customer for gate tests. +type gateTestCustomer struct{} + +func (gateTestCustomer) GetUsageAttribution() streaming.CustomerUsageAttribution { + return streaming.NewCustomerUsageAttribution("cus_1", nil, []string{"s1"}) +} + +// TestQueryCacheMeterShapeChange covers the meter-mutation axis of the cache +// key: meter definitions are mutable (UpdateMeter can add or remove group-by +// dimensions), and the group_by array is aligned to the CURRENT sorted paths. +// Rows written before and after a definition change must never combine into one +// aggregate — without the meter_hash fingerprint in the key and read filter, +// both shapes coexist under different sort keys and the cache leg double-counts +// every settled hour (SUM returns exactly 2x). +func (s *ConnectorTestSuite) TestQueryCacheMeterShapeChange() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "shape_event" + base := time.Date(2026, 11, 2, 0, 0, 0, 0, time.UTC) + from := base + to := base.Add(4 * time.Hour) + cutoff := base.Add(3 * time.Hour) + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: base.Add(10 * time.Minute), data: `{"value": 10, "model": "gpt4", "region": "us"}`}, + {subject: "s1", at: base.Add(40 * time.Minute), data: `{"value": 20, "model": "gpt4", "region": "eu"}`}, + {subject: "s1", at: base.Add(80 * time.Minute), data: `{"value": 5, "model": "mini", "region": "us"}`}, + }) + + // Shape v1: group by model only. + meterV1 := s.newMeter(parityMeter{ + name: "shape_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, groupBy: map[string]string{"model": "$.model"}, + }) + // Shape v2: SAME slug (definition updated), region dimension added. + meterV2 := meterV1 + meterV2.GroupBy = map[string]string{"model": "$.model", "region": "$.region"} + + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"model"}} + + // Populate BOTH shapes into the shared table (v1 first, then the "updated" + // definition), with no truncate between — exactly what a meter update + // followed by the next cached query produces. + s.populateCache(ctx, meterV1, params, cutoff) + s.populateCache(ctx, meterV2, params, cutoff) + + liveV2 := s.liveRows(ctx, meterV2, params) + cachedV2 := s.readCached(ctx, meterV2, params, cutoff) + if diff := compareMeterRows(liveV2, cachedV2); diff != "" { + s.Failf("updated meter shape corrupted by pre-update rows", "%s", diff) + } + + // The pre-update shape must also stay correct (its rows are keyed to its own + // fingerprint). + liveV1 := s.liveRows(ctx, meterV1, params) + cachedV1 := s.readCached(ctx, meterV1, params, cutoff) + if diff := compareMeterRows(liveV1, cachedV1); diff != "" { + s.Failf("pre-update meter shape corrupted by post-update rows", "%s", diff) + } +} + +// TestQueryCachePopulateFailureFallsBackToLive covers the populate failure +// path: the settled range is served exclusively from the cache table, so a +// failed populate must NOT proceed to the merge (which would silently drop the +// whole settled range and undercount). The connector must fall back to the full +// live query and still return the correct result. Dropping the cache table +// makes the populate INSERT fail deterministically. +func (s *ConnectorTestSuite) TestQueryCachePopulateFailureFallsBackToLive() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "fallback_event" + to := time.Now().UTC().Add(-2 * time.Hour).Truncate(time.Hour) + from := to.Add(-6 * time.Hour) + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: from.Add(30 * time.Minute), data: `{"value": 10}`}, + {subject: "s1", at: from.Add(3 * time.Hour), data: `{"value": 20}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "fallback_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + params := streaming.QueryParams{Cachable: true, From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + s.True(s.Connector.canQueryBeCached(namespace, meter, params), "query must be admitted by the gate") + + // Live control BEFORE breaking the cache table. + live, err := s.Connector.QueryMeter(ctx, namespace, meter, streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)}) + s.NoError(err) + s.NotEmpty(live) + + // Break population: the cached path must fall back to live, not undercount. + table := getTableName(s.Connector.config.Database, meterQueryRowCacheTable) + s.NoError(s.Connector.config.ClickHouse.Exec(ctx, "DROP TABLE "+table+" SYNC")) + + cached, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err, "populate failure must not fail the query") + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("populate-failure fallback must serve the full live result", "%s", diff) + } + + // The failed populate must not have claimed coverage: a claim over rows + // that were never written would make the next read skip population and + // serve the settled range as empty. + s.Zero(s.countCoverageRows(ctx, meter.Key), "no coverage claim may be stored after a failed populate") + + // Restore the table for any later suite activity. + s.NoError(s.Connector.createMeterQueryRowCacheTable(ctx)) +} + +// TestQueryCacheNewestRowWins pins the read-time collapse semantics: when the +// write-once invariant is violated (a populate racing the late-event +// invalidation DELETE persists a stale row alongside a later corrected row), +// the cache leg must deterministically serve the NEWEST row — including when +// the newest value is legitimately NULL, which a bare argMax would skip in +// favor of the stale non-NULL one. +func (s *ConnectorTestSuite) TestQueryCacheNewestRowWins() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "newest_event" + base := time.Date(2026, 12, 1, 0, 0, 0, 0, time.UTC) + from := base + to := base.Add(2 * time.Hour) + cutoff := to // all-cached: the settled range is served purely from the table + + meter := s.newMeter(parityMeter{ + name: "newest_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + table := getTableName(s.Connector.config.Database, meterQueryRowCacheTable) + hash := meterShapeHash(meter) + + insert := func(ws time.Time, sum *int64, createdAt string) { + var sumSQL string + if sum == nil { + sumSQL = "NULL" + } else { + sumSQL = fmt.Sprintf("toDecimal128(%d, 19)", *sum) + } + s.NoError(s.Connector.config.ClickHouse.Exec(ctx, fmt.Sprintf( + `INSERT INTO %s (namespace, type, meter_slug, meter_hash, windowstart, subject, group_by, sum_value, count_value, min_value, max_value, created_at) + VALUES (?, ?, ?, ?, ?, ?, [], %s, 1, NULL, NULL, toDateTime64(?, 3))`, table, sumSQL), + namespace, eventType, meter.Key, hash, ws, "s1", createdAt)) + } + + // Hour 0: stale row (100) then corrected row (60) 1ms later -> 60 must win. + insert(base, ptr(int64(100)), "2026-12-01 12:00:00.001") + insert(base, ptr(int64(60)), "2026-12-01 12:00:00.002") + // Hour 1: stale non-NULL (7) then corrected NULL -> the row must vanish + // (NULL value rows are skipped on scan, matching live). + insert(base.Add(time.Hour), ptr(int64(7)), "2026-12-01 12:00:00.001") + insert(base.Add(time.Hour), nil, "2026-12-01 12:00:00.002") + + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + rows := s.readCached(ctx, meter, params, cutoff) + + s.Require().Len(rows, 1, "hour 1's newest row is NULL and must be skipped; expected only hour 0. rows=%s", dumpRows(rows)) + s.Equal(base, rows[0].WindowStart.UTC()) + s.Equal(float64(60), rows[0].Value, "the NEWEST stored rollup must win, not the stale one") +} + +// TestQueryCacheDSTTimezoneParity compares cached vs live for a gate-admitted +// whole-hour DST timezone across the US spring-forward transition (2026-03-08 +// in America/New_York): day windows are 23 hours long that day and month +// windows span the offset change, exercising the re-tumbling of hourly-UTC +// rollup rows into tz-local windows. +func (s *ConnectorTestSuite) TestQueryCacheDSTTimezoneParity() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + ny, err := time.LoadLocation("America/New_York") + s.Require().NoError(err) + + eventType := "dst_event" + // [Mar 6 .. Mar 10) UTC, hour-aligned: spans the Mar 8 02:00 EST -> 03:00 EDT jump. + from := time.Date(2026, 3, 6, 0, 0, 0, 0, time.UTC) + to := time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC) + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: time.Date(2026, 3, 6, 12, 30, 0, 0, time.UTC), data: `{"value": 1, "region": "us"}`}, + {subject: "s1", at: time.Date(2026, 3, 8, 6, 15, 0, 0, time.UTC), data: `{"value": 2, "region": "us"}`}, // 01:15 EST, pre-jump + {subject: "s1", at: time.Date(2026, 3, 8, 7, 45, 0, 0, time.UTC), data: `{"value": 4, "region": "eu"}`}, // 03:45 EDT, post-jump + {subject: "s2", at: time.Date(2026, 3, 8, 23, 5, 0, 0, time.UTC), data: `{"value": 8, "region": "us"}`}, // 19:05 EDT + {subject: "s1", at: time.Date(2026, 3, 9, 15, 0, 0, 0, time.UTC), data: `{"value": 16, "region": "us"}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "dst_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, groupBy: map[string]string{"region": "$.region"}, + }) + + windowSizes := []*meterpkg.WindowSize{ptr(meterpkg.WindowSizeHour), ptr(meterpkg.WindowSizeDay), ptr(meterpkg.WindowSizeMonth)} + cutoffs := []time.Time{from, time.Date(2026, 3, 8, 7, 0, 0, 0, time.UTC), to} // all-fresh, mid-jump, all-cached + + for _, ws := range windowSizes { + for _, groupBy := range [][]string{nil, {"region"}, {"subject"}} { + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ws, WindowTimeZone: ny, GroupBy: groupBy} + live := s.liveRows(ctx, meter, params) + + for _, cutoff := range cutoffs { + name := fmt.Sprintf("%s/groupby=%v/cutoff=%s", windowSizeName(ws), groupBy, cutoff.Sub(from)) + s.Run(name, func() { + s.truncateCacheTable(ctx) + cached := s.cachedRows(ctx, meter, params, cutoff) + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("DST timezone parity mismatch", "%s:\n%s", name, diff) + } + }) + } + } + } +} + +// TestQueryCacheShadowParityCheck pins the production correctness net: the +// shadow verifier must report clean cache state as a match, detect a corrupted +// cached row as a mismatch, and self-heal by invalidating the namespace so the +// next query repopulates from raw data. +func (s *ConnectorTestSuite) TestQueryCacheShadowParityCheck() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "shadow_event" + base := time.Date(2026, 10, 15, 0, 0, 0, 0, time.UTC) + from := base + to := base.Add(3 * time.Hour) + cutoff := to // all-cached: corruption in the table is fully visible to the merge + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: base.Add(20 * time.Minute), data: `{"value": 10}`}, + {subject: "s1", at: base.Add(100 * time.Minute), data: `{"value": 5}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "shadow_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + params := streaming.QueryParams{From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + q := s.buildQueryMeter(meter, params, nil) + + // Clean state: populate + read, the verifier must report a match. + cached := s.cachedRows(ctx, meter, params, cutoff) + s.False(s.Connector.verifyCachedResultParity(ctx, q, cached), "clean cache must verify as a match") + + // Corrupt the cached hour 0 with a newer, wrong rollup (newest-wins makes the + // corruption deterministic) and re-read: the served result is now wrong, and + // the verifier must catch it and wipe the namespace. + table := getTableName(s.Connector.config.Database, meterQueryRowCacheTable) + s.NoError(s.Connector.config.ClickHouse.Exec(ctx, fmt.Sprintf( + `INSERT INTO %s (namespace, type, meter_slug, meter_hash, windowstart, subject, group_by, sum_value, count_value, min_value, max_value, created_at) + VALUES (?, ?, ?, ?, ?, ?, [], toDecimal128(999, 19), 1, NULL, NULL, now64(3) + toIntervalHour(1))`, table), + namespace, eventType, meter.Key, meterShapeHash(meter), base, "s1")) + + corrupted := s.readCached(ctx, meter, params, cutoff) + s.True(s.Connector.verifyCachedResultParity(ctx, q, corrupted), "corrupted cache must verify as a mismatch") + s.Zero(s.countCacheRows(ctx), "a parity mismatch must self-heal by invalidating the namespace cache") + + // After self-healing, the next populate + read verifies clean again. + healed := s.cachedRows(ctx, meter, params, cutoff) + s.False(s.Connector.verifyCachedResultParity(ctx, q, healed), "repopulated cache must verify as a match again") +} + +// TestQueryCacheTelemetry proves the observability wiring actually emits: with +// a recording tracer and an in-memory metric reader attached, a cache-served +// query must produce the query/populate spans and the queries counter, and a +// parity check must produce its span with the outcome attribute. This guards +// the "monitor correctness in production" contract — a silently detached +// instrument would otherwise look identical to a healthy quiet system. +func (s *ConnectorTestSuite) TestQueryCacheTelemetry() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + // Attach a recording tracer + in-memory metrics to the suite connector. + spanRecorder := tracetest.NewSpanRecorder() + s.Connector.tracer = sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder)).Tracer("test") + + metricReader := sdkmetric.NewManualReader() + metrics, err := newQueryCacheMetrics(sdkmetric.NewMeterProvider(sdkmetric.WithReader(metricReader)).Meter("test")) + s.Require().NoError(err) + s.Connector.queryCacheMetrics = metrics + + defer func() { + s.Connector.tracer = tracenoop.NewTracerProvider().Tracer("test") + s.Connector.queryCacheMetrics = nil + }() + + eventType := "telemetry_event" + to := time.Now().UTC().Add(-3 * time.Hour).Truncate(time.Hour) + from := to.Add(-4 * time.Hour) + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: from.Add(30 * time.Minute), data: `{"value": 10}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "telemetry_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + params := streaming.QueryParams{Cachable: true, From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + + // Cache-served query through the public path. + rows, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + s.NotEmpty(rows) + + // Parity check (synchronous call, same as the sampled shadow goroutine runs). + q := s.buildQueryMeter(meter, params, nil) + s.False(s.Connector.verifyCachedResultParity(ctx, q, rows)) + + // Spans: query + populate from the cached query, parity_check from the verify. + spanNames := map[string]bool{} + for _, span := range spanRecorder.Ended() { + spanNames[span.Name()] = true + } + s.True(spanNames["streaming.query_cache.query"], "cached query span missing; got %v", spanNames) + s.True(spanNames["streaming.query_cache.populate"], "populate span missing; got %v", spanNames) + s.True(spanNames["streaming.query_cache.parity_check"], "parity check span missing; got %v", spanNames) + + // Metrics: the queries counter and both duration histograms must have data. + var rm metricdata.ResourceMetrics + s.Require().NoError(metricReader.Collect(ctx, &rm)) + metricNames := map[string]bool{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + metricNames[m.Name] = true + } + } + s.True(metricNames["streaming.query_cache.queries"], "queries counter missing; got %v", metricNames) + s.True(metricNames["streaming.query_cache.query_duration_ms"], "query duration histogram missing; got %v", metricNames) + s.True(metricNames["streaming.query_cache.populate_duration_ms"], "populate duration histogram missing; got %v", metricNames) + s.True(metricNames["streaming.query_cache.parity_checks"], "parity checks counter missing; got %v", metricNames) +} + +// TestQueryCacheCrossTimezoneSharing proves cached rows are shared safely +// across timezones — the reason WindowTimeZone is deliberately NOT part of the +// cache key. The rollup stores timezone-agnostic hourly-UTC partials; the +// query's timezone only re-windows them at read time, and the gate admits only +// whole-hour-offset zones, where every tz-local window boundary (including DST +// transition instants) falls on a UTC hour boundary. +// +// Two invariants are pinned: +// 1. Populate output is byte-identical regardless of the querying timezone — +// if someone ever makes populate tz-aware, cross-tz sharing breaks and this +// fails. +// 2. Rows populated ONCE serve UTC, America/New_York and Europe/Budapest +// queries (hour/day/month windows, mid and all-cached cutoffs) with exact +// live parity, across the 2026 US spring-forward, EU spring-forward and US +// fall-back transitions (the fall-back range contains the repeated local +// hour: two distinct UTC hours labeled 01:xx locally). +func (s *ConnectorTestSuite) TestQueryCacheCrossTimezoneSharing() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + ny, err := time.LoadLocation("America/New_York") + s.Require().NoError(err) + budapest, err := time.LoadLocation("Europe/Budapest") + s.Require().NoError(err) + + eventType := "crosstz_event" + + ranges := []struct { + name string + from time.Time + to time.Time + }{ + // US spring forward 2026-03-08 (07:00Z): NY day is 23h. + {name: "us-spring", from: time.Date(2026, 3, 6, 0, 0, 0, 0, time.UTC), to: time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)}, + // EU spring forward 2026-03-29 (01:00Z): Budapest day is 23h. + {name: "eu-spring", from: time.Date(2026, 3, 27, 0, 0, 0, 0, time.UTC), to: time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC)}, + // US fall back 2026-11-01 (06:00Z): NY day is 25h, local 01:xx occurs twice. + {name: "us-fall", from: time.Date(2026, 10, 30, 0, 0, 0, 0, time.UTC), to: time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)}, + } + + // Events land around each transition instant, on both sides of it, plus the + // repeated local hour on the fall-back day (05:30Z and 06:30Z are both + // "01:30" in New York on 2026-11-01, in different UTC hour buckets). + events := []parityEvent{ + {subject: "s1", at: time.Date(2026, 3, 6, 12, 30, 0, 0, time.UTC), data: `{"value": 1, "region": "us"}`}, + {subject: "s1", at: time.Date(2026, 3, 8, 6, 15, 0, 0, time.UTC), data: `{"value": 2, "region": "us"}`}, + {subject: "s2", at: time.Date(2026, 3, 8, 7, 45, 0, 0, time.UTC), data: `{"value": 4, "region": "eu"}`}, + {subject: "s1", at: time.Date(2026, 3, 29, 0, 30, 0, 0, time.UTC), data: `{"value": 8, "region": "eu"}`}, + {subject: "s2", at: time.Date(2026, 3, 29, 1, 30, 0, 0, time.UTC), data: `{"value": 16, "region": "eu"}`}, + {subject: "s1", at: time.Date(2026, 11, 1, 5, 30, 0, 0, time.UTC), data: `{"value": 32, "region": "us"}`}, + {subject: "s2", at: time.Date(2026, 11, 1, 6, 30, 0, 0, time.UTC), data: `{"value": 64, "region": "us"}`}, + {subject: "s1", at: time.Date(2026, 11, 1, 15, 0, 0, 0, time.UTC), data: `{"value": 128, "region": "eu"}`}, + } + s.seedEvents(ctx, eventType, events) + + meter := s.newMeter(parityMeter{ + name: "crosstz_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, groupBy: map[string]string{"region": "$.region"}, + }) + + // Invariant 1: populate under a New York query and under a UTC query must + // write byte-identical rows. + r := ranges[0] + nyParams := streaming.QueryParams{From: &r.from, To: &r.to, WindowSize: ptr(meterpkg.WindowSizeDay), WindowTimeZone: ny} + utcParams := streaming.QueryParams{From: &r.from, To: &r.to, WindowSize: ptr(meterpkg.WindowSizeDay)} + + s.truncateCacheTable(ctx) + s.populateCache(ctx, meter, nyParams, r.to) + nySnapshot := s.snapshotCacheRows(ctx) + + s.truncateCacheTable(ctx) + s.populateCache(ctx, meter, utcParams, r.to) + utcSnapshot := s.snapshotCacheRows(ctx) + + s.Require().NotEmpty(nySnapshot) + s.Equal(utcSnapshot, nySnapshot, "populate must write identical rows regardless of the query timezone") + + // Invariant 2: rows populated once serve every admitted timezone with live + // parity, across all three DST transitions. No re-population between reads. + timezones := []*time.Location{nil, ny, budapest} // nil = UTC + windowSizes := []*meterpkg.WindowSize{ptr(meterpkg.WindowSizeHour), ptr(meterpkg.WindowSizeDay), ptr(meterpkg.WindowSizeMonth)} + + for _, r := range ranges { + s.truncateCacheTable(ctx) + s.populateCache(ctx, meter, streaming.QueryParams{From: &r.from, To: &r.to, WindowSize: ptr(meterpkg.WindowSizeHour)}, r.to) + + mid := r.from.Add(r.to.Sub(r.from) / 2).Truncate(time.Hour) + for _, tz := range timezones { + for _, ws := range windowSizes { + for _, cutoff := range []time.Time{mid, r.to} { + params := streaming.QueryParams{From: &r.from, To: &r.to, WindowSize: ws, WindowTimeZone: tz, GroupBy: []string{"region"}} + name := fmt.Sprintf("%s/tz=%v/%s/cutoff=%s", r.name, tz, windowSizeName(ws), cutoff.Sub(r.from)) + s.Run(name, func() { + live := s.liveRows(ctx, meter, params) + cached := s.readCached(ctx, meter, params, cutoff) + if diff := compareMeterRows(live, cached); diff != "" { + s.Failf("cross-timezone cache sharing parity mismatch", "%s:\n%s", name, diff) + } + }) + } + } + } + } +} + +// TestQueryCacheCoverageSkipsRepopulation pins the gap-tracking contract on +// the public path: a repeat read over a range whose coverage claim exists must +// NOT re-scan raw events. The only external observation of "did it +// re-populate" is mutating the raw data without invalidating the cache +// (bypassing BatchInsert) and checking whether the mutation leaks into the +// next cached result: it must not — until a real late event arrives through +// BatchInsert, whose invalidation must wipe the claim along with the rows and +// force a full, mutation-visible repopulation. +func (s *ConnectorTestSuite) TestQueryCacheCoverageSkipsRepopulation() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "coverage_skip_event" + to := time.Now().UTC().Add(-25 * time.Hour).Truncate(time.Hour) + from := to.Add(-6 * time.Hour) + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: from.Add(30 * time.Minute), data: `{"value": 10}`}, + {subject: "s1", at: from.Add(3 * time.Hour), data: `{"value": 20}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "coverage_skip_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + params := streaming.QueryParams{Cachable: true, From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + + // Age out the marker the seeding BatchInsert wrote, so the claim below is + // honored immediately instead of being distrusted for the skew margin. + s.clearInvalidationMarkers(ctx) + + // First read: populates and claims coverage; must match live. + first, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + if diff := compareMeterRows(s.liveRows(ctx, meter, params), first); diff != "" { + s.Failf("first cached read must match live", "%s", diff) + } + + // Mutate the settled raw data WITHOUT invalidation. The live result now + // includes the mutation (sanity check that it is a real discriminator)... + s.insertRawEventBypassingInvalidation(ctx, eventType, "s1", from.Add(90*time.Minute), `{"value": 999}`) + s.NotEmpty(compareMeterRows(first, s.liveRows(ctx, meter, params)), "the raw mutation must be visible to a live scan") + + // ...but the covered repeat read must serve the existing rollups untouched. + second, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + if diff := compareMeterRows(first, second); diff != "" { + s.Failf("covered repeat read re-populated (raw mutation leaked into the cache)", "%s", diff) + } + + // A real late event goes through BatchInsert: its invalidation must wipe + // the claim with the rows, so the next read repopulates and sees BOTH the + // mutation and the late event — full live parity again. + s.NoError(s.Connector.BatchInsert(ctx, []streaming.RawEvent{{ + Namespace: namespace, ID: ulid.Make().String(), Time: from.Add(150 * time.Minute), Type: eventType, + Source: "test", Subject: "s1", Data: `{"value": 7}`, IngestedAt: time.Now().UTC(), StoredAt: time.Now().UTC(), + }})) + s.Zero(s.countCoverageRows(ctx, meter.Key), "late-event invalidation must wipe the coverage claim") + + third, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + if diff := compareMeterRows(s.liveRows(ctx, meter, params), third); diff != "" { + s.Failf("post-invalidation read must repopulate to full live parity", "%s", diff) + } +} + +// TestQueryCacheCoverageWidensRange covers the partial-coverage read: a wide +// query over a range whose middle is already claimed must populate ONLY the +// missing part. Both directions are asserted through raw-data mutations: +// the gap's events must appear (the gap was populated) while a mutation inside +// the already-covered part must not (that part was served from the existing +// rollups). +func (s *ConnectorTestSuite) TestQueryCacheCoverageWidensRange() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "coverage_widen_event" + to := time.Now().UTC().Add(-25 * time.Hour).Truncate(time.Hour) + from := to.Add(-96 * time.Hour) + narrowFrom := from.Add(48 * time.Hour) + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: from.Add(90 * time.Minute), data: `{"value": 1}`}, // early (gap) region + {subject: "s1", at: from.Add(30 * time.Hour), data: `{"value": 2}`}, // early (gap) region + {subject: "s1", at: narrowFrom.Add(2 * time.Hour), data: `{"value": 4}`}, + {subject: "s1", at: narrowFrom.Add(32 * time.Hour), data: `{"value": 8}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "coverage_widen_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + // Age out the marker the seeding BatchInsert wrote (see clearInvalidationMarkers). + s.clearInvalidationMarkers(ctx) + + total := func(f time.Time) float64 { + params := streaming.QueryParams{Cachable: true, From: &f, To: &to} + rows, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.Require().NoError(err) + s.Require().Len(rows, 1) + return rows[0].Value + } + + // Narrow read claims [narrowFrom, to): 4 + 8. + s.Equal(float64(12), total(narrowFrom)) + + // Mutate raw data inside the covered narrow range, without invalidation. + s.insertRawEventBypassingInvalidation(ctx, eventType, "s1", narrowFrom.Add(4*time.Hour), `{"value": 1000}`) + + // Wide read: the early gap must be populated (1 + 2 appear) while the + // covered part is served from the existing rollups (1000 must not appear). + // 15 proves both at once: 1012 would mean the covered part re-populated, + // 12 would mean the gap was skipped. + s.Equal(float64(15), total(from)) + + // Fully covered now: the repeat wide read is stable. + s.Equal(float64(15), total(from)) +} + +// TestQueryCacheCoverageTiedClaimsStayAtomic pins the coverage read's +// tuple-atomicity: two claims for the same meter written with the SAME +// created_at (racing disjoint first-claims can land in the same millisecond) +// must resolve to ONE of the stored intervals — never a stitched interval +// taking covered_from from one row and covered_until from the other, which +// would assert coverage over the unpopulated gap between them. +func (s *ConnectorTestSuite) TestQueryCacheCoverageTiedClaimsStayAtomic() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + meter := s.newMeter(parityMeter{ + name: "coverage_tie_sum", eventType: "coverage_tie_event", valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + + base := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + claimA := cacheCoverage{From: base, Until: base.Add(10 * time.Hour), FirstWrittenAt: base, PopulatedAt: base} + claimB := cacheCoverage{From: base.Add(12 * time.Hour), Until: base.Add(20 * time.Hour), FirstWrittenAt: base, PopulatedAt: base} + + table := getTableName(s.Connector.config.Database, meterQueryRowCacheCoverageTable) + for _, claim := range []cacheCoverage{claimA, claimB} { + s.Require().NoError(s.Connector.config.ClickHouse.Exec(ctx, fmt.Sprintf( + `INSERT INTO %s (namespace, meter_slug, meter_hash, covered_from, covered_until, first_written_at, populated_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, toDateTime64('2026-06-02 00:00:00.000', 3))`, table), + namespace, meter.Key, meterShapeHash(meter), claim.From, claim.Until, claim.FirstWrittenAt, claim.PopulatedAt)) + } + + q := s.buildQueryMeter(meter, streaming.QueryParams{}, nil) + got, _, err := s.Connector.readMeterQueryRowCacheCoverage(ctx, q) + s.Require().NoError(err) + s.Require().NotNil(got) + + isA := got.From.Equal(claimA.From) && got.Until.Equal(claimA.Until) + isB := got.From.Equal(claimB.From) && got.Until.Equal(claimB.Until) + s.True(isA || isB, "tied claims must resolve to one stored interval, got stitched [%s, %s)", got.From, got.Until) +} + +// TestQueryCacheCoverageInvalidationBeatsRacingClaim pins the fix for the +// claim/invalidation race: a cached read that planned BEFORE an invalidation +// can commit its coverage claim AFTER the invalidation's deletes (an INSERT +// cannot be ordered against a concurrent DELETE), leaving a claim that vouches +// for wiped rollups. The invalidation marker must make every read distrust +// such a claim — its populated_at predates the marker — so the next read +// repopulates instead of serving the settled range as empty. +func (s *ConnectorTestSuite) TestQueryCacheCoverageInvalidationBeatsRacingClaim() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + eventType := "coverage_race_event" + to := time.Now().UTC().Add(-25 * time.Hour).Truncate(time.Hour) + from := to.Add(-4 * time.Hour) + + s.seedEvents(ctx, eventType, []parityEvent{ + {subject: "s1", at: from.Add(30 * time.Minute), data: `{"value": 10}`}, + }) + + meter := s.newMeter(parityMeter{ + name: "coverage_race_sum", eventType: eventType, valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + params := streaming.QueryParams{Cachable: true, From: &from, To: &to, WindowSize: ptr(meterpkg.WindowSizeHour)} + + // Normal first read: populates and claims. + _, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + + // The racing reader's plan starts HERE — before the invalidation below. + stalePlanStart := time.Now().UTC() + + // A late event arrives through the real path: the invalidation inserts the + // namespace marker and wipes the rollups + claims. + s.NoError(s.Connector.BatchInsert(ctx, []streaming.RawEvent{{ + Namespace: namespace, ID: ulid.Make().String(), Time: from.Add(90 * time.Minute), Type: eventType, + Source: "test", Subject: "s1", Data: `{"value": 5}`, IngestedAt: time.Now().UTC(), StoredAt: time.Now().UTC(), + }})) + + // The racing reader now lands its claim AFTER the invalidation — the exact + // interleaving the marker exists for. Its row is the newest in the table. + q := s.buildQueryMeter(meter, params, nil) + s.NoError(s.Connector.storeMeterQueryRowCacheCoverage(ctx, q, cacheCoverage{ + From: from, Until: to, FirstWrittenAt: stalePlanStart, PopulatedAt: stalePlanStart, + })) + + // The next read must distrust the resurrected claim (its populated_at + // predates the marker), repopulate, and serve the full result including + // the late event — never the wiped range as empty. + got, err := s.Connector.QueryMeter(ctx, namespace, meter, params) + s.NoError(err) + s.NotEmpty(got, "the settled range must not be served from the wiped cache as empty") + if diff := compareMeterRows(s.liveRows(ctx, meter, params), got); diff != "" { + s.Failf("read after a racing claim resurrection must repopulate to live parity", "%s", diff) + } +} + +// insertRawEventBypassingInvalidation writes an event straight into the events +// table with raw SQL, skipping BatchInsert's late-event cache invalidation. +// Coverage tests use it to mutate settled raw data WITHOUT wiping the cache — +// the only external way to observe whether a read re-populated (the mutation +// shows up) or served the existing rollups (it does not). +func (s *ConnectorTestSuite) insertRawEventBypassingInvalidation(ctx context.Context, eventType, subject string, at time.Time, data string) { + table := getTableName(s.Connector.config.Database, s.Connector.config.EventsTableName) + s.Require().NoError(s.Connector.config.ClickHouse.Exec(ctx, fmt.Sprintf( + `INSERT INTO %s (namespace, id, type, subject, source, time, data, ingested_at, stored_at, store_row_id) + VALUES (?, ?, ?, ?, 'coverage-test', ?, ?, ?, ?, '')`, table), + namespace, ulid.Make().String(), eventType, subject, at, data, at, at)) +} + +// clearInvalidationMarkers removes the namespace's invalidation markers. +// seedEvents ingests old events through BatchInsert, which legitimately +// invalidates the namespace — and claims written within the clock-skew margin +// of a marker are deliberately distrusted (one redundant populate in +// production). Tests that assert a covered read does NOT repopulate would see +// that redundant populate as a failure, so they age the seeding marker out +// first, as if the seed ingestion happened long ago. +func (s *ConnectorTestSuite) clearInvalidationMarkers(ctx context.Context) { + table := getTableName(s.Connector.config.Database, meterQueryRowCacheCoverageTable) + s.Require().NoError(s.Connector.config.ClickHouse.Exec(ctx, + "DELETE FROM "+table+" WHERE namespace = ? AND meter_slug = ''", namespace)) +} + +func (s *ConnectorTestSuite) countCoverageRows(ctx context.Context, meterSlug string) int { + table := getTableName(s.Connector.config.Database, meterQueryRowCacheCoverageTable) + rows, err := s.Connector.config.ClickHouse.Query(ctx, + "SELECT count() FROM "+table+" WHERE namespace = ? AND meter_slug = ?", namespace, meterSlug) + s.NoError(err) + defer rows.Close() + var count uint64 + for rows.Next() { + s.NoError(rows.Scan(&count)) + } + return int(count) +} + +// snapshotCacheRows returns a canonical dump of every cache row except +// created_at (which legitimately differs between populates). +func (s *ConnectorTestSuite) snapshotCacheRows(ctx context.Context) []string { + table := getTableName(s.Connector.config.Database, meterQueryRowCacheTable) + rows, err := s.Connector.config.ClickHouse.Query(ctx, ` + SELECT namespace, type, meter_slug, meter_hash, toString(windowstart), subject, + toString(group_by), toString(sum_value), toString(count_value), + toString(min_value), toString(max_value) + FROM `+table+` + ORDER BY namespace, type, meter_slug, meter_hash, windowstart, subject, group_by`) + s.Require().NoError(err) + defer rows.Close() + + var out []string + for rows.Next() { + cols := make([]string, 11) + dest := make([]interface{}, len(cols)) + for i := range cols { + dest[i] = &cols[i] + } + s.Require().NoError(rows.Scan(dest...)) + out = append(out, fmt.Sprintf("%v", cols)) + } + s.Require().NoError(rows.Err()) + return out +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go new file mode 100644 index 0000000000..2a14a6b7f8 --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go @@ -0,0 +1,326 @@ +package clickhouse + +import ( + "context" + "fmt" + "math" + "math/rand/v2" + "sort" + "strings" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" +) + +// parityCheckTimeout bounds the detached shadow verification query. It is +// generous on purpose: the shadow runs the FULL live query (the expensive scan +// the cache exists to avoid), off the request path. +const parityCheckTimeout = 2 * time.Minute + +// queryCacheMetrics holds the OTel instruments that make the cache's health and +// correctness observable in production. All methods are nil-receiver safe so a +// connector constructed without a metric.Meter (tests, embedded use) pays +// nothing. +// +// - streaming.query_cache.queries{result}: cached | live_fallback — how much +// traffic the cache actually serves, and how often it degrades to live. +// - streaming.query_cache.populate_errors: rollup INSERT failures (each one +// also forced a live fallback). +// - streaming.query_cache.invalidations{reason}: late_event | parity_mismatch +// namespace wipes. +// - streaming.query_cache.parity_checks{outcome}: match | mismatch | error — +// the shadow verification results. ALERT ON mismatch > 0: it means a +// cache-served result diverged from raw data. +// - streaming.query_cache.query_duration_ms / populate_duration_ms: latency of +// the merge query and of the rollup population — the two heavyweight SQL +// operations; the cache pays off only if the first stays well under the live +// scan it replaces. +type queryCacheMetrics struct { + queries metric.Int64Counter + populateErrors metric.Int64Counter + invalidations metric.Int64Counter + parityChecks metric.Int64Counter + queryDuration metric.Int64Histogram + populateDuration metric.Int64Histogram +} + +func newQueryCacheMetrics(meter metric.Meter) (*queryCacheMetrics, error) { + queries, err := meter.Int64Counter( + "streaming.query_cache.queries", + metric.WithDescription("Number of meter queries admitted to the query cache, by result (cached, live_fallback)"), + ) + if err != nil { + return nil, fmt.Errorf("create query cache queries counter: %w", err) + } + + populateErrors, err := meter.Int64Counter( + "streaming.query_cache.populate_errors", + metric.WithDescription("Number of failed query cache rollup populations"), + ) + if err != nil { + return nil, fmt.Errorf("create query cache populate errors counter: %w", err) + } + + invalidations, err := meter.Int64Counter( + "streaming.query_cache.invalidations", + metric.WithDescription("Number of namespace-wide query cache invalidations, by reason (late_event, parity_mismatch)"), + ) + if err != nil { + return nil, fmt.Errorf("create query cache invalidations counter: %w", err) + } + + parityChecks, err := meter.Int64Counter( + "streaming.query_cache.parity_checks", + metric.WithDescription("Number of sampled shadow parity checks of cache-served results against the live query, by outcome (match, mismatch, error)"), + ) + if err != nil { + return nil, fmt.Errorf("create query cache parity checks counter: %w", err) + } + + queryDuration, err := meter.Int64Histogram( + "streaming.query_cache.query_duration_ms", + metric.WithDescription("Duration of the cache merge query (cache leg + live legs) in milliseconds"), + ) + if err != nil { + return nil, fmt.Errorf("create query cache query duration histogram: %w", err) + } + + populateDuration, err := meter.Int64Histogram( + "streaming.query_cache.populate_duration_ms", + metric.WithDescription("Duration of the query cache rollup population in milliseconds"), + ) + if err != nil { + return nil, fmt.Errorf("create query cache populate duration histogram: %w", err) + } + + return &queryCacheMetrics{ + queries: queries, + populateErrors: populateErrors, + invalidations: invalidations, + parityChecks: parityChecks, + queryDuration: queryDuration, + populateDuration: populateDuration, + }, nil +} + +func (m *queryCacheMetrics) recordQuery(ctx context.Context, result string) { + if m == nil { + return + } + m.queries.Add(ctx, 1, metric.WithAttributes(attribute.String("result", result))) +} + +func (m *queryCacheMetrics) recordPopulateError(ctx context.Context) { + if m == nil { + return + } + m.populateErrors.Add(ctx, 1) +} + +func (m *queryCacheMetrics) recordInvalidation(ctx context.Context, reason string, namespaces int) { + if m == nil { + return + } + m.invalidations.Add(ctx, int64(namespaces), metric.WithAttributes(attribute.String("reason", reason))) +} + +func (m *queryCacheMetrics) recordParityCheck(ctx context.Context, outcome string) { + if m == nil { + return + } + m.parityChecks.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", outcome))) +} + +func (m *queryCacheMetrics) recordQueryDuration(ctx context.Context, elapsed time.Duration) { + if m == nil { + return + } + m.queryDuration.Record(ctx, elapsed.Milliseconds()) +} + +func (m *queryCacheMetrics) recordPopulateDuration(ctx context.Context, elapsed time.Duration) { + if m == nil { + return + } + m.populateDuration.Record(ctx, elapsed.Milliseconds()) +} + +// maybeShadowVerifyCachedResult samples cache-served results for shadow +// verification against the live query. The check runs in a detached goroutine +// (the caller's response is already on its way; cancellation of the request +// must not kill the verification) bounded by parityCheckTimeout. +// +// This is the production correctness net: unit and integration parity tests +// prove the design, the shadow check proves THIS deployment — catching whatever +// no test anticipated (replica lag, mutation races, operational surprises) at a +// configurable sampling cost of one extra live query per sampled request. +func (c *Connector) maybeShadowVerifyCachedResult(ctx context.Context, query queryMeter, cachedRows []meterpkg.MeterQueryRow) { + rate := c.config.QueryCacheParityCheckSampleRate + if rate <= 0 { + return + } + + if rate < 1 && rand.Float64() >= rate { + return + } + + // Detach from the request's cancellation but keep its values (tracing). + detached, cancel := context.WithTimeout(context.WithoutCancel(ctx), parityCheckTimeout) + + go func() { + defer cancel() + + c.verifyCachedResultParity(detached, query, cachedRows) + }() +} + +// verifyCachedResultParity re-runs the query on the live path and compares it +// to the cache-served rows on the same criteria as the parity test gate: the +// full (windowstart, windowend, subject, customer, group-by) tuple, values +// rounded to 6 decimals. On mismatch it reports loudly (error log + metric) and +// self-heals by invalidating the namespace's cache, so the next query +// repopulates from raw data. Returns whether a mismatch was found (for tests). +func (c *Connector) verifyCachedResultParity(ctx context.Context, query queryMeter, cachedRows []meterpkg.MeterQueryRow) bool { + ctx, span := c.tracer.Start(ctx, "streaming.query_cache.parity_check", trace.WithAttributes( + attribute.String("namespace", query.Namespace), + attribute.String("meter_slug", query.Meter.Key), + )) + defer span.End() + + logger := c.config.Logger.With( + "namespace", query.Namespace, + "meterSlug", query.Meter.Key, + "from", query.From, + "to", query.To, + ) + + liveRows, err := c.queryMeter(ctx, query) + if err != nil { + c.queryCacheMetrics.recordParityCheck(ctx, "error") + span.SetAttributes(attribute.String("outcome", "error")) + span.RecordError(err) + logger.Warn("query cache parity check failed to run live query", "error", err) + + return false + } + + // For the total (nil) window, QueryMeter overwrites windowstart/end with + // From/To on whatever path served the request; normalize both sides the same + // way so the comparison sees what callers see. + cached := normalizeTotalWindowBounds(cachedRows, query) + live := normalizeTotalWindowBounds(liveRows, query) + + diff := compareMeterQueryRowSets(live, cached) + if diff == "" { + c.queryCacheMetrics.recordParityCheck(ctx, "match") + span.SetAttributes(attribute.String("outcome", "match")) + + return false + } + + c.queryCacheMetrics.recordParityCheck(ctx, "mismatch") + span.SetAttributes(attribute.String("outcome", "mismatch")) + span.SetStatus(codes.Error, "cache-served result diverged from live query") + logger.Error("QUERY CACHE PARITY MISMATCH: cache-served result diverged from live query, invalidating namespace cache", "diff", diff) + + if err := c.invalidateMeterQueryRowCache(ctx, []string{query.Namespace}); err != nil { + span.RecordError(err) + logger.Error("failed to invalidate meter query cache after parity mismatch", "error", err) + + return true + } + c.queryCacheMetrics.recordInvalidation(ctx, "parity_mismatch", 1) + + return true +} + +// normalizeTotalWindowBounds applies QueryMeter's nil-window From/To overwrite +// to a copy of the rows, so cached and live results are compared in the shape +// callers actually receive. +func normalizeTotalWindowBounds(rows []meterpkg.MeterQueryRow, query queryMeter) []meterpkg.MeterQueryRow { + out := append([]meterpkg.MeterQueryRow(nil), rows...) + if query.WindowSize != nil { + return out + } + + for i := range out { + if query.From != nil { + out[i].WindowStart = *query.From + } + if query.To != nil { + out[i].WindowEnd = *query.To + } + } + + return out +} + +// compareMeterQueryRowSets returns "" when the two result sets are identical, +// keyed on the full row tuple with values compared to 6 decimal places — +// deliberately the same criteria as the parity test gate. Any difference +// returns a human-readable description for the mismatch log. +func compareMeterQueryRowSets(live, cached []meterpkg.MeterQueryRow) string { + if len(live) != len(cached) { + return fmt.Sprintf("row count differs: live=%d cached=%d", len(live), len(cached)) + } + + l := append([]meterpkg.MeterQueryRow(nil), live...) + c := append([]meterpkg.MeterQueryRow(nil), cached...) + sort.Slice(l, func(i, j int) bool { return meterQueryRowKey(l[i]) < meterQueryRowKey(l[j]) }) + sort.Slice(c, func(i, j int) bool { return meterQueryRowKey(c[i]) < meterQueryRowKey(c[j]) }) + + for i := range l { + lk, ck := meterQueryRowKey(l[i]), meterQueryRowKey(c[i]) + if lk != ck { + return fmt.Sprintf("row key differs: live=%q cached=%q", lk, ck) + } + + if math.Round(l[i].Value*1e6) != math.Round(c[i].Value*1e6) { + return fmt.Sprintf("value differs for %q: live=%v cached=%v", lk, l[i].Value, c[i].Value) + } + } + + return "" +} + +// meterQueryRowKey is the full grouping tuple of a result row: window bounds, +// subject, customer and every group-by dimension. Comparing on anything less +// would line up different groups' rows against each other. +func meterQueryRowKey(r meterpkg.MeterQueryRow) string { + var b strings.Builder + + b.WriteString(r.WindowStart.UTC().Format(time.RFC3339)) + b.WriteByte(0) + b.WriteString(r.WindowEnd.UTC().Format(time.RFC3339)) + b.WriteByte(0) + if r.Subject != nil { + b.WriteString(*r.Subject) + } + b.WriteByte(0) + if r.CustomerID != nil { + b.WriteString(*r.CustomerID) + } + b.WriteByte(0) + + keys := make([]string, 0, len(r.GroupBy)) + for k := range r.GroupBy { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + b.WriteString(k) + b.WriteByte(1) + if v := r.GroupBy[k]; v != nil { + b.WriteString(*v) + } + b.WriteByte(2) + } + + return b.String() +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go new file mode 100644 index 0000000000..04feb411bb --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go @@ -0,0 +1,285 @@ +package clickhouse + +import ( + "context" + "fmt" + "math" + "sort" + "strings" + "time" + + "github.com/oklog/ulid/v2" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" + "github.com/openmeterio/openmeter/openmeter/streaming" + "github.com/openmeterio/openmeter/pkg/filter" + "github.com/openmeterio/openmeter/pkg/models" +) + +// This file holds the billing-safety parity gate: for every (meter, window size, +// filter subset, aggregation) the cacheability gate admits — and at the two +// extremes (all-fresh cutoff==from, all-cached cutoff==to) — the cached result +// must equal the live result. It ports the cachebench parity harness into the +// connector integration suite, comparing the merge (queryCachedMeter, driven at +// an explicit cutoff) against the live queryMeter path. + +const parityCacheTable = meterQueryRowCacheTable + +type parityMeter struct { + name string + eventType string + valueProperty *string + aggregation meterpkg.MeterAggregation + groupBy map[string]string // meter group-by key -> JSON path +} + +type parityEvent struct { + subject string + at time.Time + data string +} + +func (s *ConnectorTestSuite) newMeter(m parityMeter) meterpkg.Meter { + now := time.Now().UTC() + meter := meterpkg.Meter{ + ManagedResource: models.ManagedResource{ + ID: ulid.Make().String(), + Name: m.name, + NamespacedModel: models.NamespacedModel{ + Namespace: namespace, + }, + ManagedModel: models.ManagedModel{CreatedAt: now, UpdatedAt: now}, + }, + Key: m.name, + EventType: m.eventType, + ValueProperty: m.valueProperty, + Aggregation: m.aggregation, + GroupBy: m.groupBy, + } + s.NoError(meter.Validate()) + return meter +} + +// seedEvents inserts raw events via BatchInsert. The late-event invalidation +// this triggers is harmless: every test populates the cache after seeding. +func (s *ConnectorTestSuite) seedEvents(ctx context.Context, eventType string, events []parityEvent) { + raw := make([]streaming.RawEvent, 0, len(events)) + now := time.Now().UTC() + for i, e := range events { + raw = append(raw, streaming.RawEvent{ + Namespace: namespace, + ID: fmt.Sprintf("%s-%d-%s", eventType, i, ulid.Make().String()), + Time: e.at, + Type: eventType, + Source: "parity-test", + Subject: e.subject, + Data: e.data, + IngestedAt: now, + StoredAt: now, + }) + } + s.NoError(s.Connector.BatchInsert(ctx, raw)) +} + +// liveRows runs the live meter query path for the given query params, applying +// the same nil-window From/To overwrite that the public QueryMeter applies, so +// it compares apples-to-apples with cachedRows (which also applies it). +func (s *ConnectorTestSuite) liveRows(ctx context.Context, m meterpkg.Meter, params streaming.QueryParams) []meterpkg.MeterQueryRow { + groupBy := append([]string(nil), params.GroupBy...) + sort.Strings(groupBy) + q := s.buildQueryMeter(m, params, groupBy) + rows, err := s.Connector.queryMeter(ctx, q) + s.NoError(err) + return overwriteTotalWindow(rows, params) +} + +// overwriteTotalWindow mirrors QueryMeter's behavior for the total (nil) window: +// each row's window start/end is replaced with the query's From/To. Both the +// live and cached comparison paths apply it so their windowstart/end line up. +func overwriteTotalWindow(rows []meterpkg.MeterQueryRow, params streaming.QueryParams) []meterpkg.MeterQueryRow { + if params.WindowSize != nil { + return rows + } + for i := range rows { + if params.From != nil { + rows[i].WindowStart = *params.From + } + if params.To != nil { + rows[i].WindowEnd = *params.To + } + } + return rows +} + +// cachedRows populates the settled range then runs the merge at the explicit +// cutoff, returning the merged rows. It is populate-then-read; for tests that +// must control population separately (e.g. cross-meter coexistence), use +// populateCache + readCached. +func (s *ConnectorTestSuite) cachedRows(ctx context.Context, m meterpkg.Meter, params streaming.QueryParams, cutoff time.Time) []meterpkg.MeterQueryRow { + s.populateCache(ctx, m, params, cutoff) + return s.readCached(ctx, m, params, cutoff) +} + +// populateCache rolls up the settled whole-hour range [ceil(from), floor(cutoff)) +// for the meter into the cache table, mirroring queryMeterCached's populate step. +func (s *ConnectorTestSuite) populateCache(ctx context.Context, m meterpkg.Meter, params streaming.QueryParams, cutoff time.Time) { + groupBy := append([]string(nil), params.GroupBy...) + sort.Strings(groupBy) + q := s.buildQueryMeter(m, params, groupBy) + + merge := s.newMergeQuery(q, cutoff) + cacheLo := merge.headCeil() + cacheHi := merge.cacheHi() + if cacheLo.Before(cacheHi) { + s.NoError(s.Connector.populateMeterQueryRowCache(ctx, q, cacheLo, cacheHi)) + } +} + +// readCached runs the merge (cache + live legs) WITHOUT populating, so a test can +// populate multiple meters into the shared table first and then read one. +func (s *ConnectorTestSuite) readCached(ctx context.Context, m meterpkg.Meter, params streaming.QueryParams, cutoff time.Time) []meterpkg.MeterQueryRow { + groupBy := append([]string(nil), params.GroupBy...) + sort.Strings(groupBy) + q := s.buildQueryMeter(m, params, groupBy) + + merge := s.newMergeQuery(q, cutoff) + sql, args, err := merge.toSQL() + s.Require().NoError(err) + + rows, err := s.Connector.config.ClickHouse.Query(ctx, sql, args...) + s.Require().NoError(err) + defer rows.Close() + + out, err := merge.scanRows(rows) + s.Require().NoError(err) + + // Mirror QueryMeter's nil-window overwrite so parity holds for total queries. + return overwriteTotalWindow(out, params) +} + +func (s *ConnectorTestSuite) newMergeQuery(q queryMeter, cutoff time.Time) queryCachedMeter { + return queryCachedMeter{ + Database: s.Connector.config.Database, + CacheTableName: parityCacheTable, + EventsTableName: s.Connector.config.EventsTableName, + query: q, + Cutoff: cutoff, + } +} + +func (s *ConnectorTestSuite) buildQueryMeter(m meterpkg.Meter, params streaming.QueryParams, groupBy []string) queryMeter { + return queryMeter{ + Database: s.Connector.config.Database, + EventsTableName: s.Connector.config.EventsTableName, + Namespace: namespace, + Meter: m, + From: params.From, + To: params.To, + FilterSubject: params.FilterSubject, + FilterGroupBy: params.FilterGroupBy, + GroupBy: groupBy, + WindowSize: params.WindowSize, + WindowTimeZone: params.WindowTimeZone, + EnableDecimalPrecision: s.Connector.config.EnableDecimalPrecision, + } +} + +// rowKey is the full parity key: window + windowend + subject + customer + group-by. +func rowKey(r meterpkg.MeterQueryRow) string { + var b strings.Builder + b.WriteString(r.WindowStart.UTC().Format(time.RFC3339)) + b.WriteByte(0) + b.WriteString(r.WindowEnd.UTC().Format(time.RFC3339)) + b.WriteByte(0) + if r.Subject != nil { + b.WriteString(*r.Subject) + } + b.WriteByte(0) + if r.CustomerID != nil { + b.WriteString(*r.CustomerID) + } + b.WriteByte(0) + + keys := make([]string, 0, len(r.GroupBy)) + for k := range r.GroupBy { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + b.WriteString(k) + b.WriteByte(1) + if v := r.GroupBy[k]; v != nil { + b.WriteString(*v) + } + b.WriteByte(2) + } + return b.String() +} + +// compareMeterRows returns "" if the two result sets match, keyed on the full +// tuple, exact value compared to 6 decimal places (matching cachebench's +// compareRows / digest rounding). Any difference returns a human-readable diff. +func compareMeterRows(live, cached []meterpkg.MeterQueryRow) string { + if len(live) != len(cached) { + return fmt.Sprintf("row count differs: live=%d cached=%d\nlive=%s\ncached=%s", + len(live), len(cached), dumpRows(live), dumpRows(cached)) + } + + l := append([]meterpkg.MeterQueryRow(nil), live...) + c := append([]meterpkg.MeterQueryRow(nil), cached...) + sort.Slice(l, func(i, j int) bool { return rowKey(l[i]) < rowKey(l[j]) }) + sort.Slice(c, func(i, j int) bool { return rowKey(c[i]) < rowKey(c[j]) }) + + for i := range l { + if rowKey(l[i]) != rowKey(c[i]) { + return fmt.Sprintf("key[%d] differs:\nlive= %q\ncached=%q", i, rowKey(l[i]), rowKey(c[i])) + } + if round6(l[i].Value) != round6(c[i].Value) { + return fmt.Sprintf("value @ %q differs: live=%v cached=%v", rowKey(l[i]), l[i].Value, c[i].Value) + } + } + return "" +} + +func round6(f float64) float64 { + return math.Round(f*1e6) / 1e6 +} + +func dumpRows(rows []meterpkg.MeterQueryRow) string { + var b strings.Builder + for _, r := range rows { + fmt.Fprintf(&b, "\n ws=%s we=%s subj=%v gb=%v val=%v", + r.WindowStart.UTC().Format(time.RFC3339), r.WindowEnd.UTC().Format(time.RFC3339), + ptrStr(r.Subject), groupByStr(r.GroupBy), r.Value) + } + return b.String() +} + +func ptrStr(s *string) string { + if s == nil { + return "" + } + return *s +} + +func groupByStr(m map[string]*string) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + for _, k := range keys { + b.WriteString(k) + b.WriteByte('=') + b.WriteString(ptrStr(m[k])) + b.WriteByte(',') + } + return b.String() +} + +func ptr[T any](v T) *T { return &v } + +func filterEq(v string) filter.FilterString { + return filter.FilterString{Eq: &v} +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_query.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_query.go new file mode 100644 index 0000000000..6c8c5262e5 --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_query.go @@ -0,0 +1,543 @@ +package clickhouse + +import ( + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/alpacahq/alpacadecimal" + "github.com/huandu/go-sqlbuilder" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" + "github.com/openmeterio/openmeter/pkg/filter" +) + +// queryCachedMeter builds the single-statement merge that serves a cacheable +// meter query. It reads settled hourly rollups from the cache table for +// [from, cutoff) and aggregates raw events for the fresh tail [cutoff, to), +// UNIONs them at hourly grain, then re-windows and re-aggregates to the +// requested window size and group-by subset. +// +// Structure: +// +// SELECT , , , (value) AS value +// FROM ( +// -- cache leg: settled whole hours, collapsed to dedupe concurrent populates (newest wins) +// SELECT windowstart AS windowstart_hourly, subject, group_by, argMax(, created_at) AS value +// FROM cache WHERE ns/type/meter/window range GROUP BY windowstart, subject, group_by +// UNION ALL +// -- live legs (sub-hour head + fresh tail): raw events -> hourly rollup, full group-by set +// SELECT tumbleStart(time,1h,UTC) AS windowstart_hourly, subject, [json paths] AS group_by, AS value +// FROM events WHERE ns/type/time range GROUP BY windowstart, subject, group exprs +// ) +// WHERE AND +// GROUP BY , , +// ORDER BY windowstart +// +// The cache leg's collapse (inner argMax-over-created_at GROUP BY) is the +// read-time half of the dedup-before-re-aggregation fix (§4.2): even before +// ReplacingMergeTree merges asynchronously, duplicate settled-window rows +// collapse to one row here — deterministically the NEWEST — so the outer sum() +// cannot double-count and a stale row from a populate/invalidation race loses +// to the corrected one. +type queryCachedMeter struct { + Database string + CacheTableName string + EventsTableName string + query queryMeter + // Cutoff is the freshness boundary (hour-aligned): [from, cutoff) is served + // from the cache, [cutoff, to) is scanned live. + Cutoff time.Time +} + +func (q queryCachedMeter) toSQL() (string, []interface{}, error) { + cacheTable := getTableName(q.Database, q.CacheTableName) + eventsTable := getTableName(q.Database, q.EventsTableName) + getColumn := columnFactory(q.EventsTableName) + timeColumn := getColumn("time") + + column, recombine, ok := aggCacheColumn(q.query.Meter.Aggregation) + if !ok { + return "", nil, fmt.Errorf("aggregation %q is not cacheable", q.query.Meter.Aggregation) + } + + paths := cacheGroupByPaths(q.query.Meter) + pathIndex := make(map[string]int, len(paths)) + for i, p := range paths { + pathIndex[p] = i + } + + // All legs emit an hourly windowstart under a DISTINCT alias + // (windowstart_hourly) so the outer layer can derive the requested window + // (hour/day/month) and its GROUP BY from it without the output alias + // windowstart shadowing the column it is computed from. Using windowstart on + // both sides risks a cyclic-alias resolution that groups at the wrong grain. + // + // The query range [from, to) is partitioned into non-overlapping legs. The + // cache serves only COMPLETE hour windows [cacheLo, cacheHi): + // cacheLo = ceil(from, 1h) — partial FIRST hour excluded (served live) + // cacheHi = floor(cutoff, 1h) — partial LAST hour excluded (served live) + // Both boundaries are hour-aligned so every stored window spans a full hour of + // the query. If that interval is empty (cacheLo >= cacheHi, e.g. a sub-hour or + // fully-fresh range) there is NO cache leg and the whole range is scanned live. + // When it is non-empty the live legs are the sub-hour head [from, cacheLo) and + // the tail [cacheHi, to). These three ranges always tile [from, to) with no gap + // and no overlap, so no event is dropped or double-counted. + // + // Storing only complete hour windows is the write-once invariant that makes + // concurrent/repeated population byte-identical: a partial window would store a + // different value for the same key depending on the query's `from`/`to`, and + // the read-time collapse would then serve the wrong total to a query with + // different bounds. + from := q.cacheFrom() + to := q.freshTo() + cacheLo := q.headCeil() + cacheHi := q.cacheHi() + + // --- Live legs (raw events -> hourly rollup, FULL group-by set) ------------ + liveValueExpr, err := q.freshLegValueExpr() + if err != nil { + return "", nil, err + } + + liveGroupExprs := make([]string, len(paths)) + for i, path := range paths { + liveGroupExprs[i] = q.query.groupByValueExpr(path) + } + liveGroupArray := "[]" + if len(liveGroupExprs) > 0 { + liveGroupArray = "[" + strings.Join(liveGroupExprs, ", ") + "]" + } + + liveWindowStart := fmt.Sprintf("tumbleStart(%s, %s, 'UTC')", timeColumn, cacheGrainInterval) + liveGroupByTerms := append([]string{liveWindowStart, "subject"}, liveGroupExprs...) + + // liveLeg produces a raw-event hourly rollup over the half-open [lo, hi) time + // range, reused for every live segment. + liveLegSQL := fmt.Sprintf(`SELECT %[1]s AS windowstart_hourly, subject, %[2]s AS group_by, %[3]s AS value + FROM %[4]s + WHERE namespace = ? AND type = ? AND %[5]s >= ? AND %[5]s < ? + GROUP BY %[6]s`, + liveWindowStart, liveGroupArray, liveValueExpr, eventsTable, timeColumn, strings.Join(liveGroupByTerms, ", ")) + + var legs []string + var cacheArgs, liveArgs []interface{} + + if cacheLo.Before(cacheHi) { + // Cache leg (settled WHOLE hours), collapsed per key BEFORE the union so + // the outer recombine cannot double-count duplicate stored rows. + // - Filtered by meter_slug (distinct meters can share an event type) AND + // meter_hash (a meter definition change orphans old-shape rows instead of + // letting two incompatible group_by shapes combine into one aggregate). + // - The collapse is argMax over created_at — NEWEST Wins — wrapped in + // tuple() because bare argMax skips NULL args (a stale non-NULL row would + // beat a newer legitimately-NULL rollup, and an all-NULL window would + // yield a spurious zero). Under the write-once invariant duplicates are + // byte-identical and this equals any(); when a populate races the + // late-event invalidation DELETE and persists a stale row, newest-wins + // makes reads deterministic and self-healing (the next populate writes a + // corrected row with a later created_at). + legs = append(legs, fmt.Sprintf(`SELECT windowstart AS windowstart_hourly, subject, group_by, tupleElement(argMax(tuple(%[1]s), created_at), 1) AS value + FROM %[2]s + WHERE namespace = ? AND type = ? AND meter_slug = ? AND meter_hash = ? AND windowstart >= toDateTime(?) AND windowstart < toDateTime(?) + GROUP BY windowstart, subject, group_by`, + column, cacheTable)) + cacheArgs = []interface{}{q.query.Namespace, q.query.Meter.EventType, q.query.Meter.Key, meterShapeHash(q.query.Meter), cacheLo.Unix(), cacheHi.Unix()} + + // Head leg: sub-hour partial first window [from, cacheLo). + if from.Before(cacheLo) { + legs = append(legs, liveLegSQL) + liveArgs = append(liveArgs, q.query.Namespace, q.query.Meter.EventType, from.Unix(), cacheLo.Unix()) + } + // Tail leg: fresh tail [cacheHi, to). + if cacheHi.Before(to) { + legs = append(legs, liveLegSQL) + liveArgs = append(liveArgs, q.query.Namespace, q.query.Meter.EventType, cacheHi.Unix(), to.Unix()) + } + } else { + // No cacheable whole-hour range: scan the entire query range live. + legs = append(legs, liveLegSQL) + liveArgs = append(liveArgs, q.query.Namespace, q.query.Meter.EventType, from.Unix(), to.Unix()) + } + + inner := strings.Join(legs, "\n\tUNION ALL\n\t") + + // --- Outer layer: filter, re-window, project subset, re-aggregate --------- + outerSelect, outerGroupBy, err := q.outerWindowColumns() + if err != nil { + return "", nil, err + } + + // Group-by subset projection + subject column. + for _, key := range q.query.GroupBy { + switch key { + case "subject": + outerSelect = append(outerSelect, "subject") + outerGroupBy = append(outerGroupBy, "subject") + case "customer_id": + // customer_id is derived from subject and is not cacheable; the gate + // rejects it. Defensive: refuse rather than emit a wrong column. + return "", nil, fmt.Errorf("customer_id group by is not cacheable") + default: + idx, ok := pathIndex[key] + if !ok { + return "", nil, fmt.Errorf("group by %q is not a stored meter dimension", key) + } + col := sqlbuilder.Escape(key) + // ClickHouse arrays are 1-based. + outerSelect = append(outerSelect, fmt.Sprintf("group_by[%d] AS %s", idx+1, col)) + outerGroupBy = append(outerGroupBy, col) + } + } + + outerSelect = append(outerSelect, fmt.Sprintf("%s(value) AS value", recombine)) + + // Read-time filters (applied identically to both legs via the UNION output). + var whereClauses []string + var whereArgs []interface{} + + // Subject allow-list: both legs carry all subjects, so restrict here. + if len(q.query.FilterSubject) > 0 { + subjects := append([]string(nil), q.query.FilterSubject...) + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(subjects)), ", ") + whereClauses = append(whereClauses, fmt.Sprintf("subject IN (%s)", placeholders)) + for _, s := range subjects { + whereArgs = append(whereArgs, s) + } + } + + // FilterGroupBy on stored dimensions. + if len(q.query.FilterGroupBy) > 0 { + filterKeys := make([]string, 0, len(q.query.FilterGroupBy)) + for k := range q.query.FilterGroupBy { + filterKeys = append(filterKeys, k) + } + sort.Strings(filterKeys) + + for _, key := range filterKeys { + fs := q.query.FilterGroupBy[key] + if fs.IsEmpty() { + continue + } + if err := fs.Validate(); err != nil { + return "", nil, fmt.Errorf("invalid filter for group by %s: %w", key, err) + } + + var column string + switch key { + case "subject": + column = "subject" + case "customer_id": + return "", nil, fmt.Errorf("customer_id filter is not cacheable") + default: + idx, ok := pathIndex[key] + if !ok { + return "", nil, fmt.Errorf("filter group by %q is not a stored meter dimension", key) + } + column = fmt.Sprintf("group_by[%d]", idx+1) + } + + expr, exprArgs := filterStringWhere(fs, column) + whereClauses = append(whereClauses, expr) + whereArgs = append(whereArgs, exprArgs...) + } + } + + whereSQL := "" + if len(whereClauses) > 0 { + whereSQL = "\nWHERE " + strings.Join(whereClauses, " AND ") + } + + orderBySQL := "" + if q.query.WindowSize != nil { + orderBySQL = "\nORDER BY windowstart" + } + + groupBySQL := "" + if len(outerGroupBy) > 0 { + groupBySQL = "\nGROUP BY " + strings.Join(outerGroupBy, ", ") + } + + sql := fmt.Sprintf(`SELECT %s +FROM ( + %s +)%s%s%s`, + strings.Join(outerSelect, ", "), + inner, + whereSQL, + groupBySQL, + orderBySQL, + ) + + // Arg order matches left-to-right placeholder order in the final SQL: outer + // window args (total: from/to), cache leg, live legs (head then tail), where. + args := append([]interface{}{}, q.outerWindowArgs()...) + args = append(args, cacheArgs...) + args = append(args, liveArgs...) + args = append(args, whereArgs...) + + // Query settings, mirroring the live path. + settings := make([]string, 0, len(q.query.QuerySettings)) + for key, value := range q.query.QuerySettings { + settings = append(settings, fmt.Sprintf("%s = %s", key, value)) + } + if len(settings) > 0 { + sql += fmt.Sprintf("\nSETTINGS %s", strings.Join(settings, ", ")) + } + + return sql, args, nil +} + +// cacheFrom returns the effective query from (merging Meter.EventFrom). It is +// the lower bound of the sub-hour head leg, NOT the cache leg (which starts at +// headCeil so it only ever stores complete hour windows). +func (q queryCachedMeter) cacheFrom() time.Time { + from := q.query.from() + if from == nil { + // canQueryBeCached requires a from; defensive fallback to cutoff. + return q.Cutoff + } + return *from +} + +// headCeil returns the effective from rounded UP to the next hour boundary — the +// start of the first COMPLETE hour window (cache lower bound). The sub-hour head +// [from, headCeil) is served live. If from is already hour-aligned, headCeil == +// from and the head leg is empty. +func (q queryCachedMeter) headCeil() time.Time { + from := q.cacheFrom() + floored := from.Truncate(cacheGrain) + if floored.Equal(from) { + return from + } + return floored.Add(cacheGrain) +} + +// cacheHi returns the cutoff rounded DOWN to the hour boundary — the end of the +// last COMPLETE hour window (cache upper bound, exclusive). The partial last hour +// [cacheHi, cutoff) and everything after is served live via the tail leg. This is +// the symmetric counterpart to headCeil: cutoff can be mid-hour (it is clamped to +// the query's arbitrary `to`), and caching a partial last hour under a whole-hour +// key would poison it exactly as a partial first hour would. +func (q queryCachedMeter) cacheHi() time.Time { + return q.Cutoff.Truncate(cacheGrain) +} + +// freshTo returns the upper bound of the fresh tail leg. +func (q queryCachedMeter) freshTo() time.Time { + if q.query.To != nil { + return *q.query.To + } + return time.Now().UTC() +} + +// freshLegValueExpr returns the fresh-tail aggregate that produces the same +// per-window partial the cache column stores, so both legs are recombined +// identically. COUNT is count(*); SUM/MIN/MAX wrap rawValueExpr. +func (q queryCachedMeter) freshLegValueExpr() (string, error) { + switch q.query.Meter.Aggregation { + case meterpkg.MeterAggregationCount: + return "count(*)", nil + case meterpkg.MeterAggregationSum: + return fmt.Sprintf("sum(%s)", q.query.rawValueExpr()), nil + case meterpkg.MeterAggregationMin: + return fmt.Sprintf("min(%s)", q.query.rawValueExpr()), nil + case meterpkg.MeterAggregationMax: + return fmt.Sprintf("max(%s)", q.query.rawValueExpr()), nil + default: + return "", fmt.Errorf("aggregation %q is not cacheable", q.query.Meter.Aggregation) + } +} + +// outerWindowColumns returns the windowstart/windowend SELECT expressions and +// their GROUP BY terms for the requested window size, mirroring meter_query.go +// EXACTLY. The input is the inner hourly windowstart (UTC DateTime). The gate +// guarantees a whole-hour timezone offset, so re-tumbling hourly rows into +// day/month windows in the target tz is exact. +// +// For WindowSize == nil (total) there is no window grouping; windowstart and +// windowend are placeholder From/To constants that QueryMeter overwrites. +func (q queryCachedMeter) outerWindowColumns() (selectCols []string, groupBy []string, err error) { + if q.query.WindowSize == nil { + // Total: emit From/To placeholders (overwritten by QueryMeter), no window + // grouping. + return []string{"toDateTime(?) AS windowstart", "toDateTime(?) AS windowend"}, nil, nil + } + + tz := "UTC" + if q.query.WindowTimeZone != nil { + tz = q.query.WindowTimeZone.String() + } + + // The window start/end are derived from windowstart_hourly (the inner hourly + // grain) and GROUP BY the derived windowstart so distinct output windows + // collapse. windowend is derived from the same tumbled start, mirroring + // meter_query.go exactly for each size. + switch *q.query.WindowSize { + case meterpkg.WindowSizeHour: + return []string{ + fmt.Sprintf("tumbleStart(windowstart_hourly, toIntervalHour(1), '%s') AS windowstart", tz), + fmt.Sprintf("tumbleEnd(windowstart_hourly, toIntervalHour(1), '%s') AS windowend", tz), + }, []string{"windowstart", "windowend"}, nil + + case meterpkg.WindowSizeDay: + return []string{ + fmt.Sprintf("tumbleStart(windowstart_hourly, toIntervalDay(1), '%s') AS windowstart", tz), + fmt.Sprintf("tumbleStart(windowstart_hourly, toIntervalDay(1), '%s') + toIntervalDay(1) AS windowend", tz), + }, []string{"windowstart", "windowend"}, nil + + case meterpkg.WindowSizeMonth: + return []string{ + fmt.Sprintf("toDateTime(tumbleStart(windowstart_hourly, toIntervalMonth(1), '%s'), '%s') AS windowstart", tz, tz), + fmt.Sprintf("toDateTime(tumbleEnd(windowstart_hourly, toIntervalMonth(1), '%s'), '%s') AS windowend", tz, tz), + }, []string{"windowstart", "windowend"}, nil + + case meterpkg.WindowSizeMinute: + // Minute windows cannot be reconstructed from hourly rollups; the gate + // routes these to the live path. + return nil, nil, fmt.Errorf("minute window size is not cacheable") + + default: + return nil, nil, fmt.Errorf("invalid window size type: %s", *q.query.WindowSize) + } +} + +// outerWindowArgs returns the args consumed by outerWindowColumns (the From/To +// placeholders for the total case, none otherwise). +func (q queryCachedMeter) outerWindowArgs() []interface{} { + if q.query.WindowSize != nil { + return nil + } + + // Total: placeholders overwritten by QueryMeter, but must be valid times. + from := q.cacheFrom() + to := q.freshTo() + return []interface{}{from.Unix(), to.Unix()} +} + +// scanRows scans the merge result into cached rows (exact decimals), then maps +// them to MeterQueryRow honoring the query's requested group-by subset. It +// mirrors meter_query.go's scanRows: first three columns are windowstart, +// windowend, value; the remaining columns are the group-by subset (subject / +// customer_id top-level, or a meter dimension). +func (q queryCachedMeter) scanRows(rows driver.Rows) ([]meterpkg.MeterQueryRow, error) { + values := []meterpkg.MeterQueryRow{} + + columns := rows.Columns() + if len(columns) < 3 { + return nil, fmt.Errorf("cache query returned %d columns, expected at least 3", len(columns)) + } + if columns[0] != "windowstart" { + return nil, fmt.Errorf("first column is not windowstart") + } + if columns[1] != "windowend" { + return nil, fmt.Errorf("second column is not windowend") + } + // value is last (SELECT appends it after the group columns). + valueIdx := len(columns) - 1 + if columns[valueIdx] != "value" { + return nil, fmt.Errorf("last column is not value") + } + + // Rows are collected as exact decimals; duplicate keys cannot occur (the + // outer GROUP BY emits one row per key, and the cache leg's argMax collapse + // deterministically resolves duplicate stored rollups before recombination). + var cached []cachedMeterRow + + for rows.Next() { + var ( + windowStart time.Time + windowEnd time.Time + // Reuse the live path's NullDecimal so COUNT (UInt64) and Decimal128 + // values scan through exactly the same code as meter_query.go. + value NullDecimal + ) + + // Group columns sit between windowend (idx 1) and value (valueIdx). + groupColumns := columns[2:valueIdx] + groupDest := make([]string, len(groupColumns)) + + dest := []interface{}{&windowStart, &windowEnd} + for i := range groupDest { + dest = append(dest, &groupDest[i]) + } + dest = append(dest, &value) + + if err := rows.Scan(dest...); err != nil { + return nil, fmt.Errorf("cache query row scan: %w", err) + } + + row := cachedMeterRow{ + WindowStart: windowStart, + WindowEnd: windowEnd, + GroupBy: append([]string(nil), groupDest...), + ValueValid: value.Valid, + } + if value.Valid { + row.Value = alpacadecimal.NewFromDecimal(value.Decimal) + } + cached = append(cached, row) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("cache query rows error: %w", err) + } + + groupColumns := columns[2:valueIdx] + for _, r := range cached { + // Skip null-value rows, matching the live scan. + if !r.ValueValid { + continue + } + + mrow := meterpkg.MeterQueryRow{ + WindowStart: r.WindowStart, + WindowEnd: r.WindowEnd, + GroupBy: map[string]*string{}, + // Match live scanRows: convert to float64 only at this boundary. + Value: r.Value.InexactFloat64(), + } + + // Attach group-by subset values by column name (mirrors the mapping in + // meter_query.go's scanRows: subject/customer_id are top-level). + for i, column := range groupColumns { + s := r.GroupBy[i] + switch { + case column == "subject": + mrow.Subject = &s + case column == "customer_id": + mrow.CustomerID = &s + case slices.Contains(q.query.GroupBy, column): + mrow.GroupBy[column] = &s + default: + return nil, fmt.Errorf("column %s is not a valid group by", column) + } + } + + values = append(values, mrow) + } + + return values, nil +} + +// filterStringWhere renders a filter.FilterString into a WHERE fragment and its +// args, using the connector's sqlbuilder for parameter placeholders. It mirrors +// how meter_query.go applies FilterString via SelectWhereExpr, but produces a +// standalone fragment usable inside the merge's outer WHERE. +func filterStringWhere(fs filter.FilterString, column string) (string, []interface{}) { + // Build against a throwaway select builder to reuse the filter's own `?` + // placeholder generation and positional args, then extract the fragment + // after WHERE. sqlbuilder.ClickHouse emits positional `?` placeholders whose + // args are returned in order, so splicing the fragment into the merge and + // appending its args in the same left-to-right order preserves binding. + sb := sqlbuilder.ClickHouse.NewSelectBuilder() + expr := fs.SelectWhereExpr(column, sb) + sb.Select("1").From("_").Where(expr) + built, args := sb.Build() + + _, frag, _ := strings.Cut(built, "WHERE ") + return frag, args +} diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_test.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_test.go new file mode 100644 index 0000000000..85bb08bbe4 --- /dev/null +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_test.go @@ -0,0 +1,113 @@ +package clickhouse + +import ( + "testing" + "time" + + "github.com/samber/lo" + "github.com/stretchr/testify/require" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" +) + +// TestMeterShapeHash pins the fingerprint that isolates cache rows per meter +// extraction shape. Any definition change that alters what a row's value or +// group_by array MEANS must change the hash (so old rows orphan instead of +// double-counting), while semantically irrelevant differences must not. +func TestMeterShapeHash(t *testing.T) { + base := meterpkg.Meter{ + Key: "m1", + EventType: "t", + Aggregation: meterpkg.MeterAggregationSum, + ValueProperty: lo.ToPtr("$.value"), + GroupBy: map[string]string{"model": "$.model", "region": "$.region"}, + } + + require.Equal(t, meterShapeHash(base), meterShapeHash(base), "hash must be deterministic") + + // Same content, different map construction order — Go map iteration order is + // random, so this also guards the sorted-key alignment. + reordered := base + reordered.GroupBy = map[string]string{"region": "$.region", "model": "$.model"} + require.Equal(t, meterShapeHash(base), meterShapeHash(reordered), "group-by map order must not change the hash") + + addedDim := base + addedDim.GroupBy = map[string]string{"model": "$.model", "region": "$.region", "tier": "$.tier"} + require.NotEqual(t, meterShapeHash(base), meterShapeHash(addedDim), "adding a group-by dimension must change the hash") + + changedPath := base + changedPath.GroupBy = map[string]string{"model": "$.model_id", "region": "$.region"} + require.NotEqual(t, meterShapeHash(base), meterShapeHash(changedPath), "changing a dimension's JSON path must change the hash") + + changedValue := base + changedValue.ValueProperty = lo.ToPtr("$.tokens") + require.NotEqual(t, meterShapeHash(base), meterShapeHash(changedValue), "changing the value property must change the hash") + + changedAgg := base + changedAgg.Aggregation = meterpkg.MeterAggregationMax + require.NotEqual(t, meterShapeHash(base), meterShapeHash(changedAgg), "changing the aggregation must change the hash") + + // A meter deleted and recreated under the same slug with a different event + // type must not inherit the old rows or coverage claim: the cache leg + // filters rows by type, so an inherited claim would vouch for rows the + // read can never see. + changedType := base + changedType.EventType = "t2" + require.NotEqual(t, meterShapeHash(base), meterShapeHash(changedType), "changing the event type must change the hash") +} + +func TestIsCacheableWindowSize(t *testing.T) { + ws := func(s meterpkg.WindowSize) *meterpkg.WindowSize { return &s } + + require.True(t, isCacheableWindowSize(nil), "total (nil) is cacheable") + require.True(t, isCacheableWindowSize(ws(meterpkg.WindowSizeHour))) + require.True(t, isCacheableWindowSize(ws(meterpkg.WindowSizeDay))) + require.True(t, isCacheableWindowSize(ws(meterpkg.WindowSizeMonth))) + require.False(t, isCacheableWindowSize(ws(meterpkg.WindowSizeMinute)), "minute is finer than the hourly grain") +} + +func TestIsWholeHourTimeZone(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.AddDate(0, 0, 30) + + require.True(t, isWholeHourTimeZone(nil, from, to), "nil zone (UTC) is whole-hour") + require.True(t, isWholeHourTimeZone(time.UTC, from, to), "UTC is whole-hour") + + // Whole-hour zone year-round. + ny, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + require.True(t, isWholeHourTimeZone(ny, from, to), "America/New_York is a whole-hour offset") + + // Fractional-hour zone — cannot be composed from hourly-UTC windows. + kolkata, err := time.LoadLocation("Asia/Kolkata") + require.NoError(t, err) + require.False(t, isWholeHourTimeZone(kolkata, from, to), "Asia/Kolkata (+05:30) is fractional-hour") + + kathmandu, err := time.LoadLocation("Asia/Kathmandu") + require.NoError(t, err) + require.False(t, isWholeHourTimeZone(kathmandu, from, to), "Asia/Kathmandu (+05:45) is fractional-hour") +} + +func TestAggCacheColumn(t *testing.T) { + cases := []struct { + agg meterpkg.MeterAggregation + column string + recombine string + ok bool + }{ + {meterpkg.MeterAggregationSum, "sum_value", "sum", true}, + {meterpkg.MeterAggregationCount, "count_value", "sum", true}, + {meterpkg.MeterAggregationMin, "min_value", "min", true}, + {meterpkg.MeterAggregationMax, "max_value", "max", true}, + {meterpkg.MeterAggregationAvg, "", "", false}, + {meterpkg.MeterAggregationUniqueCount, "", "", false}, + {meterpkg.MeterAggregationLatest, "", "", false}, + } + + for _, tc := range cases { + col, rec, ok := aggCacheColumn(tc.agg) + require.Equal(t, tc.ok, ok, "agg %s ok", tc.agg) + require.Equal(t, tc.column, col, "agg %s column", tc.agg) + require.Equal(t, tc.recombine, rec, "agg %s recombine", tc.agg) + } +} diff --git a/openmeter/streaming/clickhouse/zz_audit_test.go b/openmeter/streaming/clickhouse/zz_audit_test.go new file mode 100644 index 0000000000..0d77b321a9 --- /dev/null +++ b/openmeter/streaming/clickhouse/zz_audit_test.go @@ -0,0 +1,254 @@ +package clickhouse + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/ClickHouse/clickhouse-go/v2" + + meterpkg "github.com/openmeterio/openmeter/openmeter/meter" + "github.com/openmeterio/openmeter/pkg/filter" +) + +func auditMeter(agg meterpkg.MeterAggregation, groupBy map[string]string) meterpkg.Meter { + vp := "$.value" + m := meterpkg.Meter{ + Key: "my_meter", + Aggregation: agg, + EventType: "my_event", + ValueProperty: &vp, + GroupBy: groupBy, + } + return m +} + +func dumpShape(t *testing.T, name string, q queryCachedMeter) { + sql, args, err := q.toSQL() + if err != nil { + t.Fatalf("%s: %v", name, err) + } + nPlace := strings.Count(sql, "?") + fmt.Printf("\n===== %s =====\n", name) + fmt.Printf("placeholders=%d args=%d\n", nPlace, len(args)) + fmt.Printf("SQL:\n%s\n", sql) + fmt.Printf("ARGS: %#v\n", args) + if nPlace != len(args) { + t.Errorf("%s: PLACEHOLDER/ARG MISMATCH placeholders=%d args=%d", name, nPlace, len(args)) + } +} + +func TestAuditPlaceholderArgParity(t *testing.T) { + from := time.Date(2026, 3, 1, 0, 30, 0, 0, time.UTC) // mid-hour -> head leg present + to := time.Date(2026, 3, 5, 12, 30, 0, 0, time.UTC) // mid-hour -> tail leg present + ws := meterpkg.WindowSizeDay + + base := func(mod func(*queryMeter)) queryMeter { + q := queryMeter{ + Database: "openmeter", + EventsTableName: "om_events", + Namespace: "ns_A", + Meter: auditMeter(meterpkg.MeterAggregationSum, map[string]string{"region": "$.region"}), + From: &from, + To: &to, + WindowSize: &ws, + EnableDecimalPrecision: true, + } + if mod != nil { + mod(&q) + } + return q + } + + // Cutoff mid-range so cache leg + head + tail all present. + cutoff := time.Date(2026, 3, 4, 0, 0, 0, 0, time.UTC) + + // Shape 1: window=day, cache+head+tail, no filters + dumpShape(t, "day_cache_head_tail_nofilter", queryCachedMeter{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: base(nil), Cutoff: cutoff, + }) + + // Shape 2: with subject filter + dumpShape(t, "day_subject_filter", queryCachedMeter{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: base(func(q *queryMeter) { q.FilterSubject = []string{"s1", "s2"} }), Cutoff: cutoff, + }) + + // Shape 3: with group-by filter + dumpShape(t, "day_groupby_filter", queryCachedMeter{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: base(func(q *queryMeter) { + q.GroupBy = []string{"region"} + q.FilterGroupBy = map[string]filter.FilterString{ + "region": {Eq: strptr("us")}, + } + }), Cutoff: cutoff, + }) + + // Shape 4: total window (nil) - outer window args present + dumpShape(t, "total_window", queryCachedMeter{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: base(func(q *queryMeter) { q.WindowSize = nil }), Cutoff: cutoff, + }) + + // Shape 5: total window + subject filter + groupby filter (max placeholders) + dumpShape(t, "total_subject_and_groupby_filter", queryCachedMeter{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: base(func(q *queryMeter) { + q.WindowSize = nil + q.FilterSubject = []string{"s1"} + q.GroupBy = []string{"region"} + q.FilterGroupBy = map[string]filter.FilterString{"region": {Eq: strptr("us")}} + }), Cutoff: cutoff, + }) + + // Shape 6: from hour-aligned (no head), cutoff hour-aligned but < to (tail present) + fromAligned := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + dumpShape(t, "no_head_tail_present", queryCachedMeter{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: base(func(q *queryMeter) { q.From = &fromAligned }), Cutoff: cutoff, + }) + + // Shape 7: fully cached (cutoff >= to, floored) - cache only, no head/tail? head present (mid-hour from) + cutoffLate := to + dumpShape(t, "cutoff_at_to", queryCachedMeter{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: base(nil), Cutoff: cutoffLate, + }) +} + +func strptr(s string) *string { return &s } + +// TestAuditLiveTenantBinding executes the REAL generated merge SQL against live +// ClickHouse with positional args, proving the namespace placeholder binds the +// namespace value (no positional desync) and that a ns_A query never returns +// ns_B / other-type / other-meter / other-hash rows. +func TestAuditLiveTenantBinding(t *testing.T) { + dsn := os.Getenv("TEST_CLICKHOUSE_DSN") + if dsn == "" { + t.Skip("TEST_CLICKHOUSE_DSN not set") + } + opts, err := clickhouse.ParseDSN(dsn) + if err != nil { + t.Fatal(err) + } + conn, err := clickhouse.Open(opts) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx := t.Context() + + db := "openmeter" + // Build the total-window merge for ns_A over self-seeded cache rows: the + // target row for ns_A plus decoys differing ONLY in namespace, type, or + // meter_hash. With no matching events in om_events the live legs contribute + // nothing, so the result is exactly the cache leg's value for ns_A — any + // positional-arg desync or missing key filter surfaces as a decoy value. + from := time.Date(2026, 3, 1, 0, 30, 0, 0, time.UTC) + to := time.Date(2026, 3, 5, 12, 30, 0, 0, time.UTC) + cutoff := time.Date(2026, 3, 4, 0, 0, 0, 0, time.UTC) + + q := queryMeter{ + Database: db, + EventsTableName: "om_events", + Namespace: "ns_A", + Meter: auditMeter(meterpkg.MeterAggregationSum, map[string]string{"region": "$.region"}), + From: &from, + To: &to, + WindowSize: nil, // total + EnableDecimalPrecision: true, + } + hash := meterShapeHash(q.Meter) + t.Logf("meter_hash=%s", hash) + + // Seed: ensure the cache table exists, wipe this meter's slug, then insert + // the target row and the decoys under a windowstart inside [cacheLo, cacheHi). + if err := conn.Exec(ctx, createMeterQueryRowCacheTable{Database: db, TableName: meterQueryRowCacheTable}.toSQL()); err != nil { + t.Fatal(err) + } + table := getTableName(db, meterQueryRowCacheTable) + if err := conn.Exec(ctx, "DELETE FROM "+table+" WHERE meter_slug = 'my_meter'"); err != nil { + t.Fatal(err) + } + windowstart := time.Date(2026, 3, 1, 1, 0, 0, 0, time.UTC) // = ceil(from, 1h) + for _, seed := range []struct { + namespace string + eventType string + hash string + value int + }{ + {"ns_A", "my_event", hash, 111}, // the target + {"ns_B", "my_event", hash, 999}, // other tenant + {"ns_A", "other_event", hash, 555}, // other type + {"ns_A", "my_event", "deadbeef00000000", 333}, // other shape hash + } { + if err := conn.Exec(ctx, fmt.Sprintf( + `INSERT INTO %s (namespace, type, meter_slug, meter_hash, windowstart, subject, group_by, sum_value, count_value, min_value, max_value) + VALUES (?, ?, 'my_meter', ?, ?, 's1', ['us'], toDecimal128(%d, 19), 0, NULL, NULL)`, table, seed.value), + seed.namespace, seed.eventType, seed.hash, windowstart); err != nil { + t.Fatal(err) + } + } + + merge := queryCachedMeter{ + Database: db, CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: q, Cutoff: cutoff, + } + sql, args, err := merge.toSQL() + if err != nil { + t.Fatal(err) + } + t.Logf("ARGS: %#v", args) + + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + t.Fatalf("query: %v\nSQL:\n%s", err, sql) + } + defer rows.Close() + + var total float64 + n := 0 + for rows.Next() { + var ws, we time.Time + var v NullDecimal + if err := rows.Scan(&ws, &we, &v); err != nil { + t.Fatalf("scan: %v", err) + } + if v.Valid { + total += v.Decimal.InexactFloat64() + } + n++ + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + t.Logf("ns_A total=%v rows=%d", total, n) + if total != 111 { + t.Fatalf("TENANT LEAK or desync: expected ns_A value 111, got %v", total) + } +} + +func TestAuditPopulateArgs(t *testing.T) { + from := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + cutoff := time.Date(2026, 3, 4, 0, 0, 0, 0, time.UTC) + q := queryMeter{ + Database: "openmeter", EventsTableName: "om_events", Namespace: "ns_A", + Meter: auditMeter(meterpkg.MeterAggregationSum, map[string]string{"region": "$.region"}), + From: &from, + EnableDecimalPrecision: true, + } + p := populateMeterQueryRowCache{ + Database: "openmeter", CacheTableName: meterQueryRowCacheTable, EventsTableName: "om_events", + query: q, From: from, Cutoff: cutoff, + } + sql, args := p.toSQL() + nPlace := strings.Count(sql, "?") + fmt.Printf("\n===== POPULATE =====\nplaceholders=%d args=%d\nSQL:\n%s\nARGS: %#v\n", nPlace, len(args), sql, args) + if nPlace != len(args) { + t.Errorf("POPULATE PLACEHOLDER/ARG MISMATCH placeholders=%d args=%d", nPlace, len(args)) + } +} diff --git a/openmeter/streaming/query_params.go b/openmeter/streaming/query_params.go index 4584ef0560..07d30aa733 100644 --- a/openmeter/streaming/query_params.go +++ b/openmeter/streaming/query_params.go @@ -23,6 +23,9 @@ type QueryParams struct { GroupBy []string WindowSize *meter.WindowSize WindowTimeZone *time.Location + + // Cachable opts a query into the meter query-result cache. + Cachable bool } // Validate validates query params focusing on `from` and `to` being aligned with query and meter window sizes diff --git a/openmeter/streaming/testutils/streaming.go b/openmeter/streaming/testutils/streaming.go index b6632fdd61..3889c9c58f 100644 --- a/openmeter/streaming/testutils/streaming.go +++ b/openmeter/streaming/testutils/streaming.go @@ -5,6 +5,7 @@ import ( "fmt" "slices" "strings" + "sync" "testing" "time" @@ -34,11 +35,30 @@ type SimpleEvent struct { type MockStreamingConnector struct { rows map[string][]meter.MeterQueryRow events map[string][]SimpleEvent + + // recordedQueryParams captures the params of every QueryMeter call, in order, + // so tests can assert call-site behavior. + // Guarded by mu: production callers query in parallel. + mu sync.Mutex + recordedQueryParams []streaming.QueryParams } func (m *MockStreamingConnector) Reset() { m.rows = map[string][]meter.MeterQueryRow{} m.events = map[string][]SimpleEvent{} + + m.mu.Lock() + defer m.mu.Unlock() + m.recordedQueryParams = nil +} + +// RecordedQueryParams returns a copy of the params of every QueryMeter call so +// far, in call order. +func (m *MockStreamingConnector) RecordedQueryParams() []streaming.QueryParams { + m.mu.Lock() + defer m.mu.Unlock() + + return append([]streaming.QueryParams(nil), m.recordedQueryParams...) } type AddOption func(event *SimpleEvent) @@ -105,6 +125,10 @@ func (m *MockStreamingConnector) ListEventsV2(ctx context.Context, params stream // Returns the result query set for the given params. If the query set is not found, // it will try to approximate the result by aggregating the simple events func (m *MockStreamingConnector) QueryMeter(ctx context.Context, namespace string, mm meter.Meter, params streaming.QueryParams) ([]meter.MeterQueryRow, error) { + m.mu.Lock() + m.recordedQueryParams = append(m.recordedQueryParams, params) + m.mu.Unlock() + rows := []meter.MeterQueryRow{} _, rowOk := m.rows[mm.Key] From 72ae914a9061f6e6d6e6fdc0caae54966bb17d62 Mon Sep 17 00:00:00 2001 From: Andras Toth <4157749+tothandras@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:26:37 +0200 Subject: [PATCH 2/3] fix(streaming): make tenant-binding audit test hermetic TestAuditLiveTenantBinding assumed the openmeter database and om_events table exist (true on a dev machine, not in CI's fresh ClickHouse). The test now creates its own database and both tables through the production DDL builders, so it carries no environment dependency. Co-Authored-By: Claude Fable 5 --- .../streaming/clickhouse/zz_audit_test.go | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/openmeter/streaming/clickhouse/zz_audit_test.go b/openmeter/streaming/clickhouse/zz_audit_test.go index 0d77b321a9..06c28f53d6 100644 --- a/openmeter/streaming/clickhouse/zz_audit_test.go +++ b/openmeter/streaming/clickhouse/zz_audit_test.go @@ -142,7 +142,20 @@ func TestAuditLiveTenantBinding(t *testing.T) { defer conn.Close() ctx := t.Context() - db := "openmeter" + // Hermetic database: CI ClickHouse has no pre-existing openmeter schema, so + // the test creates everything it touches (events table for the live legs, + // cache table for the seeded rows) via the production DDL builders. + db := "zz_audit_tenant_binding" + if err := conn.Exec(ctx, "DROP DATABASE IF EXISTS "+db+" SYNC"); err != nil { + t.Fatal(err) + } + if err := conn.Exec(ctx, "CREATE DATABASE "+db); err != nil { + t.Fatal(err) + } + if err := conn.Exec(ctx, createEventsTable{Database: db, EventsTableName: "om_events"}.toSQL()); err != nil { + t.Fatal(err) + } + // Build the total-window merge for ns_A over self-seeded cache rows: the // target row for ns_A plus decoys differing ONLY in namespace, type, or // meter_hash. With no matching events in om_events the live legs contribute @@ -165,15 +178,12 @@ func TestAuditLiveTenantBinding(t *testing.T) { hash := meterShapeHash(q.Meter) t.Logf("meter_hash=%s", hash) - // Seed: ensure the cache table exists, wipe this meter's slug, then insert - // the target row and the decoys under a windowstart inside [cacheLo, cacheHi). + // Seed: create the cache table, then insert the target row and the decoys + // under a windowstart inside [cacheLo, cacheHi). if err := conn.Exec(ctx, createMeterQueryRowCacheTable{Database: db, TableName: meterQueryRowCacheTable}.toSQL()); err != nil { t.Fatal(err) } table := getTableName(db, meterQueryRowCacheTable) - if err := conn.Exec(ctx, "DELETE FROM "+table+" WHERE meter_slug = 'my_meter'"); err != nil { - t.Fatal(err) - } windowstart := time.Date(2026, 3, 1, 1, 0, 0, 0, time.UTC) // = ceil(from, 1h) for _, seed := range []struct { namespace string From 92f66ccf12a8d55f53f19ee49eadb2a0a9c373a0 Mon Sep 17 00:00:00 2001 From: Andras Toth <4157749+tothandras@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:53:33 +0200 Subject: [PATCH 3/3] fix(streaming): address query cache PR review findings CodeRabbit: - Post-merge invalidation guard: a namespace invalidation landing between a cached read's populate and its merge SELECT could blank the settled range for that one response (the marker only orders plans). The read now re-checks the marker after the merge and falls back to the live query when it moved. - Gate unknown GroupBy keys to the live path so they surface as the live path's validation error instead of a cached-path SQL-build error. - Parity mismatch logs name the differing fields instead of dumping raw row keys (subject/customer identifiers stay out of error logs). - Self-healing invalidation after a parity mismatch runs on its own bounded context instead of the shadow check's possibly-exhausted one. - Parity comparison keys distinguish nil from empty-string values (monitor and test comparator). Greptile: - ClickHouse UNKNOWN_TABLE detection uses the typed exception with a string fallback for transports that wrap server errors as text. - filterStringWhere fails loudly when the WHERE fragment cannot be extracted instead of silently dropping the filter. Co-Authored-By: Claude Fable 5 --- .../clickhouse/meterqueryrow_cache_exec.go | 86 +++++++++++++----- .../clickhouse/meterqueryrow_cache_gate.go | 13 +++ .../meterqueryrow_cache_integration_test.go | 60 +++++++++++++ .../clickhouse/meterqueryrow_cache_monitor.go | 90 ++++++++++++++++--- .../meterqueryrow_cache_parity_test.go | 26 +++--- .../clickhouse/meterqueryrow_cache_query.go | 19 +++- .../clickhouse/meterqueryrow_cache_test.go | 20 +++++ 7 files changed, 267 insertions(+), 47 deletions(-) diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go index c3e1c6624b..746a9ada3e 100644 --- a/openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_exec.go @@ -2,10 +2,12 @@ package clickhouse import ( "context" + "errors" "fmt" "strings" "time" + "github.com/ClickHouse/clickhouse-go/v2" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" @@ -14,6 +16,19 @@ import ( "github.com/openmeterio/openmeter/openmeter/streaming" ) +// isUnknownTableError reports whether err is ClickHouse error code 60 +// (UNKNOWN_TABLE). The typed exception covers the native protocol; the string +// fallback covers transports (e.g. HTTP) that surface server errors as plain +// text rather than a typed exception. +func isUnknownTableError(err error) bool { + var exception *clickhouse.Exception + if errors.As(err, &exception) { + return exception.Code == 60 + } + + return strings.Contains(strings.ToLower(err.Error()), "code: 60") +} + // queryMeterCached serves a cacheable meter query. It lazily rolls up the // settled whole-hour range into the cache table, then runs the merge query that // UNIONs the settled rollup with live scans of the sub-hour head and fresh tail. @@ -66,8 +81,27 @@ func (c *Connector) queryMeterCached(ctx context.Context, query queryMeter) ([]m "cutoff", cutoff, ) - if cacheLo.Before(cacheHi) { - plan := c.planCachedRangePopulation(ctx, query, cacheLo, cacheHi) + // planInvalidatedAt is the namespace's invalidation marker observed at plan + // time; the post-merge guard below compares against it. + var planInvalidatedAt time.Time + cacheLegUsed := cacheLo.Before(cacheHi) + + if cacheLegUsed { + // Plan-start is captured BEFORE the coverage read: it becomes the stored + // claim's PopulatedAt, so an invalidation marker landing at any later + // instant — including between our populates and our claim INSERT — makes + // the claim distrusted. A coverage read failure degrades to "no claim" + // (populate the whole range), which is always safe. + planStart := time.Now().UTC() + + coverage, invalidatedAt, covErr := c.readMeterQueryRowCacheCoverage(ctx, query) + if covErr != nil { + logger.Warn("failed to read meter query cache coverage, repopulating the full range", "error", covErr) + coverage = nil + } + planInvalidatedAt = invalidatedAt + + plan := planCachePopulation(cacheLo, cacheHi, coverage, invalidatedAt, planStart) span.SetAttributes(attribute.Int("populate_ranges", len(plan.Populate))) for _, r := range plan.Populate { @@ -122,6 +156,28 @@ func (c *Connector) queryMeterCached(ctx context.Context, query queryMeter) ([]m return nil, fmt.Errorf("scan cached query rows: %w", err) } + // In-flight invalidation guard: the marker orders PLANS, but this read's + // populate and merge are separate statements — an invalidation landing + // between them can delete rollup rows the merge was about to read (or the + // merge can read rows a late event just made stale), undercounting THIS + // response with no error. Re-checking the marker after the merge closes + // that window: if it moved since plan time, discard the merge and serve + // the full live query. Costs one tiny SELECT per cached read. + if cacheLegUsed { + invalidated, checkErr := c.invalidatedSince(ctx, query, planInvalidatedAt) + if checkErr != nil || invalidated { + if checkErr != nil { + logger.Warn("failed to verify invalidation marker after merge, falling back to live query", "error", checkErr) + } else { + logger.Info("namespace invalidated during cached read, falling back to live query") + } + c.queryCacheMetrics.recordQuery(ctx, "live_fallback") + span.SetAttributes(attribute.Bool("live_fallback", true)) + + return c.queryMeter(ctx, query) + } + } + c.queryCacheMetrics.recordQuery(ctx, "cached") span.SetAttributes(attribute.Int("rows", len(values))) @@ -130,25 +186,15 @@ func (c *Connector) queryMeterCached(ctx context.Context, query queryMeter) ([]m return values, nil } -// planCachedRangePopulation reads the stored coverage claim and the -// namespace's invalidation marker for the query's meter shape and plans which -// sub-ranges of [cacheLo, cacheHi) still need rolling up. The plan-start time -// is captured BEFORE the coverage read: it becomes the stored claim's -// PopulatedAt, so an invalidation marker landing at any later instant — -// including between our populates and our claim INSERT — makes the claim -// distrusted. A coverage read failure degrades to "no claim" (populate the -// whole range), which is always safe — never fail or under-populate the query -// because the metadata was unreadable. -func (c *Connector) planCachedRangePopulation(ctx context.Context, query queryMeter, cacheLo, cacheHi time.Time) cachePlan { - planStart := time.Now().UTC() - - coverage, invalidatedAt, err := c.readMeterQueryRowCacheCoverage(ctx, query) +// invalidatedSince reports whether the namespace's invalidation marker is +// newer than the one observed at plan time (zero = none was present). +func (c *Connector) invalidatedSince(ctx context.Context, query queryMeter, planInvalidatedAt time.Time) (bool, error) { + _, latest, err := c.readMeterQueryRowCacheCoverage(ctx, query) if err != nil { - c.config.Logger.Warn("failed to read meter query cache coverage, repopulating the full range", "error", err, "namespace", query.Namespace, "meter", query.Meter.Key) - coverage = nil + return false, err } - return planCachePopulation(cacheLo, cacheHi, coverage, invalidatedAt, planStart) + return latest.After(planInvalidatedAt), nil } // readMeterQueryRowCacheCoverage returns the meter shape's newest claim (nil @@ -305,7 +351,7 @@ func (c *Connector) invalidateMeterQueryRowCache(ctx context.Context, namespaces if err := c.config.ClickHouse.Exec(ctx, markerSQL, markerArgs...); err != nil { // A missing table (code 60) means nothing is cached: invalidation is // always-on even with the cache disabled. - if !strings.Contains(err.Error(), "code: 60") { + if !isUnknownTableError(err) { // Fall back to deleting the claims outright: that still kills every // COMMITTED claim (only a claim racing this exact invalidation could // survive, which is the pre-marker exposure). If the delete fails @@ -352,7 +398,7 @@ func (c *Connector) invalidateMeterQueryRowCache(ctx context.Context, namespaces {rollupSQL, rollupArgs}, } { if err := c.config.ClickHouse.Exec(ctx, cleanup.sql, cleanup.args...); err != nil { - if strings.Contains(err.Error(), "code: 60") { + if isUnknownTableError(err) { continue } diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go index f2dbae61a8..0f33eb3963 100644 --- a/openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_gate.go @@ -138,6 +138,19 @@ func (c *Connector) canQueryBeCached(namespace string, meter meterpkg.Meter, par return false } + // Every GroupBy key must be "subject" or a meter dimension, mirroring the + // live path's validation: an unknown key is a validation error (400) there, + // and routing it live preserves that instead of turning it into a + // cached-path SQL-build error (500). + for _, key := range params.GroupBy { + if key == "subject" { + continue + } + if _, ok := meter.GroupBy[key]; !ok { + return false + } + } + for key := range params.FilterGroupBy { if _, ok := meter.GroupBy[key]; !ok { return false diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go index a9f451742c..5d22fccb31 100644 --- a/openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_integration_test.go @@ -595,6 +595,20 @@ func (s *ConnectorTestSuite) TestQueryCacheGateRouting() { params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"customer_id"}}, want: false, }, + { + // The live path rejects unknown group-by keys with a validation error; + // routing to live preserves that instead of a cached-path SQL-build error. + name: "group by unknown key -> live", + meter: dimMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"nonexistent"}}, + want: false, + }, + { + name: "group by subject and meter dimension -> cacheable", + meter: dimMeter, + params: streaming.QueryParams{Cachable: true, From: &oldFrom, To: &oldTo, WindowSize: ptr(meterpkg.WindowSizeHour), GroupBy: []string{"subject", "region"}}, + want: true, + }, { name: "too-fresh range -> live", meter: sumMeter, @@ -1552,6 +1566,52 @@ func (s *ConnectorTestSuite) insertRawEventBypassingInvalidation(ctx context.Con namespace, ulid.Make().String(), eventType, subject, at, data, at, at)) } +// TestQueryCacheInvalidatedSince pins the post-merge in-flight guard's +// semantics: a marker landing after the plan-time observation must report +// invalidated (the merge may have read wiped or stale rows), while a marker +// the plan already saw must not. +func (s *ConnectorTestSuite) TestQueryCacheInvalidatedSince() { + if s.T().Skipped() { + return + } + ctx := s.T().Context() + s.enableCacheOnConnector() + + meter := s.newMeter(parityMeter{ + name: "invalidated_since_sum", eventType: "invalidated_since_event", valueProperty: ptr("$.value"), + aggregation: meterpkg.MeterAggregationSum, + }) + q := s.buildQueryMeter(meter, streaming.QueryParams{}, nil) + s.clearInvalidationMarkers(ctx) + + // No marker at plan time, none now: not invalidated. + invalidated, err := s.Connector.invalidatedSince(ctx, q, time.Time{}) + s.NoError(err) + s.False(invalidated, "no marker must not report invalidated") + + // A marker lands after a plan that saw none: invalidated. + s.NoError(s.Connector.invalidateMeterQueryRowCache(ctx, []string{namespace})) + invalidated, err = s.Connector.invalidatedSince(ctx, q, time.Time{}) + s.NoError(err) + s.True(invalidated, "a marker newer than the plan's view must report invalidated") + + // A plan that already observed the marker is not invalidated by it. + _, seen, err := s.Connector.readMeterQueryRowCacheCoverage(ctx, q) + s.NoError(err) + invalidated, err = s.Connector.invalidatedSince(ctx, q, seen) + s.NoError(err) + s.False(invalidated, "the marker the plan already saw must not report invalidated") + + // A second invalidation moves the marker past the previously observed one. + time.Sleep(2 * time.Millisecond) // created_at has millisecond resolution + s.NoError(s.Connector.invalidateMeterQueryRowCache(ctx, []string{namespace})) + invalidated, err = s.Connector.invalidatedSince(ctx, q, seen) + s.NoError(err) + s.True(invalidated, "a marker newer than the observed one must report invalidated") + + s.clearInvalidationMarkers(ctx) +} + // clearInvalidationMarkers removes the namespace's invalidation markers. // seedEvents ingests old events through BatchInsert, which legitimately // invalidates the namespace — and claims written within the clock-skew margin diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go index 2a14a6b7f8..dd465def5b 100644 --- a/openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_monitor.go @@ -22,6 +22,12 @@ import ( // the cache exists to avoid), off the request path. const parityCheckTimeout = 2 * time.Minute +// parityInvalidateTimeout bounds the self-healing namespace invalidation after +// a detected mismatch. It gets its own budget derived from a fresh context: the +// shadow context's remaining deadline may be nearly exhausted by the live +// query, and a canceled wipe would leave the diverged cache rows serving. +const parityInvalidateTimeout = 30 * time.Second + // queryCacheMetrics holds the OTel instruments that make the cache's health and // correctness observable in production. All methods are nil-receiver safe so a // connector constructed without a metric.Meter (tests, embedded use) pays @@ -228,7 +234,10 @@ func (c *Connector) verifyCachedResultParity(ctx context.Context, query queryMet span.SetStatus(codes.Error, "cache-served result diverged from live query") logger.Error("QUERY CACHE PARITY MISMATCH: cache-served result diverged from live query, invalidating namespace cache", "diff", diff) - if err := c.invalidateMeterQueryRowCache(ctx, []string{query.Namespace}); err != nil { + invalidateCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), parityInvalidateTimeout) + defer cancel() + + if err := c.invalidateMeterQueryRowCache(invalidateCtx, []string{query.Namespace}); err != nil { span.RecordError(err) logger.Error("failed to invalidate meter query cache after parity mismatch", "error", err) @@ -277,34 +286,91 @@ func compareMeterQueryRowSets(live, cached []meterpkg.MeterQueryRow) string { for i := range l { lk, ck := meterQueryRowKey(l[i]), meterQueryRowKey(c[i]) if lk != ck { - return fmt.Sprintf("row key differs: live=%q cached=%q", lk, ck) + return fmt.Sprintf("row key differs at sorted index %d: %s", i, describeRowKeyDiff(l[i], c[i])) } if math.Round(l[i].Value*1e6) != math.Round(c[i].Value*1e6) { - return fmt.Sprintf("value differs for %q: live=%v cached=%v", lk, l[i].Value, c[i].Value) + return fmt.Sprintf("value differs at sorted index %d (window %s): live=%v cached=%v", + i, l[i].WindowStart.UTC().Format(time.RFC3339), l[i].Value, c[i].Value) } } return "" } +// describeRowKeyDiff names WHICH key fields differ between two rows without +// reproducing their values: subject, customer and group-by values are user +// identifiers that must not land in error logs. Window bounds are logged (they +// locate the mismatch and carry no identity), everything else only by field +// name. +func describeRowKeyDiff(live, cached meterpkg.MeterQueryRow) string { + var fields []string + + if !live.WindowStart.Equal(cached.WindowStart) || !live.WindowEnd.Equal(cached.WindowEnd) { + fields = append(fields, fmt.Sprintf("window (live %s..%s, cached %s..%s)", + live.WindowStart.UTC().Format(time.RFC3339), live.WindowEnd.UTC().Format(time.RFC3339), + cached.WindowStart.UTC().Format(time.RFC3339), cached.WindowEnd.UTC().Format(time.RFC3339))) + } + if !equalOptionalString(live.Subject, cached.Subject) { + fields = append(fields, "subject") + } + if !equalOptionalString(live.CustomerID, cached.CustomerID) { + fields = append(fields, "customer_id") + } + + groupKeys := map[string]struct{}{} + for k := range live.GroupBy { + groupKeys[k] = struct{}{} + } + for k := range cached.GroupBy { + groupKeys[k] = struct{}{} + } + for k := range groupKeys { + if !equalOptionalString(live.GroupBy[k], cached.GroupBy[k]) { + fields = append(fields, "group_by."+k) + } + } + + sort.Strings(fields) + if len(fields) == 0 { + return "unknown field" + } + + return strings.Join(fields, ", ") +} + +func equalOptionalString(a, b *string) bool { + if a == nil || b == nil { + return a == b + } + + return *a == *b +} + // meterQueryRowKey is the full grouping tuple of a result row: window bounds, // subject, customer and every group-by dimension. Comparing on anything less -// would line up different groups' rows against each other. +// would line up different groups' rows against each other. Optional values are +// prefixed with a nil/value marker so that nil and "" — which differ in the +// API response — never collapse into the same key. func meterQueryRowKey(r meterpkg.MeterQueryRow) string { var b strings.Builder + writeOptional := func(v *string) { + if v == nil { + b.WriteByte('n') + } else { + b.WriteByte('v') + b.WriteString(*v) + } + } + b.WriteString(r.WindowStart.UTC().Format(time.RFC3339)) b.WriteByte(0) b.WriteString(r.WindowEnd.UTC().Format(time.RFC3339)) b.WriteByte(0) - if r.Subject != nil { - b.WriteString(*r.Subject) - } + writeOptional(r.Subject) b.WriteByte(0) - if r.CustomerID != nil { - b.WriteString(*r.CustomerID) - } + writeOptional(r.CustomerID) b.WriteByte(0) keys := make([]string, 0, len(r.GroupBy)) @@ -316,9 +382,7 @@ func meterQueryRowKey(r meterpkg.MeterQueryRow) string { for _, k := range keys { b.WriteString(k) b.WriteByte(1) - if v := r.GroupBy[k]; v != nil { - b.WriteString(*v) - } + writeOptional(r.GroupBy[k]) b.WriteByte(2) } diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go index 04feb411bb..71af1a787f 100644 --- a/openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_parity_test.go @@ -184,20 +184,28 @@ func (s *ConnectorTestSuite) buildQueryMeter(m meterpkg.Meter, params streaming. } } -// rowKey is the full parity key: window + windowend + subject + customer + group-by. +// rowKey is the full parity key: window + windowend + subject + customer + +// group-by. Optional values carry a nil/value marker so nil and "" — distinct +// in the API response — never compare equal. func rowKey(r meterpkg.MeterQueryRow) string { var b strings.Builder + + writeOptional := func(v *string) { + if v == nil { + b.WriteByte('n') + } else { + b.WriteByte('v') + b.WriteString(*v) + } + } + b.WriteString(r.WindowStart.UTC().Format(time.RFC3339)) b.WriteByte(0) b.WriteString(r.WindowEnd.UTC().Format(time.RFC3339)) b.WriteByte(0) - if r.Subject != nil { - b.WriteString(*r.Subject) - } + writeOptional(r.Subject) b.WriteByte(0) - if r.CustomerID != nil { - b.WriteString(*r.CustomerID) - } + writeOptional(r.CustomerID) b.WriteByte(0) keys := make([]string, 0, len(r.GroupBy)) @@ -208,9 +216,7 @@ func rowKey(r meterpkg.MeterQueryRow) string { for _, k := range keys { b.WriteString(k) b.WriteByte(1) - if v := r.GroupBy[k]; v != nil { - b.WriteString(*v) - } + writeOptional(r.GroupBy[k]) b.WriteByte(2) } return b.String() diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_query.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_query.go index 6c8c5262e5..b089af8074 100644 --- a/openmeter/streaming/clickhouse/meterqueryrow_cache_query.go +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_query.go @@ -240,7 +240,10 @@ func (q queryCachedMeter) toSQL() (string, []interface{}, error) { column = fmt.Sprintf("group_by[%d]", idx+1) } - expr, exprArgs := filterStringWhere(fs, column) + expr, exprArgs, err := filterStringWhere(fs, column) + if err != nil { + return "", nil, fmt.Errorf("render filter for group by %s: %w", key, err) + } whereClauses = append(whereClauses, expr) whereArgs = append(whereArgs, exprArgs...) } @@ -527,7 +530,7 @@ func (q queryCachedMeter) scanRows(rows driver.Rows) ([]meterpkg.MeterQueryRow, // args, using the connector's sqlbuilder for parameter placeholders. It mirrors // how meter_query.go applies FilterString via SelectWhereExpr, but produces a // standalone fragment usable inside the merge's outer WHERE. -func filterStringWhere(fs filter.FilterString, column string) (string, []interface{}) { +func filterStringWhere(fs filter.FilterString, column string) (string, []interface{}, error) { // Build against a throwaway select builder to reuse the filter's own `?` // placeholder generation and positional args, then extract the fragment // after WHERE. sqlbuilder.ClickHouse emits positional `?` placeholders whose @@ -538,6 +541,14 @@ func filterStringWhere(fs filter.FilterString, column string) (string, []interfa sb.Select("1").From("_").Where(expr) built, args := sb.Build() - _, frag, _ := strings.Cut(built, "WHERE ") - return frag, args + // The extraction depends on sqlbuilder's output format ("... WHERE "). + // If a library change ever breaks that assumption, the query must fail + // loudly: silently splicing an empty fragment would DROP the filter and + // return unfiltered (over-counted) results. + _, frag, found := strings.Cut(built, "WHERE ") + if !found || frag == "" { + return "", nil, fmt.Errorf("no WHERE fragment in sqlbuilder output %q", built) + } + + return frag, args, nil } diff --git a/openmeter/streaming/clickhouse/meterqueryrow_cache_test.go b/openmeter/streaming/clickhouse/meterqueryrow_cache_test.go index 85bb08bbe4..8c6ca30451 100644 --- a/openmeter/streaming/clickhouse/meterqueryrow_cache_test.go +++ b/openmeter/streaming/clickhouse/meterqueryrow_cache_test.go @@ -56,6 +56,26 @@ func TestMeterShapeHash(t *testing.T) { require.NotEqual(t, meterShapeHash(base), meterShapeHash(changedType), "changing the event type must change the hash") } +// TestMeterQueryRowKeyDistinguishesNilFromEmpty guards the parity comparison +// key: nil and "" differ in the API response, so a cached/live regression that +// flips one into the other must not slip past the shadow check. +func TestMeterQueryRowKeyDistinguishesNilFromEmpty(t *testing.T) { + base := meterpkg.MeterQueryRow{ + WindowStart: time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC), + WindowEnd: time.Date(2026, 5, 2, 0, 0, 0, 0, time.UTC), + } + + emptySubject := base + emptySubject.Subject = lo.ToPtr("") + require.NotEqual(t, meterQueryRowKey(base), meterQueryRowKey(emptySubject), "nil and empty subject must key differently") + + nilGroup := base + nilGroup.GroupBy = map[string]*string{"model": nil} + emptyGroup := base + emptyGroup.GroupBy = map[string]*string{"model": lo.ToPtr("")} + require.NotEqual(t, meterQueryRowKey(nilGroup), meterQueryRowKey(emptyGroup), "nil and empty group-by values must key differently") +} + func TestIsCacheableWindowSize(t *testing.T) { ws := func(s meterpkg.WindowSize) *meterpkg.WindowSize { return &s }