Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion e2e_cleanup/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.26.0
require (
cloud.google.com/go/bigquery v1.74.0
cloud.google.com/go/pubsub/v2 v2.3.0
github.com/snowflakedb/gosnowflake/v2 v2.0.2
github.com/snowflakedb/gosnowflake/v2 v2.1.0
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78
google.golang.org/api v0.279.0
)
Expand Down
4 changes: 2 additions & 2 deletions e2e_cleanup/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -211,8 +211,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/snowflakedb/gosnowflake/v2 v2.0.2 h1:8UZo+v1T2Y9sgoPk3JYT3RatAUd9o6q6yjL40TyHluA=
github.com/snowflakedb/gosnowflake/v2 v2.0.2/go.mod h1:c0hIqJ/dxgaMl7g1o8n4Ca3Mf5YCiiVx9igio/PNqC8=
github.com/snowflakedb/gosnowflake/v2 v2.1.0 h1:rfjs6NAMnbLKCBYlOarqQX/UKgQVrXi43TZNHCP5/jw=
github.com/snowflakedb/gosnowflake/v2 v2.1.0/go.mod h1:c0hIqJ/dxgaMl7g1o8n4Ca3Mf5YCiiVx9igio/PNqC8=
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down
128 changes: 65 additions & 63 deletions flow/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ require (
cloud.google.com/go/kms v1.31.0
cloud.google.com/go/pubsub/v2 v2.3.0
cloud.google.com/go/storage v1.62.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/v2 v2.0.2
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.3.0
github.com/ClickHouse/ch-go v0.72.0
github.com/ClickHouse/ch-go v0.73.0
github.com/ClickHouse/clickhouse-go/v2 v2.46.0
github.com/PeerDB-io/glua64 v1.0.1
github.com/PeerDB-io/gluabit32 v1.0.2
Expand All @@ -22,20 +22,20 @@ require (
github.com/PeerDB-io/peerdb/flow/pkg v0.0.0
github.com/Shopify/toxiproxy/v2 v2.12.0
github.com/apache/arrow-go/v18 v18.6.0
github.com/aws/aws-sdk-go-v2 v1.41.7
github.com/aws/aws-sdk-go-v2/config v1.32.18
github.com/aws/aws-sdk-go-v2/credentials v1.19.17
github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.23
github.com/aws/aws-sdk-go-v2 v1.42.0
github.com/aws/aws-sdk-go-v2/config v1.32.25
github.com/aws/aws-sdk-go-v2/credentials v1.19.24
github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.29
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.0 // PINNED(DBI-500): v1.22.0 deprecates NewUploader in favor of feature/s3/transfermanager
github.com/aws/aws-sdk-go-v2/service/kms v1.52.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/aws/aws-sdk-go-v2/service/ses v1.34.24
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1
github.com/aws/smithy-go v1.25.1
github.com/cockroachdb/pebble/v2 v2.1.5
github.com/aws/aws-sdk-go-v2/service/kms v1.53.4
github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0
github.com/aws/aws-sdk-go-v2/service/ses v1.35.2
github.com/aws/aws-sdk-go-v2/service/sts v1.43.3
github.com/aws/smithy-go v1.27.2
github.com/cockroachdb/pebble/v2 v2.1.6
github.com/elastic/go-elasticsearch/v8 v8.19.6
github.com/go-logr/logr v1.4.3
github.com/go-mysql-org/go-mysql v1.15.1-0.20260622021642-60bfbfea8dd0
github.com/go-mysql-org/go-mysql v1.15.1-0.20260701093637-35ca5c6ee8c8
github.com/gogo/googleapis v1.4.1
github.com/google/uuid v1.6.0
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3
Expand All @@ -46,7 +46,7 @@ require (
github.com/jackc/pgx/v5 v5.10.0
github.com/joho/godotenv v1.5.1
github.com/json-iterator/go v1.1.12
github.com/lestrrat-go/httprc/v3 v3.0.5
github.com/lestrrat-go/httprc/v3 v3.0.6
github.com/lestrrat-go/jwx/v3 v3.1.1
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee
github.com/nickbruun/pgsplit v0.0.0-20240103043353-43e6c2dddfad
Expand All @@ -58,43 +58,43 @@ require (
github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17
github.com/quasilyte/go-ruleguard/dsl v0.3.23
github.com/shopspring/decimal v1.4.0
github.com/slack-go/slack v0.24.0
github.com/snowflakedb/gosnowflake/v2 v2.0.2
github.com/slack-go/slack v0.26.0
github.com/snowflakedb/gosnowflake/v2 v2.1.0
github.com/stretchr/testify v1.11.1
github.com/testcontainers/testcontainers-go v0.42.0
github.com/twmb/franz-go v1.21.2
github.com/testcontainers/testcontainers-go v0.43.0
github.com/twmb/franz-go v1.21.4
github.com/twmb/franz-go/pkg/kadm v1.18.0
github.com/twmb/franz-go/plugin/kslog v1.0.0
github.com/twpayne/go-geos v0.20.4
github.com/urfave/cli/v3 v3.9.0
github.com/urfave/cli/v3 v3.10.0
github.com/xdg-go/scram v1.2.0
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78
github.com/yuin/gopher-lua v1.1.2
go.mongodb.org/mongo-driver/v2 v2.6.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0
go.opentelemetry.io/otel v1.43.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0
go.opentelemetry.io/otel/metric v1.43.0
go.opentelemetry.io/otel/sdk v1.43.0
go.opentelemetry.io/otel/sdk/metric v1.43.0
go.opentelemetry.io/otel/trace v1.43.0
go.temporal.io/api v1.62.12
go.temporal.io/sdk v1.44.0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel v1.43.0 → v1.44.0 — new default cardinality limit of 2000 per metric instrument.

Starting in v1.44.0, the OTel metric SDK enforces a default cardinality limit of 2000 unique attribute sets per instrument. Attribute sets beyond this are dropped into an overflow bucket and silently lost.

PeerDB's metrics use per-flow and per-table attributes (flowName, sourcePeerName, destinationPeerName, destinationTableName — see otel_metrics/attributes.go). For deployments with many flows × many tables, the cross-product of these attributes could exceed 2000 per instrument (e.g. RecordsSyncedPerTableCounter), causing silent metric data loss.

Currently no WithCardinalityLimit is configured in otel_metrics/otel_manager.go. If the previous unlimited behavior is desired, add sdkmetric.WithCardinalityLimit(0) to the NewMeterProvider call, or set an explicit higher limit.

Other OTel v1.44.0 changes: OTLP exporters now enforce a 64 MiB request size limit (pre-compression) — unlikely to matter. Value.Emit deprecated in favor of Value.String. Go 1.23 support dropped (not relevant — PeerDB uses Go 1.26).

go.mongodb.org/mongo-driver/v2 v2.7.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Informational] otelgrpc v0.69.0 upgraded semantic conventions to v1.40.0, which removes the following metrics and span events:

  • rpc.client.request.size, rpc.client.response.size, rpc.server.request.size, rpc.server.response.size — no longer emitted
  • network.protocol.name, network.protocol.version, network.transport attributes removed from RPC spans/metrics
  • rpc.message span events no longer emitted
  • Bucket boundaries for rpc.server.call.duration and rpc.client.call.duration were corrected

PeerDB uses otelgrpc.NewServerHandler() (not the removed deprecated interceptors), so no code breakage. However, if any observability dashboards or alerts depend on the removed gRPC metrics, they will need updating.

Also removes deprecated functions Extract, Inject, StreamServerInterceptor, WithSpanOptions — PeerDB doesn't use these (confirmed at flow/cmd/api.go:238).

go.opentelemetry.io/otel v1.44.0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ OTel v1.44.0 introduces a default metric cardinality limit of 2000 per instrument.

PeerDB metrics use high-cardinality attributes (FlowName × DestinationTableName × SourcePeerName × RecordOperationType etc.). For deployments with many flows and tables, the unique attribute-set count per instrument could exceed 2000, causing silent data loss into an overflow bucket.

The meter provider in flow/otel_metrics/otel_manager.go:710 does not set an explicit WithCardinalityLimit. Consider adding one if the default 2000 is too low, or verify that per-instrument cardinality stays within bounds for your largest deployments.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Informational] OpenTelemetry v1.44.0 introduces a default metrics cardinality limit of 2000 per instrument. Attribute-set combinations beyond this are dropped into an overflow bucket.

PeerDB attaches per-flow attributes to metrics (flow name, source/destination peer name, peer type, hostname, status, etc.). For most deployments this should be fine, but if a deployment has 2000+ active flows, metrics could be silently dropped.

If this becomes an issue, setting WithCardinalityLimit(0) on the metric reader restores unlimited behavior. No action needed now -- just flagging the behavioral change.

Also note: WithSpanOptions was removed from otelgrpc v0.69.0, but PeerDB does not use it (verified via grep), so no compile risk.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Informational — OTel v1.44.0 introduces a default cardinality limit of 2000 per metric instrument.

Previously unlimited, the Metrics SDK now drops attribute sets exceeding 2000 unique combinations per instrument, aggregating overflow under otel.metric.overflow=true. This is a silent behavior change — no compile-time or runtime error, just quietly dropped metric dimensions.

PeerDB's metric attributes (flow name, source/dest peer, status, operation, activity type, etc.) are bounded by the number of active flows × operations, which should comfortably stay under 2000 for typical deployments. However, for large-scale deployments with many concurrent flows, this is worth keeping in mind.

To restore the old (unlimited) behavior if needed: WithCardinalityLimit(0) on the metric view.

No action required — just flagging for awareness since this is a default behavior change from upstream.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Info (not blocking): OTel v1.44.0 introduces a default cardinality limit of 2000 per metric instrument. Attribute sets beyond this limit are silently dropped and aggregated under an otel.metric.overflow=true marker. PeerDB does not explicitly configure a cardinality limit in flow/otel_metrics/otel_manager.go. This is almost certainly fine -- PeerDB metrics use bounded attribute sets (peer names, table names, flow names). But if any metric has high-cardinality attributes (e.g. per-partition keys), data could be silently dropped.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel SDK v1.44.0 introduces a default metrics cardinality limit of 2000 (previously unlimited). Once a metric instrument hits 2000 unique attribute-set combinations, new attribute sets are silently dropped and aggregated into a special overflow set tagged otel.metric.overflow=true.

PeerDB metrics use per-flow attributes (FlowName, SourcePeerName, DestinationPeerName, PipeName, TemporalActivityType, etc. in flow/otel_metrics/observables.go:115-152). The total cardinality per metric instrument scales with flows x pipes x operations x activity_types.

For most deployments this should be fine (2000 is generous), but large deployments with hundreds of mirrors could silently lose metric data.

To restore unlimited cardinality if needed: sdkmetric.WithCardinalityLimit(0) when creating the MeterProvider (in otel_manager.go:710).

Not a blocker -- the new default follows the OTel spec and is a safety improvement -- but worth being aware of.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ OpenTelemetry v1.44.0 introduces a default cardinality limit of 2000 per instrument.

Previously the limit was unlimited. When exceeded, excess attribute sets are silently dropped and aggregated into an otel.metric.overflow=true series.

PeerDB's NewMeterProvider in otel_metrics/otel_manager.go does not set WithCardinalityLimit, so the new default of 2000 applies. Metrics are emitted with per-flow, per-peer, per-activity attributes — large deployments with many mirrors could potentially approach this limit.

To restore unlimited behavior: pass sdkmetric.WithCardinalityLimit(0) to NewMeterProvider, or set OTEL_GO_X_CARDINALITY_LIMIT=0.

Recommendation: This is likely fine for most deployments, but worth being aware of. Consider adding WithCardinalityLimit(0) if you expect high-cardinality deployments, or monitor for the otel.metric.overflow attribute in your metrics backend after merging.

Additionally, this bump includes two security fixes:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Medium] OTel v1.44.0 introduces a default metric cardinality limit of 2000 per metric instrument. Attribute sets beyond this limit are silently dropped into an overflow bucket.

PeerDB emits metrics with per-flow attributes (FlowName, SourcePeerName, DestinationPeerName, FlowStatus, etc. -- see flow/otel_metrics/observables.go). For most deployments this should be well under 2000, but large installations with many mirrors could potentially hit this ceiling.

PeerDB does not currently configure WithCardinalityLimit, so the new default applies. If concerned, the limit can be raised or disabled via WithCardinalityLimit(0) on the metric reader.

Also notable: this release patches two security advisories (file handle leak in schema ParseFile, baggage header log flooding) and deprecates Value.Emit() in favor of Value.String().

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel v1.44.0 introduces a default metric cardinality limit of 2000.

Previously unlimited, sdk/metric now drops new attribute sets beyond 2000 unique combinations per instrument, aggregating overflow under otel.metric.overflow=true. PeerDB's metrics use per-flow attributes (flow name, source/destination peer name/type/hostname, status, pipe name, operation, activity type, workflow type) — deployments with many concurrent flows could exceed this limit and silently lose metric granularity.

Additionally, OTLP exporters now impose a 64 MiB default request size limit (before compression); oversized requests become non-retryable errors.

Recommendation: This is likely fine for most deployments, but worth being aware of. If metric cardinality issues surface post-upgrade, the limit can be raised via WithCardinalityLimit() or set to 0 (unlimited).

go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0
go.opentelemetry.io/otel/metric v1.44.0
go.opentelemetry.io/otel/sdk v1.44.0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Warning: go.opentelemetry.io/otel/sdk/metric v1.44.0 introduces a default cardinality limit of 2000 per instrument. When the limit is reached, new attribute sets are silently dropped into an overflow bucket. PeerDB defines 40+ metric instruments in flow/otel_metrics/otel_manager.go, some keyed by dynamic attributes (per-table, per-flow). If any single instrument exceeds 2000 unique attribute combinations, metrics will be silently lost. For most deployments this is probably fine (2000 is generous), but worth verifying for large multi-tenant deployments. If needed, override with sdkmetric.WithCardinalityLimit(0) in setupMetricsAndProvider().

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test comment

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note (informational, non-blocking): OpenTelemetry SDK v1.44.0 introduces a default cardinality limit of 2000 for the metrics SDK. When more than 2000 unique attribute sets are recorded for a single metric instrument, new attribute sets are dropped and aggregated into an overflow bucket. Previously there was no limit. If PeerDB has any high-cardinality metrics (e.g. metrics tagged per-table, per-peer, or per-flow), this could silently truncate metric data. To restore old behavior use WithCardinalityLimit(0). Also: attribute.Value.Emit() is deprecated in favor of Value.String(); security fixes for schema file handle leak and baggage extraction DoS; new default 64 MiB OTLP request size limit. All other bumps are clean - see review summary comment for details.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test inline

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel SDK v1.44.0 introduces a default 2000 cardinality limit.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel SDK v1.44.0 introduces a default 2000 cardinality limit on metrics.

The metrics SDK now enforces a default cap of 2000 unique attribute-set combinations per metric instrument. Excess sets are silently dropped into an overflow bucket.

PeerDB does not currently configure any cardinality limits (verified via grep). If any custom OTel metrics exceed 2000 unique attribute combinations, metric data will be silently dropped.

Worth verifying for high-cardinality dimensions (per-table, per-mirror, per-partition metrics). If unlimited cardinality is needed, pass WithCardinalityLimit(0) when creating the meter provider.

Also: Value.Emit is deprecated in favor of Value.String.

go.opentelemetry.io/otel/sdk/metric v1.44.0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Informational — OTel v1.44.0 introduces a default cardinality limit of 2000 per metric instrument.

Previously unlimited, new attribute sets beyond 2000 unique combinations per instrument are now dropped and aggregated under otel.metric.overflow=true. PeerDB uses attributes like DestinationTableName, FlowName, PeerName, and SlotName on its metrics. For typical deployments this should be well under 2000 per instrument, but deployments with a very large number of mirrors/tables could hit this.

To restore the old unlimited behavior if needed: metric.WithCardinalityLimit(0) or env OTEL_GO_X_CARDINALITY_LIMIT=0.

Also: OTLP exporters now cap request size at 64 MiB (configurable via WithMaxRequestSize). Oversized requests fail as non-retryable errors.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Informational — OTel v1.44.0 default cardinality limit]

This bump introduces a new default cardinality limit of 2,000 attribute sets per metric instrument (changelog). Excess attribute sets are dropped into an overflow bucket tagged otel.metric.overflow=true.

PeerDB's RecordsSyncedPerTableCounter / RecordsSyncedPerTableGauge use DestinationTableNameKey × RecordOperationTypeKey as attributes, so cardinality ≈ num_tables × 3. Deployments replicating ~667+ destination tables through a single flow could silently lose per-table metric granularity.

Additionally, OTLP exporters now enforce a 64 MiB default request size limit — oversized payloads become non-retryable errors.

Both limits are configurable (WithCardinalityLimit() on the metric view, WithMaxRequestSize() on the exporter). No action needed if table counts stay well under 667 per flow.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Informational] OTel SDK v1.44.0 introduces a default cardinality limit of 2000 per metric instrument (per the OTel spec). When exceeded, new attribute sets are silently dropped and aggregated into an overflow bucket with otel.metric.overflow=true.

PeerDB metrics use per-flow, per-peer, and per-table attributes (e.g. FlowName, SourcePeerName, DestinationTableNameKey in otel_metrics/observables.go). For deployments with many mirrors/tables, some instruments could theoretically exceed 2000 unique attribute combinations.

Impact: Likely fine for most deployments, but large-scale deployments should monitor for otel.metric.overflow attributes. To restore unlimited behavior, add sdkmetric.WithCardinalityLimit(0) to the meter provider config in otel_metrics/otel_manager.go:713.

Additionally, OTel v1.44.0 adds a default 64 MiB OTLP request size limit (configurable via WithMaxRequestSize). Unlikely to be hit under normal conditions.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ OTel v1.44.0 introduces a default cardinality limit of 2000 per metric instrument.

PeerDB's RecordsSyncedPerTableGauge and RecordsSyncedPerTableCounter (defined in otel_metrics/otel_manager.go) use DestinationTableName × RecordOperationType × contextual flow attributes. For deployments with many flows/tables, unique attribute combinations could exceed 2000, causing metrics to silently overflow.

If needed, add sdkmetric.WithCardinalityLimit(0) to the NewMeterProvider call at otel_metrics/otel_manager.go:710, or set a higher custom limit.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel v1.44.0 introduces a default cardinality limit of 2000 attribute combinations per instrument. When this limit is hit, new attribute sets are silently dropped and measurements aggregate into an overflow set.

PeerDB metrics use high-dimensional attributes (flow name, source/destination peer info, table name, activity type, etc.). For metrics like records_synced_per_table, the cardinality is flows x tables x other attributes. Deployments with many active flows and tables could plausibly exceed 2000 unique combinations on a single instrument.

This is unlikely to be an issue for most deployments, but worth being aware of. If you see metric data points going missing after this upgrade, you can configure sdkmetric.WithCardinalityLimit(0) on the MeterProvider to restore unlimited cardinality (as was the default before v1.44.0).

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Informational] OTel v1.44.0 introduces a default metric cardinality limit of 2000 per instrument. When exceeded, new attribute sets are silently dropped and aggregated into an overflow bucket.

PeerDB metrics use attributes like flowName, peerName, sourcePeerName, destinationPeerName, flowStatus, operation, activityType, and workflowType. For most deployments the unique combinations per instrument should be well under 2000, but deployments with hundreds of concurrent flows could approach this limit.

If this becomes a concern, the limit can be raised with WithCardinalityLimit() on individual instruments, or set to 0 for unlimited. Worth keeping in mind for observability -- hitting this limit would silently drop metric dimensions with no error.

All other bumps in this PR checked out clean:

  • AWS SDK Go v2 (v1.41.7 to v1.41.12) + smithy-go (v1.25.1 to v1.27.2): Patch-level fixes, ioutil cleanup, security improvement (credential file 0600 perms). smithy-go serialization regressions were fixed in v1.27.1/v1.27.2.
  • gosnowflake (v2.0.2 to v2.1.0): No breaking changes. Improved context-aware stream cancellation.
  • slack-go (v0.24.0 to v0.25.0): Breaking TableBlock.Rows type change does not affect PeerDB (only uses SendMessageContext + header/section blocks).
  • pebble (v2.1.5 to v2.1.6): Single IsLowerBound boundary fix -- unlikely to affect PeerDB.
  • temporal SDK (v1.44.0 to v1.44.1): Improved graceful worker shutdown -- beneficial.
  • temporal API (v1.62.12 to v1.62.13): Minor fix + Nexus definitions.
  • golang.org/x/sync, golang.org/x/net, genproto: Routine updates.
  • No PII/secret logging concerns found in any of the bumped packages.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel SDK metric v1.44.0 introduces a default cardinality limit of 2000 per instrument (release notes).

PeerDB uses ContextAwareInt64SyncGauge wrappers that inject contextual attributes (flow name, source/dest peer info, flow status, operation type, temporal activity info) on top of explicit attributes. The RecordsSyncedPerTableGauge additionally includes DestinationTableName and RecordOperationType. In deployments with many flows and tables, the unique attribute-set count per instrument could exceed 2000 (e.g. 100 flows x 10 tables x 3 ops = 3000 combinations).

When the limit is exceeded, new attribute sets are silently dropped into an overflow bucket (otel.metric.overflow=true), meaning per-table metrics would go missing without any error.

Mitigation options (no action required to merge, but worth tracking):

  • Set WithCardinalityLimit(0) on the metric reader to restore unlimited behavior
  • Set env var OTEL_GO_X_CARDINALITY_LIMIT=0 at deployment
  • Or accept the 2000 default if deployments are small enough

Also note: attribute.Value.Emit() is deprecated in favor of Value.String() -- PeerDB does not appear to call Emit() directly, so no action needed there.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Behavioral change in OTel SDK v1.44.0: The default metric cardinality limit is now 2000 per instrument (previously unlimited). When this limit is reached, new attribute sets are silently dropped.

PeerDB metrics use per-flow attributes (FlowName, SourcePeerName, DestinationPeerName, sourcePeerHostname, destinationPeerHostname, FlowStatus, FlowOperation, PipeName, etc. — see flow/otel_metrics/observables.go:115-141). In large deployments with many active mirrors, the cardinality per metric could approach or exceed 2000, causing silent metric loss.

Currently setupMetricsAndProvider in otel_manager.go:710 does not set an explicit WithCardinalityLimit. Worth considering whether to add sdkmetric.WithCardinalityLimit(0) (unlimited, preserving prior behavior) or a higher explicit limit, especially if large deployments are expected.

Upstream reference: OTel Go v1.44.0 release notes

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel v1.44.0 introduces a default cardinality limit of 2000 per instrument.

PeerDB emits per-table metrics (records_synced_per_table, records_synced_per_table_counter) with attributes FlowName × DestinationTableName × RecordOperationTypeKey. In large deployments with many flows and tables, the number of unique attribute combinations could approach or exceed 2000, at which point excess attribute sets would be silently dropped.

This is unlikely to bite most deployments (e.g. 200 flows × 3 tables × 3 ops = 1800, still under the limit), but worth being aware of. If needed, the limit can be raised via WithCardinalityLimit() in the sdkmetric.View.

Additionally, OTLP exporters now enforce a 64 MiB default request size limit (non-retryable error if exceeded).

Not a blocker — just flagging for awareness.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTel sdk/metric v1.44.0 — new default cardinality limit of 2000

This version introduces a default cardinality limit of 2000 unique attribute sets per instrument. Previously, cardinality was unlimited.

PeerDB metrics include high-cardinality attributes (flow name, source/destination peer type+name+hostname, flow status, pipe name, operation type, temporal activity details) and per-table metrics like records_synced_per_table. For deployments with many mirrors and tables, the number of unique attribute combinations per instrument could approach or exceed 2000, causing new data points to be silently dropped.

If this is a concern, you can restore the previous behavior by passing sdkmetric.WithView(sdkmetric.NewView(sdkmetric.Instrument{}, sdkmetric.Stream{CardinalityLimit: 0})) to the MeterProvider in otel_manager.go, or set a higher explicit limit.

Also notable: this release includes two security fixes — a file descriptor leak in schema ParseFile (GHSA-995v-fvrw-c78m) and log flooding from malformed baggage headers (GHSA-5wrp-cwcj-q835).

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[informational] OTel v1.43.0 to v1.44.0 introduces a default cardinality limit of 2000 per metric instrument in the metrics SDK. Attribute sets beyond this limit are aggregated into an overflow bucket (otel.metric.overflow=true).

PeerDB attaches per-flow attributes (flow name, source/destination peer names, table names, activity/workflow types) to metrics. For typical deployments this is well within the limit, but very large deployments with hundreds of flows and many tables per flow could hit it -- metrics would silently lose granularity.

This release also includes two security fixes: GHSA-995v-fvrw-c78m (schema file descriptor leaks) and GHSA-5wrp-cwcj-q835 (baggage extraction error flooding, now rate-limited).

No action required now, but worth keeping in mind. If needed, the limit can be raised with WithCardinalityLimit(0) on individual views.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[info] OTel SDK metric v1.44.0 introduces a default cardinality limit of 2000 per metric instrument. When exceeded, new attribute sets are dropped and aggregated into an otel.metric.overflow=true bucket.

PeerDB's RecordsSyncedPerTableGauge and RecordsSyncedPerTableCounter metrics emit per-table attributes (destinationTableName × recordOperationType × flow-level context attributes). For large deployments with many flows and tables, the number of unique attribute combinations could approach this limit.

Workaround if needed: Set env var OTEL_GO_X_CARDINALITY_LIMIT=0 to disable the limit, or call sdkmetric.WithCardinalityLimit(0) in the meter provider setup.

Also notable in this release:

  • Two security fixes: GHSA-995v-fvrw-c78m (schema file descriptor leak in ParseFile) and GHSA-5wrp-cwcj-q835 (baggage header flooding — extraction now enforces 8192-byte spec limit).
  • OTLP exporters now enforce a 64 MiB request size limit by default (oversized requests become non-retryable errors).
  • Value.Emit deprecated in favor of Value.String.
  • Fixed race condition in FixedSizeReservoir exemplar and Prometheus exporter concurrent Collect data race.

go.opentelemetry.io/otel/trace v1.44.0
go.temporal.io/api v1.63.0
go.temporal.io/sdk v1.45.0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[informational] Temporal SDK v1.44.0 to v1.45.0 has several behavioral changes:

  1. gRPC gzip compression is now enabled by default on client connections. Generally beneficial (reduces bandwidth), but could cause issues with proxies/interceptors that do not expect compressed payloads. Disable with client.GrpcCompressionNone if needed.

  2. Nil/unset search attributes are now omitted from workflow start/signal-with-start/continue-as-new calls (previously persisted as binary/null). PeerDB always sets search attributes to non-nil string values via shared.NewSearchAttributes(), so this should not affect current usage.

  3. PollerStartCounter metric is no longer emitted for autoscaling pollers -- dashboards/alerts using it will break.

  4. Two SDK behavior-fix flags (SDKFlagBlockedSelectorSignalReceive, SDKFlagWorkflowNewChannelLostMessages) are now enabled by default for new workflow executions -- these fix lost-message bugs but could surface latent issues.

No action required, but worth monitoring after rollout.

go.temporal.io/sdk/contrib/opentelemetry v0.7.0
go.uber.org/automaxprocs v1.6.0
golang.org/x/crypto v0.52.0
golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a
golang.org/x/mod v0.36.0
golang.org/x/sync v0.20.0
golang.org/x/tools v0.45.0
golang.org/x/crypto v0.53.0
golang.org/x/exp v0.0.0-20260611194520-c48552f49976
golang.org/x/mod v0.37.0
golang.org/x/sync v0.21.0
golang.org/x/tools v0.46.0
google.golang.org/api v0.279.0
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa
google.golang.org/grpc v1.81.1
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
k8s.io/apimachinery v0.36.1
k8s.io/apimachinery v0.36.2
k8s.io/client-go v0.35.3 // Note: v0.* are newer than v1.*
)

Expand All @@ -109,12 +109,12 @@ require (
filippo.io/edwards25519 v1.2.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect
github.com/Azure/go-amqp v1.5.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
github.com/BurntSushi/toml v1.6.0 // indirect
github.com/DataDog/zstd v1.5.7 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
Expand All @@ -127,18 +127,18 @@ require (
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/apache/arrow/go/v15 v15.0.2 // indirect
github.com/apache/thrift v0.22.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
Expand Down Expand Up @@ -232,7 +232,7 @@ require (
github.com/minio/minlz v1.1.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.2.0 // indirect
github.com/moby/moby/api v1.54.1 // indirect
github.com/moby/moby/api v1.54.2 // indirect
github.com/moby/moby/client v0.4.0 // indirect
github.com/moby/patternmatcher v0.6.1 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
Expand All @@ -248,7 +248,7 @@ require (
github.com/opentracing/basictracer-go v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/paulmach/orb v0.13.0 // indirect
github.com/pierrec/lz4/v4 v4.1.26 // indirect
github.com/pierrec/lz4/v4 v4.1.27 // indirect
github.com/pingcap/failpoint v0.0.0-20260406204437-bbc9d102c19e // indirect
github.com/pingcap/kvproto v0.0.0-20260331120830-0d407c8b3f6e // indirect
github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 // indirect
Expand All @@ -267,7 +267,7 @@ require (
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/segmentio/asm v1.2.1 // indirect
github.com/shirou/gopsutil/v3 v3.24.5 // indirect
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
github.com/shirou/gopsutil/v4 v4.26.5 // indirect
github.com/shoenig/go-m1cpu v0.2.1 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
Expand All @@ -292,19 +292,19 @@ require (
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.54.0 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 // indirect
golang.org/x/term v0.43.0 // indirect
golang.org/x/text v0.37.0
golang.org/x/sys v0.46.0 // indirect
golang.org/x/telemetry v0.0.0-20260610154732-fb80ec83bdd9 // indirect
golang.org/x/term v0.44.0 // indirect
golang.org/x/text v0.38.0
golang.org/x/time v0.15.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
Expand All @@ -322,6 +322,8 @@ require (
sigs.k8s.io/yaml v1.6.0 // indirect
)

require github.com/nexus-rpc/nexus-proto-annotations v0.1.0 // indirect

replace github.com/tikv/client-go/v2 => github.com/PeerDB-io/tikv-client-go/v2 v2.0.7

replace github.com/tikv/pd/client => github.com/tikv/pd/client v0.0.0-20251229071808-6173d50c004c // PINNED(DBI-444): the underlying dependency changed an interface
Expand Down
Loading
Loading