diff --git a/cmd/tools/integration/packages.json b/cmd/tools/integration/packages.json index 43a07f1187..4772ad5c41 100644 --- a/cmd/tools/integration/packages.json +++ b/cmd/tools/integration/packages.json @@ -16,6 +16,7 @@ {"path":"./internal/impl/couchbase"}, {"path":"./internal/impl/cyborgdb","skip":true}, {"path":"./internal/impl/cypher","skip":true}, + {"path":"./internal/impl/db2","timeout":"30m"}, {"path":"./internal/impl/elasticsearch/v8"}, {"path":"./internal/impl/elasticsearch/v9"}, {"path":"./internal/impl/gcp"}, diff --git a/docs/modules/components/pages/inputs/db2_cdc.adoc b/docs/modules/components/pages/inputs/db2_cdc.adoc new file mode 100644 index 0000000000..1b04fc0f1e --- /dev/null +++ b/docs/modules/components/pages/inputs/db2_cdc.adoc @@ -0,0 +1,575 @@ += db2_cdc +:type: input +:status: beta +:categories: ["Services"] + + + +//// + THIS FILE IS AUTOGENERATED! + + To make changes, edit the corresponding source file under: + + https://github.com/redpanda-data/connect/tree/main/internal/impl/. + + And: + + https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl +//// + +// © 2026 Redpanda Data Inc. + + +component_type_dropdown::[] + + +Consumes change data capture (CDC) events from IBM DB2 using SQL Replication. + +Introduced in version 4.92.0. + + +[tabs] +====== +Common:: ++ +-- + +```yml +# Common config fields, showing default values +input: + label: "" + db2_cdc: + dsn: '!!!SECRET_SCRUBBED!!!' # No default (required) + schema: DB2ADMIN # No default (required) + tables: [] # No default (optional) + snapshot_mode: initial + auto_replay_nacks: true + batching: + count: 0 + byte_size: 0 + period: "" + check: "" +``` + +-- +Advanced:: ++ +-- + +```yml +# All config fields, showing default values +input: + label: "" + db2_cdc: + dsn: '!!!SECRET_SCRUBBED!!!' # No default (required) + schema: DB2ADMIN # No default (required) + tables: [] # No default (optional) + table_include_regex: [] # No default (optional) + table_exclude_regex: [] # No default (optional) + cdc_schema: ASNCDC + snapshot_mode: initial + snapshot_max_batch_size: 1024 + snapshot_parallelism: 1 + snapshot_isolation_mode: repeatable_read + checkpoint_mode: auto + checkpoint_cache: my_redis_cache # No default (optional) + checkpoint_cache_key: db2_cdc_checkpoint + checkpoint_cache_table_name: RPCN.CDC_CHECKPOINT + checkpoint_limit: 4096 + stream_backoff_interval: 1s + poll_batch_size: 2048 + heartbeat_interval: 30s + emit_schema_changes: true + signal_table: "" # No default (optional) + auto_replay_nacks: true + batching: + count: 0 + byte_size: 0 + period: "" + check: "" + processors: [] # No default (optional) +``` + +-- +====== + +Streams change events from IBM DB2 tables using SQL Replication (the ASNCDC schema). + +This connector uses *pure Go* with `github.com/ebitengine/purego` for DB2 CLI bindings — no CGO or C toolchain required at build time. + +== How it works + +DB2 SQL Replication maintains a set of *change tables* (also called CD tables) alongside your source tables. A capture daemon reads the DB2 transaction log and writes INSERT, UPDATE, and DELETE operations into those tables. This input polls those change tables and converts each row into a Redpanda Connect message. + +The connector operates in two phases: + +1. *Snapshot* (when `snapshot_mode: initial` or `always`): reads all existing rows from each monitored table inside a single `REPEATABLE READ` read-only transaction. The CDC position (CSN) is captured *before* the first row is read, so no changes are missed while the snapshot is in progress. +2. *Streaming*: polls each change table for new rows with `IBMSNAP_COMMITSEQ` greater than the last checkpoint. The maximum `SYNCHPOINT` from `ASNCDC.IBMSNAP_REGISTER` is used as an upper bound on every poll to avoid reading uncommitted rows. + +== UPDATE representation + +DB2 LUW SQL Replication encodes UPDATE operations as a DELETE record followed by an INSERT record that share the same `IBMSNAP_COMMITSEQ` value. This connector detects these pairs using LEAD/LAG window functions (ported from the Debezium DB2 connector) and merges them into a single `op: u` (update) event. The `before` field contains the old row data and `after` contains the new row data. + +== Message format (Debezium-compatible) + +The message body matches the https://debezium.io/documentation/reference/stable/connectors/db2.html[Debezium DB2 connector^] envelope format, making this connector a drop-in replacement for existing Debezium consumers. + +```json +{ + "before": null, + "after": { "ID": 42, "NAME": "Alice", "SALARY": 75000 }, + "source": { + "connector": "redpanda.db2", + "name": "redpanda.db2_cdc", + "schema": "DB2ADMIN", + "table": "EMPLOYEES", + "commit_lsn": "CSN:000000000000C350", + "change_lsn": null, + "snapshot": "false", + "ts_ms": 1705315845000 + }, + "op": "c", + "ts_ms": 1705315845000 +} +``` + +**`op` codes** match Debezium: `c` = create/insert, `u` = update, `d` = delete, `r` = read (snapshot), `hb` = heartbeat, `schema_change` = schema change. + +**`before` field**: Populated per Debezium semantics. For `op: u` (update) events, `before` contains the old row data and `after` contains the new row data. For `op: d` (delete) events, `before` contains the deleted row and `after` is `null`. For `op: c` (insert) and `op: r` (snapshot read) events, `before` is `null`. + +**`change_lsn`**: DB2 LUW SQL Replication exposes only `IBMSNAP_COMMITSEQ` (the commit LSN). The intra-transaction LSN (`change_lsn` in Debezium) is not available and is always `null`. + +The following metadata keys are set on every message: + +- `db2_schema`: source table schema +- `db2_table`: source table name +- `db2_operation`: human-readable op — `read`, `insert`, `update`, `delete`, `heartbeat`, or `schema_change` +- `db2_op`: Debezium op code — `r`, `c`, `u`, `d`, `hb`, or `schema_change` +- `db2_csn`: commit sequence number string (empty for snapshot events; backward-compat alias for `db2_commit_lsn`) +- `db2_commit_lsn`: same as `db2_csn` (Debezium field name) +- `db2_connector`: always `redpanda.db2` +- `db2_snapshot`: `true` for snapshot rows, `false` for streaming rows +- `db2_timestamp`: `IBMSNAP_LOGMARKER` timestamp from the change table (RFC3339Nano; omitted for snapshot events) + +== Prerequisites + +1. IBM DB2 10.1 or later with SQL Replication installed. +2. The DB2 CLI shared library must be present at runtime: + - Linux: `libdb2.so.1` (from the IBM Data Server Driver package) + - macOS: `libdb2.dylib` + - Windows: `db2cli.dll` + +=== Installing on Debian/Ubuntu + +Download the IBM Data Server Driver Package (dsdriver) from the IBM support site and run: + +```sh +tar xzf ibm_data_server_driver_package_linuxx64.tar.gz +cd dsdriver && bash installDSDriver +export LD_LIBRARY_PATH=/opt/ibm/dsdriver/lib:$LD_LIBRARY_PATH +``` + +=== Installing on RHEL/CentOS + +```sh +rpm -ivh ibm-datasrvrmgr-*.rpm +export LD_LIBRARY_PATH=/opt/ibm/dsdriver/lib:$LD_LIBRARY_PATH +``` + +=== Kubernetes init container + +```yaml +initContainers: + - name: install-db2-driver + image: ibmcom/db2:11.5.9.0 + command: ["/bin/sh", "-c", "cp /opt/ibm/db2/V11.5/lib64/libdb2.so.1 /shared/lib/"] + volumeMounts: + - name: db2-lib + mountPath: /shared/lib +volumes: + - name: db2-lib + emptyDir: {} +``` + +3. SQL Replication must be configured and the capture daemon running: +```sql +-- Start the CDC capture daemon +CALL ASNCDC.ASNCDCSERVICES('start', 'asncdc'); + +-- Register each table you want to capture (repeat for each table) +CALL ASNCDC.ADDTABLE('MYSCHEMA', 'EMPLOYEES'); +CALL ASNCDC.ADDTABLE('MYSCHEMA', 'ORDERS'); +``` + +== Minimal configuration + +```yaml +input: + db2_cdc: + dsn: "DATABASE=SAMPLE;HOSTNAME=localhost;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" + schema: "DB2ADMIN" + tables: ["EMPLOYEES", "ORDERS"] + # snapshot_mode defaults to "initial": snapshot on first start, stream continuously thereafter. + # Set to "never" to skip the snapshot and stream from the current log position only. +``` + +== Checkpoint persistence + +The connector tracks the highest processed `IBMSNAP_COMMITSEQ` so it can resume after a restart without replaying already-delivered events. By default a `RPCN.CDC_CHECKPOINT` table is created in DB2 (the `RPCN` schema must already exist). Set `checkpoint_cache` to use an external https://www.docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] instead. + + +== Fields + +=== `dsn` + +DB2 connection string in the keyword=value format accepted by SQLDriverConnect. Common keywords: `DATABASE` (database alias), `HOSTNAME`, `PORT` (default 50000), `PROTOCOL` (`TCPIP`), `UID`, `PWD`. The DB2 CLI shared library must be installed separately — this connector does not bundle it. +[CAUTION] +==== +This field contains sensitive information that usually shouldn't be added to a config directly, read our xref:configuration:secrets.adoc[secrets page for more info]. +==== + + + +*Type*: `string` + + +```yml +# Examples + +dsn: DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret +``` + +=== `schema` + +DB2 schema (TABSCHEMA) that owns the monitored tables. Maps to SOURCE_OWNER in ASNCDC.IBMSNAP_REGISTER. Case-insensitive — automatically normalized to uppercase. + + +*Type*: `string` + + +```yml +# Examples + +schema: DB2ADMIN +``` + +=== `tables` + +List of table names (without schema prefix) to capture changes from. Each table must already be registered with SQL Replication via ASNCDC.ADDTABLE before the connector starts. The connector will fail at startup if any listed table is missing from ASNCDC.IBMSNAP_REGISTER. When empty, all CDC-registered tables in the schema are discovered dynamically from ASNCDC.IBMSNAP_REGISTER. + + +*Type*: `array` + + +```yml +# Examples + +tables: + - EMPLOYEES + - ORDERS +``` + +=== `table_include_regex` + +Optional list of regular expressions; only tables whose names match at least one pattern are captured. Applied after `tables`. When `tables` is empty, all CDC-registered tables in the schema are discovered first and this filter narrows the set. Patterns are matched against the bare table name (without schema prefix). + + +*Type*: `array` + + +```yml +# Examples + +table_include_regex: + - ^EMP + - ^ORDER +``` + +=== `table_exclude_regex` + +Optional list of regular expressions; tables whose names match any pattern are excluded from capture. Applied after `table_include_regex`. + + +*Type*: `array` + + +```yml +# Examples + +table_exclude_regex: + - _TEST$ + - _STAGING$ +``` + +=== `cdc_schema` + +Schema that owns the SQL Replication control tables (IBMSNAP_REGISTER and the generated change tables). Defaults to `ASNCDC`, which is the standard installation schema. Change this only if SQL Replication was installed under a custom schema. + + +*Type*: `string` + +*Default*: `"ASNCDC"` + +=== `snapshot_mode` + +Controls when an initial full-table snapshot is performed: + +- `initial` (default): snapshot only on first start (when no checkpoint exists). Skipped on restart. +- `always`: snapshot on every start, regardless of existing checkpoint. +- `never`: skip snapshot; start streaming from the current log position. +- `initial_only`: snapshot then stop (no streaming phase). +- `when_needed`: snapshot if and only if no prior checkpoint exists; equivalent to `initial` for fresh deployments but useful when the checkpoint table may have been wiped. +- `no_data`: Debezium-compatible alias for `never`; no rows are read during snapshot. + + +*Type*: `string` + +*Default*: `"initial"` + +Options: +`initial` +, `always` +, `never` +, `initial_only` +, `when_needed` +, `no_data` +. + +=== `snapshot_max_batch_size` + +Number of rows fetched per round-trip during the initial snapshot. Rows are retrieved using keyset pagination (ordered by primary key), so reducing this value lowers memory usage at the cost of more DB2 round-trips. Increase for large tables with small rows to improve snapshot throughput. + + +*Type*: `int` + +*Default*: `1024` + +=== `snapshot_parallelism` + +Number of tables to snapshot concurrently during the initial snapshot phase. Currently fixed at 1 (sequential): all workers share a single read-only transaction and the underlying db2cli driver uses a 1-connection pool, so concurrency > 1 would serialize anyway and risk statement-handle data races. Values greater than 1 are accepted and stored but have no effect until per-worker connection support is added in a future release. + + +*Type*: `int` + +*Default*: `1` + +=== `snapshot_isolation_mode` + +SQL transaction isolation level used during the snapshot phase. `repeatable_read` (default) uses DB2 Repeatable Read (RR) — no phantom reads, strongest consistency. `read_committed` uses DB2 Cursor Stability (CS) — lower lock overhead but snapshot consistency is not guaranteed across tables. + + +*Type*: `string` + +*Default*: `"repeatable_read"` + +Options: +`repeatable_read` +, `read_committed` +. + +=== `checkpoint_mode` + +Checkpoint tracking mode. `auto` (default) detects the DB2 version and selects CSN-based checkpointing on DB2 10.1+. Set to `csn` to require CSN mode explicitly and fail fast on older DB2 versions. + + +*Type*: `string` + +*Default*: `"auto"` + +=== `checkpoint_cache` + +Name of a https://www.docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] to use for storing the checkpoint CSN. If not set, the connector automatically creates and manages a `checkpoint_cache_table_name` table in DB2. Use an external cache (e.g. Redis) when you want checkpoint state to survive a DB2 rebuild or when the connector user lacks DDL privileges. + + +*Type*: `string` + + +```yml +# Examples + +checkpoint_cache: my_redis_cache +``` + +=== `checkpoint_cache_key` + +Key used to store and retrieve the checkpoint in the cache or DB2 checkpoint table. Change this value if you run multiple DB2 CDC connectors that share the same cache resource or the same DB2 instance, to avoid checkpoint collisions. + + +*Type*: `string` + +*Default*: `"db2_cdc_checkpoint"` + +=== `checkpoint_cache_table_name` + +Fully-qualified DB2 table name for the built-in checkpoint store. The table is created automatically if it does not exist (SQLSTATE 42710 is silently ignored). The connector will attempt to CREATE the schema if it does not already exist; if the connector user lacks CREATE SCHEMA privileges, the schema must be created manually beforehand. Only used when `checkpoint_cache` is not set. + + +*Type*: `string` + +*Default*: `"RPCN.CDC_CHECKPOINT"` + +=== `checkpoint_limit` + +Maximum number of messages that can be in-flight (sent but not yet acknowledged) at any one time. The CSN checkpoint is only advanced once all messages at a given position are acknowledged, preserving at-least-once delivery guarantees. Increase this value to allow larger output batches and higher throughput; decrease it to reduce the number of re-delivered messages after a restart. + + +*Type*: `int` + +*Default*: `4096` + +=== `stream_backoff_interval` + +How long to wait between change-table polls when no new rows are found. Note that the DB2 capture daemon itself introduces latency — reducing this below the capture daemon's commit interval has no benefit. Increase for low-traffic tables to reduce unnecessary DB2 queries. + + +*Type*: `string` + +*Default*: `"1s"` + +=== `poll_batch_size` + +Maximum number of change rows fetched per change-table query. Each poll issues one query per monitored table. Reduce this value if individual change events are large (e.g. wide rows). Increase it to improve throughput for high-velocity tables. + + +*Type*: `int` + +*Default*: `2048` + +=== `heartbeat_interval` + +Interval at which heartbeat messages (`op=hb`) are emitted when no CDC changes are available. Heartbeats keep downstream consumers alive on low-traffic tables and advance the checkpoint on idle connectors. The default of 30s matches the Debezium DB2 connector recommended setting. Set to 0s to disable. + + +*Type*: `string` + +*Default*: `"30s"` + +=== `emit_schema_changes` + +When true, emit a schema change event (`op=schema_change`) whenever a new table is added to SQL Replication (i.e., when `ASNCDC.ADDTABLE` is called while the connector is running). The connector polls `ASNCDC.IBMSNAP_REGISTER` for new entries on each backoff interval. Defaults to true to match Debezium behaviour; set to false to suppress schema change events. + + +*Type*: `bool` + +*Default*: `true` + +=== `signal_table` + +Fully-qualified DB2 table name for the signal channel (e.g. `DB2INST1.CDC_SIGNALS`). When set, the connector polls this table for `execute-snapshot` signals. An `execute-snapshot` signal triggers an ad-hoc snapshot of the specified tables without restarting the connector. Create the table with: `CREATE TABLE . (ID VARCHAR(255) NOT NULL, TYPE VARCHAR(64) NOT NULL, DATA VARCHAR(2048), PRIMARY KEY (ID))` + + +*Type*: `string` + + +=== `auto_replay_nacks` + +Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. + + +*Type*: `bool` + +*Default*: `true` + +=== `batching` + +Allows you to configure a xref:configuration:batching.adoc[batching policy]. + + +*Type*: `object` + + +```yml +# Examples + +batching: + byte_size: 5000 + count: 0 + period: 1s + +batching: + count: 10 + period: 1s + +batching: + check: this.contains("END BATCH") + count: 0 + period: 1m +``` + +=== `batching.count` + +A number of messages at which the batch should be flushed. If `0` disables count based batching. + + +*Type*: `int` + +*Default*: `0` + +=== `batching.byte_size` + +An amount of bytes at which the batch should be flushed. If `0` disables size based batching. + + +*Type*: `int` + +*Default*: `0` + +=== `batching.period` + +A period in which an incomplete batch should be flushed regardless of its size. + + +*Type*: `string` + +*Default*: `""` + +```yml +# Examples + +period: 1s + +period: 1m + +period: 500ms +``` + +=== `batching.check` + +A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch. + + +*Type*: `string` + +*Default*: `""` + +```yml +# Examples + +check: this.type == "end_of_transaction" +``` + +=== `batching.processors` + +A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op. + + +*Type*: `array` + + +```yml +# Examples + +processors: + - archive: + format: concatenate + +processors: + - archive: + format: lines + +processors: + - archive: + format: json_array +``` + + diff --git a/internal/impl/db2/cdc_unit_test.go b/internal/impl/db2/cdc_unit_test.go new file mode 100644 index 0000000000..0bf8d7746e --- /dev/null +++ b/internal/impl/db2/cdc_unit_test.go @@ -0,0 +1,1453 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2 + +import ( + "context" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "testing" + "time" + + "github.com/Jeffail/checkpoint" + "github.com/Jeffail/shutdown" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + + "golang.org/x/text/encoding/unicode" + + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2cli" + "github.com/redpanda-data/connect/v4/internal/impl/db2/replication" +) + +// testLogger returns a *service.Logger suitable for unit tests. +func testLogger(t *testing.T) *service.Logger { + t.Helper() + return service.MockResources().Logger() +} + +// --------------------------------------------------------------------------- +// isAlreadyExistsError +// --------------------------------------------------------------------------- + +func TestIsAlreadyExistsError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + want bool + }{ + {"nil error", nil, false}, + {"SQLSTATE=42710", fmt.Errorf("sql error SQLSTATE=42710 object already exists"), true}, + {"SQLSTATE 42710", fmt.Errorf("DB2 SQL Error: SQLCODE=-601, SQLSTATE 42710"), true}, + {"unrelated error", fmt.Errorf("connection refused"), false}, + // bare "42710" without SQLSTATE prefix must not match (too broad) + {"bare 42710 substring", fmt.Errorf("some message 42710 embedded"), false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, isAlreadyExistsError(tc.err)) + }) + } +} + +// --------------------------------------------------------------------------- +// determineCheckpointMode +// --------------------------------------------------------------------------- + +func TestDetermineCheckpointMode(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + mode string + version replication.Version + wantMode string + wantErr bool + errSubstr string + }{ + { + name: "auto with 11.5 → csn", + mode: "auto", + version: replication.Version{Major: 11, Minor: 5}, + wantMode: "csn", + }, + { + name: "auto with 10.1 → csn", + mode: "auto", + version: replication.Version{Major: 10, Minor: 1}, + wantMode: "csn", + }, + { + name: "auto with 9.7 → error", + mode: "auto", + version: replication.Version{Major: 9, Minor: 7}, + wantErr: true, + errSubstr: "does not support CSN", + }, + { + name: "explicit csn with 11.5 → csn", + mode: "csn", + version: replication.Version{Major: 11, Minor: 5}, + wantMode: "csn", + }, + { + name: "explicit csn with 9.7 → error", + mode: "csn", + version: replication.Version{Major: 9, Minor: 7}, + wantErr: true, + errSubstr: "requires DB2 10.1+", + }, + { + name: "unknown mode → error", + mode: "timestamp", + version: replication.Version{Major: 11, Minor: 5}, + wantErr: true, + errSubstr: "invalid checkpoint_mode", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + d := &db2CDCInput{ + checkpointMode: tc.mode, + version: tc.version, + log: testLogger(t), + } + + mode, err := d.determineCheckpointMode() + if tc.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errSubstr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantMode, mode) + }) + } +} + +// --------------------------------------------------------------------------- +// eventToMessage +// --------------------------------------------------------------------------- + +func TestEventToMessage(t *testing.T) { + t.Parallel() + + ts := time.Date(2024, 1, 15, 10, 30, 45, 0, time.UTC) + + tests := []struct { + name string + event replication.ChangeEvent + wantMeta map[string]string + }{ + { + name: "snapshot read event", + event: replication.ChangeEvent{ + Schema: "DB2ADMIN", + Table: "EMPLOYEES", + Operation: replication.OpTypeRead, + CSN: replication.NullCSN(), + Data: map[string]any{"ID": 1, "NAME": "Alice"}, + }, + wantMeta: map[string]string{ + "db2_schema": "DB2ADMIN", + "db2_table": "EMPLOYEES", + "db2_operation": "read", + "db2_csn": "", + }, + }, + { + name: "insert event with CSN and timestamp", + event: replication.ChangeEvent{ + Schema: "DB2ADMIN", + Table: "ORDERS", + Operation: replication.OpTypeInsert, + CSN: replication.NewCSN(12345), + Timestamp: ts, + Data: map[string]any{"ORDER_ID": 99}, + }, + wantMeta: map[string]string{ + "db2_schema": "DB2ADMIN", + "db2_table": "ORDERS", + "db2_operation": "insert", + "db2_csn": "CSN:0000000000003039", + "db2_timestamp": ts.Format(time.RFC3339Nano), + }, + }, + { + name: "delete event", + event: replication.ChangeEvent{ + Schema: "SCHEMA", + Table: "T", + Operation: replication.OpTypeDelete, + CSN: replication.NewCSN(999), + Data: map[string]any{"ID": 7}, + }, + wantMeta: map[string]string{ + "db2_schema": "SCHEMA", + "db2_table": "T", + "db2_operation": "delete", + }, + }, + } + + d := &db2CDCInput{} + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + msg, err := d.eventToMessage(tc.event) + require.NoError(t, err) + require.NotNil(t, msg) + + for key, want := range tc.wantMeta { + got, exists := msg.MetaGet(key) + assert.True(t, exists, "meta key %q should exist", key) + assert.Equal(t, want, got, "meta key %q", key) + } + }) + } +} + +// --------------------------------------------------------------------------- +// detectVersion +// --------------------------------------------------------------------------- + +func TestDetectVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + queryFunc func(query string, args []driver.Value) ([]string, [][]driver.Value, error) + wantMajor int + wantMinor int + wantErr bool + }{ + { + name: "ENV_INST_INFO returns SQL11050", + queryFunc: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "ENV_INST_INFO") { + return []string{"SERVICE_LEVEL"}, [][]driver.Value{{"SQL11050"}}, nil + } + return nil, nil, fmt.Errorf("unexpected query: %s", q) + }, + wantMajor: 11, + wantMinor: 5, + }, + { + name: "fallback to PROD_RELEASE when ENV_INST_INFO fails", + queryFunc: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "ENV_INST_INFO") { + return nil, nil, fmt.Errorf("table does not exist") + } + // Fallback query + return []string{"PROD_RELEASE"}, [][]driver.Value{{"11.1.0.0"}}, nil + }, + wantMajor: 11, + wantMinor: 1, + }, + { + name: "both queries fail → error", + queryFunc: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return nil, nil, fmt.Errorf("connection error") + }, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openInputFakeDB(t, &inputFakeHandlers{query: tc.queryFunc}) + d := &db2CDCInput{db: db, log: testLogger(t)} + + ver, err := d.detectVersion(context.Background()) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantMajor, ver.Major) + assert.Equal(t, tc.wantMinor, ver.Minor) + }) + } +} + +// --------------------------------------------------------------------------- +// initCheckpointTable +// --------------------------------------------------------------------------- + +func TestInitCheckpointTable(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + tableName string + createErr error + wantErr bool + }{ + { + name: "creates table successfully", + tableName: "RPCN.CDC_CHECKPOINT", + createErr: nil, + }, + { + name: "table already exists (42710) is silently ignored", + tableName: "RPCN.CDC_CHECKPOINT", + createErr: fmt.Errorf("SQLSTATE=42710 object already exists"), + }, + { + name: "other exec error propagates", + tableName: "RPCN.CDC_CHECKPOINT", + createErr: fmt.Errorf("permission denied"), + wantErr: true, + }, + { + name: "unqualified table name (no schema) creates table", + tableName: "CDC_CHECKPOINT", + createErr: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + execCount := 0 + db := openInputFakeDB(t, &inputFakeHandlers{ + exec: func(q string, _ []driver.Value) error { + execCount++ + if strings.Contains(q, "CREATE SCHEMA") { + return nil // CREATE SCHEMA always succeeds in tests + } + return tc.createErr + }, + }) + + d := &db2CDCInput{ + db: db, + cpCacheTableName: tc.tableName, + log: testLogger(t), + } + + err := d.initCheckpointTable(context.Background()) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +// --------------------------------------------------------------------------- +// loadCheckpoint (DB table path) +// --------------------------------------------------------------------------- + +func TestLoadCheckpointFromDB(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + queryRows [][]driver.Value + noRows bool + queryErr error + wantCSN uint64 + wantNull bool + wantErr bool + }{ + { + name: "checkpoint found", + queryRows: [][]driver.Value{{"CSN:0000000000003039"}}, // 12345 + wantCSN: 12345, + }, + { + name: "no checkpoint → NullCSN", + noRows: true, + wantNull: true, + }, + { + name: "query error propagates", + queryErr: fmt.Errorf("connection timeout"), + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openInputFakeDB(t, &inputFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if tc.queryErr != nil { + return nil, nil, tc.queryErr + } + if tc.noRows { + return []string{"CACHE_VAL"}, nil, nil // no rows → sql.ErrNoRows + } + return []string{"CACHE_VAL"}, tc.queryRows, nil + }, + }) + + d := &db2CDCInput{ + db: db, + cpCacheName: "", + cpCacheTableName: "RPCN.CDC_CHECKPOINT", + checkpointCacheKey: "db2_cdc_checkpoint", + log: testLogger(t), + } + + csn, err := d.loadCheckpoint(context.Background()) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + if tc.wantNull { + assert.True(t, csn.IsNull()) + } else { + assert.Equal(t, tc.wantCSN, csn.Uint64()) + } + }) + } +} + +// --------------------------------------------------------------------------- +// saveCheckpoint (DB table path) +// --------------------------------------------------------------------------- + +func TestSaveCheckpointToDB(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + csn replication.CSN + execErr error + wantErr bool + }{ + { + name: "saves successfully", + csn: replication.NewCSN(12345), + }, + { + name: "exec error propagates", + csn: replication.NewCSN(99), + execErr: fmt.Errorf("deadlock detected"), + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var capturedArgs []driver.Value + db := openInputFakeDB(t, &inputFakeHandlers{ + exec: func(_ string, args []driver.Value) error { + capturedArgs = args + return tc.execErr + }, + }) + + d := &db2CDCInput{ + db: db, + cpCacheName: "", + cpCacheTableName: "RPCN.CDC_CHECKPOINT", + checkpointCacheKey: "db2_cdc_checkpoint", + log: testLogger(t), + } + + err := d.saveCheckpoint(context.Background(), tc.csn) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + // Verify the args include our checkpoint key and CSN string. + require.Len(t, capturedArgs, 2) + assert.Equal(t, "db2_cdc_checkpoint", capturedArgs[0]) + assert.Equal(t, tc.csn.String(), capturedArgs[1]) + }) + } +} + +// --------------------------------------------------------------------------- +// eventToMessage data type and message body tests +// --------------------------------------------------------------------------- + +// TestEventToMessageNullData verifies that ChangeEvents whose Data map contains +// nil values are serialised correctly: nil values appear as JSON null, not as +// absent keys or zero values. Downstream consumers must be able to distinguish +// "column is NULL" from "column was not selected". +// +// In the Debezium-compatible envelope, row data appears under the "after" key. +func TestEventToMessageNullData(t *testing.T) { + t.Parallel() + + event := replication.ChangeEvent{ + Schema: "DB2ADMIN", + Table: "EMPLOYEES", + Operation: replication.OpTypeInsert, + CSN: replication.NewCSN(1), + Data: map[string]any{ + "ID": int64(42), + "MANAGER_ID": nil, // explicit NULL from the source + "DEPT": nil, + }, + } + + d := &db2CDCInput{} + msg, err := d.eventToMessage(event) + require.NoError(t, err) + + b, err := msg.AsBytes() + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(b, &decoded)) + + // Row data is under "after" in the Debezium envelope. + after, ok := decoded["after"].(map[string]any) + require.True(t, ok, "after field must be a JSON object") + + assert.Equal(t, float64(42), after["ID"]) + + // Nil values must appear as JSON null, not as absent keys. + managerVal, managerExists := after["MANAGER_ID"] + assert.True(t, managerExists, "MANAGER_ID key must be present even when NULL") + assert.Nil(t, managerVal, "MANAGER_ID must be JSON null") + + deptVal, deptExists := after["DEPT"] + assert.True(t, deptExists, "DEPT key must be present even when NULL") + assert.Nil(t, deptVal, "DEPT must be JSON null") + + // "before" must be null for INSERT events. + assert.Nil(t, decoded["before"], "before must be null for INSERT events") + + // Verify top-level envelope fields. + assert.Equal(t, "c", decoded["op"]) +} + +// TestEventToMessageNumericTypes verifies that numeric DB2 column types +// (INTEGER, BIGINT, FLOAT) round-trip through eventToMessage without precision +// loss or silent type coercion. +// +// In the Debezium-compatible envelope, row data appears under the "after" key. +func TestEventToMessageNumericTypes(t *testing.T) { + t.Parallel() + + event := replication.ChangeEvent{ + Schema: "DB2ADMIN", + Table: "METRICS", + Operation: replication.OpTypeRead, + CSN: replication.NullCSN(), + Data: map[string]any{ + "INT_COL": int64(2147483647), // max int32 as int64 + "BIGINT_COL": int64(9000000000000), // requires full int64 range + "FLOAT_COL": float64(3.14159), + }, + } + + d := &db2CDCInput{} + msg, err := d.eventToMessage(event) + require.NoError(t, err) + + b, err := msg.AsBytes() + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(b, &decoded)) + + // Row data is under "after" in the Debezium envelope. + after, ok := decoded["after"].(map[string]any) + require.True(t, ok) + + // JSON numbers become float64 after Unmarshal; verify round-trip values. + assert.Equal(t, float64(2147483647), after["INT_COL"]) + assert.Equal(t, float64(9000000000000), after["BIGINT_COL"]) + assert.InDelta(t, 3.14159, after["FLOAT_COL"], 1e-10) + + // Snapshot rows use op="r" and snapshot="true". + assert.Equal(t, "r", decoded["op"]) +} + +// TestEventToMessageStringAndBinaryTypes verifies that VARCHAR columns appear +// as JSON strings and that binary ([]byte) columns are base64-encoded strings +// in the JSON output (standard encoding/json behaviour for []byte). +// +// In the Debezium-compatible envelope, row data appears under the "after" key. +func TestEventToMessageStringAndBinaryTypes(t *testing.T) { + t.Parallel() + + binaryPayload := []byte{0xDE, 0xAD, 0xBE, 0xEF} + + event := replication.ChangeEvent{ + Schema: "DB2ADMIN", + Table: "DOCUMENTS", + Operation: replication.OpTypeInsert, + CSN: replication.NewCSN(5), + Data: map[string]any{ + "TITLE": "Hello World", // VARCHAR → JSON string + "CONTENT": binaryPayload, // BLOB → []byte → base64 in JSON + }, + } + + d := &db2CDCInput{} + msg, err := d.eventToMessage(event) + require.NoError(t, err) + + b, err := msg.AsBytes() + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(b, &decoded)) + + // Row data is under "after" in the Debezium envelope. + after, ok := decoded["after"].(map[string]any) + require.True(t, ok) + + // VARCHAR must come through as a plain JSON string. + assert.Equal(t, "Hello World", after["TITLE"]) + + // []byte values are base64-encoded by encoding/json. + contentStr, isStr := after["CONTENT"].(string) + require.True(t, isStr, "binary column must be a JSON string (base64-encoded)") + assert.NotEmpty(t, contentStr) +} + +// TestEventToMessageMetadataKeys verifies all required message metadata keys +// are set for each operation type. Snapshot events (OpTypeRead) must carry an +// empty db2_csn since NullCSN().String() == "". Streaming events must carry +// the canonical "CSN:" string and, when the timestamp is non-zero, the +// RFC3339Nano-formatted db2_timestamp. +// +// The db2_operation key carries the human-readable form ("read", "insert", +// "delete"); db2_op carries the Debezium one-character code ("r", "c", "d"). +// db2_commit_lsn is an alias for db2_csn using the Debezium field name. +func TestEventToMessageMetadataKeys(t *testing.T) { + t.Parallel() + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + + tests := []struct { + name string + event replication.ChangeEvent + wantCSN string + wantHasTS bool + wantDebezOp string // Debezium single-char op code + wantSnapshot string + }{ + { + name: "snapshot read — no CSN, no timestamp", + event: replication.ChangeEvent{ + Schema: "S", Table: "T", Operation: replication.OpTypeRead, + CSN: replication.NullCSN(), + }, + wantCSN: "", + wantHasTS: false, + wantDebezOp: "r", + wantSnapshot: "true", + }, + { + name: "streaming insert — CSN and timestamp present", + event: replication.ChangeEvent{ + Schema: "S", Table: "T", Operation: replication.OpTypeInsert, + CSN: replication.NewCSN(500), Timestamp: ts, + }, + wantCSN: "CSN:00000000000001F4", + wantHasTS: true, + wantDebezOp: "c", + wantSnapshot: "false", + }, + { + name: "streaming delete — CSN present, zero timestamp", + event: replication.ChangeEvent{ + Schema: "S", Table: "T", Operation: replication.OpTypeDelete, + CSN: replication.NewCSN(501), + }, + wantCSN: "CSN:00000000000001F5", + wantHasTS: false, + wantDebezOp: "d", + wantSnapshot: "false", + }, + } + + d := &db2CDCInput{} + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + msg, err := d.eventToMessage(tc.event) + require.NoError(t, err) + + schema, _ := msg.MetaGet("db2_schema") + table, _ := msg.MetaGet("db2_table") + op, _ := msg.MetaGet("db2_operation") + debezOp, debezOpExists := msg.MetaGet("db2_op") + csn, csnExists := msg.MetaGet("db2_csn") + commitLSN, commitLSNExists := msg.MetaGet("db2_commit_lsn") + connector, connectorExists := msg.MetaGet("db2_connector") + snapshot, snapshotExists := msg.MetaGet("db2_snapshot") + + assert.Equal(t, "S", schema) + assert.Equal(t, "T", table) + // db2_operation carries the human-readable form. + assert.Equal(t, string(tc.event.Operation), op) + // db2_op carries the Debezium one-character code. + assert.True(t, debezOpExists, "db2_op must always be set") + assert.Equal(t, tc.wantDebezOp, debezOp) + // db2_csn (backward-compat) and db2_commit_lsn (Debezium name) must match. + assert.True(t, csnExists, "db2_csn must always be set") + assert.Equal(t, tc.wantCSN, csn) + assert.True(t, commitLSNExists, "db2_commit_lsn must always be set") + assert.Equal(t, tc.wantCSN, commitLSN, "db2_commit_lsn must equal db2_csn") + // db2_connector must always be "redpanda.db2". + assert.True(t, connectorExists, "db2_connector must always be set") + assert.Equal(t, "redpanda.db2", connector) + // db2_snapshot. + assert.True(t, snapshotExists, "db2_snapshot must always be set") + assert.Equal(t, tc.wantSnapshot, snapshot) + + tsVal, tsExists := msg.MetaGet("db2_timestamp") + if tc.wantHasTS { + assert.True(t, tsExists, "db2_timestamp must be set when Timestamp is non-zero") + assert.Equal(t, ts.Format(time.RFC3339Nano), tsVal) + } else { + if tsExists { + assert.Empty(t, tsVal, "db2_timestamp must be empty when Timestamp is zero") + } + } + }) + } +} + +// --------------------------------------------------------------------------- +// Checkpoint resume tests +// --------------------------------------------------------------------------- + +// TestLoadCheckpointParseCSNFormats verifies that loadCheckpoint correctly +// round-trips CSN values stored in the checkpoint table in all supported string +// formats. The connector writes "CSN:" canonically but must also handle +// decimal and 0x-prefixed hex for backward compatibility. +func TestLoadCheckpointParseCSNFormats(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stored string // value stored in CACHE_VAL column + wantCSN uint64 + }{ + { + name: "CSN prefix hex (canonical write format)", + stored: "CSN:0000000000003039", + wantCSN: 12345, + }, + { + name: "decimal integer", + stored: "12345", + wantCSN: 12345, + }, + { + name: "hex with 0x prefix", + stored: "0x3039", + wantCSN: 12345, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openInputFakeDB(t, &inputFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{"CACHE_VAL"}, [][]driver.Value{{tc.stored}}, nil + }, + }) + + d := &db2CDCInput{ + db: db, + cpCacheTableName: "RPCN.CDC_CHECKPOINT", + checkpointCacheKey: "db2_cdc_checkpoint", + log: testLogger(t), + } + + csn, err := d.loadCheckpoint(context.Background()) + require.NoError(t, err) + assert.Equal(t, tc.wantCSN, csn.Uint64()) + }) + } +} + +// TestSaveCheckpointWritesCanonicalFormat verifies that saveCheckpoint always +// writes the canonical "CSN:" format to the checkpoint table and uses a +// MERGE (upsert) statement so both first-time writes and updates work correctly. +func TestSaveCheckpointWritesCanonicalFormat(t *testing.T) { + t.Parallel() + + var capturedMerge string + var capturedArgs []driver.Value + + db := openInputFakeDB(t, &inputFakeHandlers{ + exec: func(q string, args []driver.Value) error { + if strings.Contains(q, "MERGE") { + capturedMerge = q + capturedArgs = args + } + return nil + }, + }) + + d := &db2CDCInput{ + db: db, + cpCacheTableName: "RPCN.CDC_CHECKPOINT", + checkpointCacheKey: "my_connector_key", + log: testLogger(t), + } + + csn := replication.NewCSN(50000) // 0xC350 + err := d.saveCheckpoint(context.Background(), csn) + require.NoError(t, err) + + assert.Contains(t, capturedMerge, "MERGE INTO RPCN.CDC_CHECKPOINT", + "saveCheckpoint must use a MERGE statement for upsert semantics") + require.Len(t, capturedArgs, 2) + assert.Equal(t, "my_connector_key", capturedArgs[0], + "first MERGE argument must be the checkpoint cache key") + assert.Equal(t, "CSN:000000000000C350", capturedArgs[1], + "second MERGE argument must be the canonical CSN: string") +} + +// TestStreamingSkipsSnapshotWhenCheckpointExists verifies the checkpoint resume +// behaviour: when a saved CSN is loaded from the checkpoint table, the connector +// sets StreamConfig.StartingCSN to the saved CSN and the snapshot phase is +// skipped (because StartingCSN is no longer null). +// +// The stream poll query uses IBMSNAP_COMMITSEQ > X'', so setting +// StartingCSN = savedCSN is correct — the > predicate already excludes the +// saved position itself. Using .Next() was an off-by-one that dropped the +// first event after a restart. +func TestStreamingSkipsSnapshotWhenCheckpointExists(t *testing.T) { + t.Parallel() + + savedCSN := replication.NewCSN(999) + + db := openInputFakeDB(t, &inputFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "CDC_CHECKPOINT") { + return []string{"CACHE_VAL"}, [][]driver.Value{{savedCSN.String()}}, nil + } + return nil, nil, nil + }, + }) + + d := &db2CDCInput{ + db: db, + cpCacheTableName: "RPCN.CDC_CHECKPOINT", + checkpointCacheKey: "db2_cdc_checkpoint", + snapshotMode: snapshotModeInitial, // snapshot is enabled in config + streamConfig: replication.StreamConfig{ + Schemas: []string{"DB2INST1"}, + Tables: []string{"EMPLOYEES"}, + PollBatchSize: 100, + }, + log: testLogger(t), + } + + csn, err := d.loadCheckpoint(context.Background()) + require.NoError(t, err) + require.False(t, csn.IsNull()) + + // Simulate what Connect() does after loading a non-null checkpoint. + d.streamConfig.StartingCSN = csn + + // With a saved checkpoint, StartingCSN is set to savedCSN. + // runCDC skips the snapshot for snapshotModeInitial when StartingCSN is not null. + // Since StartingCSN is NOT null, the snapshot must be skipped. + assert.False(t, d.streamConfig.StartingCSN.IsNull(), + "after checkpoint resume, StartingCSN must not be null so snapshot is skipped") + assert.Equal(t, savedCSN.Uint64(), d.streamConfig.StartingCSN.Uint64(), + "StartingCSN must equal the saved checkpoint (poll uses >, so no double-increment needed)") +} + +// TestSnapshotModes verifies the doSnapshot decision for each snapshot_mode value. +func TestSnapshotModes(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + mode snapshotMode + hasCheckpoint bool + expectSnapshot bool + }{ + {"initial_no_checkpoint", snapshotModeInitial, false, true}, + {"initial_with_checkpoint", snapshotModeInitial, true, false}, + {"when_needed_no_checkpoint", snapshotModeWhenNeeded, false, true}, + {"when_needed_with_checkpoint", snapshotModeWhenNeeded, true, false}, + {"always_no_checkpoint", snapshotModeAlways, false, true}, + {"always_with_checkpoint", snapshotModeAlways, true, true}, + {"never_no_checkpoint", snapshotModeNever, false, false}, + {"never_with_checkpoint", snapshotModeNever, true, false}, + {"no_data_no_checkpoint", snapshotModeNoData, false, false}, + {"no_data_with_checkpoint", snapshotModeNoData, true, false}, + {"initial_only_no_checkpoint", snapshotModeInitialOnly, false, true}, + {"initial_only_with_checkpoint", snapshotModeInitialOnly, true, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var startCSN replication.CSN + if tc.hasCheckpoint { + startCSN = replication.NewCSN(12345) + } else { + startCSN = replication.NullCSN() + } + assert.Equal(t, tc.expectSnapshot, shouldTakeSnapshot(tc.mode, startCSN)) + }) + } +} + +// --------------------------------------------------------------------------- +// Table registration / filtering tests +// --------------------------------------------------------------------------- + +// TestInitializeFiltersMissingTables verifies that Initialize returns a +// descriptive error when a configured table is absent from IBMSNAP_REGISTER. +// This prevents silent data loss where a typo in the tables config causes the +// connector to start but capture zero changes for the misnamed table. +func TestInitializeFiltersMissingTables(t *testing.T) { + t.Parallel() + + db := openInputFakeDB(t, &inputFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + // Only EMPLOYEES is registered; PAYROLL is not. + return []string{"SOURCE_OWNER", "SOURCE_TABLE", "CD_OWNER", "CD_TABLE"}, + [][]driver.Value{ + {"DB2INST1", "EMPLOYEES", "ASNCDC", "EMPLOYEES_CT"}, + }, nil + }, + }) + + // NewStreamer is accessible from the replication package and exercises the + // same Initialize code path that runStreaming calls internally. + s := replication.NewStreamer(db, replication.StreamConfig{ + Schemas: []string{"DB2INST1"}, + Tables: []string{"EMPLOYEES", "PAYROLL"}, + }, replication.Version{}) + + err := s.Initialize(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "PAYROLL", + "error must name the missing table so the operator knows which registration to add") +} + +// --------------------------------------------------------------------------- +// eventToMessage — heartbeat +// --------------------------------------------------------------------------- + +func TestEventToMessageHeartbeat(t *testing.T) { + t.Parallel() + + ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + event := replication.ChangeEvent{ + Operation: replication.OpTypeHeartbeat, + Timestamp: ts, + } + + d := &db2CDCInput{} + msg, err := d.eventToMessage(event) + require.NoError(t, err) + + b, err := msg.AsBytes() + require.NoError(t, err) + + var envelope map[string]any + require.NoError(t, json.Unmarshal(b, &envelope)) + + assert.Equal(t, "hb", envelope["op"]) + assert.Equal(t, float64(ts.UnixMilli()), envelope["ts_ms"]) + + op, _ := msg.MetaGet("db2_operation") + assert.Equal(t, "heartbeat", op) + opCode, _ := msg.MetaGet("db2_op") + assert.Equal(t, "hb", opCode) + + schemaVal, schemaExists := msg.MetaGet("db2_schema") + assert.True(t, schemaExists, "db2_schema must be set on heartbeat") + assert.Empty(t, schemaVal) + + tableVal, tableExists := msg.MetaGet("db2_table") + assert.True(t, tableExists, "db2_table must be set on heartbeat") + assert.Empty(t, tableVal) + + csnVal, csnExists := msg.MetaGet("db2_csn") + assert.True(t, csnExists, "db2_csn must be set on heartbeat") + assert.Empty(t, csnVal) + + commitLSN, commitLSNExists := msg.MetaGet("db2_commit_lsn") + assert.True(t, commitLSNExists, "db2_commit_lsn must be set on heartbeat") + assert.Empty(t, commitLSN) + + connector, connectorExists := msg.MetaGet("db2_connector") + assert.True(t, connectorExists, "db2_connector must be set on heartbeat") + assert.Equal(t, "redpanda.db2", connector) + + snapshot, snapshotExists := msg.MetaGet("db2_snapshot") + assert.True(t, snapshotExists, "db2_snapshot must be set on heartbeat") + assert.Equal(t, "false", snapshot) + + tsVal, tsExists := msg.MetaGet("db2_timestamp") + assert.True(t, tsExists, "db2_timestamp must be set when Timestamp is non-zero") + assert.Equal(t, ts.Format(time.RFC3339Nano), tsVal) + + // Heartbeat must not have before/after/source fields. + assert.NotContains(t, envelope, "before") + assert.NotContains(t, envelope, "after") + assert.NotContains(t, envelope, "source") +} + +// --------------------------------------------------------------------------- +// eventToMessage — schema change +// --------------------------------------------------------------------------- + +func TestEventToMessageSchemaChange(t *testing.T) { + t.Parallel() + + ts := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + event := replication.ChangeEvent{ + Schema: "DB2INST1", + Table: "NEW_TABLE", + Operation: replication.OpTypeSchemaChange, + CSN: replication.NewCSN(12345), + Timestamp: ts, + } + + d := &db2CDCInput{} + msg, err := d.eventToMessage(event) + require.NoError(t, err) + + b, err := msg.AsBytes() + require.NoError(t, err) + + var envelope map[string]any + require.NoError(t, json.Unmarshal(b, &envelope)) + + assert.Equal(t, "schema_change", envelope["op"]) + assert.Equal(t, float64(ts.UnixMilli()), envelope["ts_ms"]) + + src, ok := envelope["source"].(map[string]any) + require.True(t, ok, "source must be a map") + assert.Equal(t, "DB2INST1", src["schema"]) + assert.Equal(t, "NEW_TABLE", src["table"]) + + schemaVal, _ := msg.MetaGet("db2_schema") + assert.Equal(t, "DB2INST1", schemaVal) + tableVal, _ := msg.MetaGet("db2_table") + assert.Equal(t, "NEW_TABLE", tableVal) + opVal, _ := msg.MetaGet("db2_operation") + assert.Equal(t, "schema_change", opVal) + + opCode, opCodeExists := msg.MetaGet("db2_op") + assert.True(t, opCodeExists, "db2_op must be set on schema_change") + assert.Equal(t, "schema_change", opCode) + + expectedCSN := replication.NewCSN(12345).String() + csnVal, csnExists := msg.MetaGet("db2_csn") + assert.True(t, csnExists, "db2_csn must be set on schema_change") + assert.Equal(t, expectedCSN, csnVal) + + commitLSN, commitLSNExists := msg.MetaGet("db2_commit_lsn") + assert.True(t, commitLSNExists, "db2_commit_lsn must be set on schema_change") + assert.Equal(t, expectedCSN, commitLSN) + + connector, connectorExists := msg.MetaGet("db2_connector") + assert.True(t, connectorExists, "db2_connector must be set on schema_change") + assert.Equal(t, "redpanda.db2", connector) + + snapshot, snapshotExists := msg.MetaGet("db2_snapshot") + assert.True(t, snapshotExists, "db2_snapshot must be set on schema_change") + assert.Equal(t, "false", snapshot) + + tsVal, tsExists := msg.MetaGet("db2_timestamp") + assert.True(t, tsExists, "db2_timestamp must be set when Timestamp is non-zero") + assert.Equal(t, ts.Format(time.RFC3339Nano), tsVal) +} + +// --------------------------------------------------------------------------- +// parseSnapshotSignalTables +// --------------------------------------------------------------------------- + +func TestParseSnapshotSignalTables(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + data string + want []string + }{ + { + name: "json data-collections with schema prefix", + data: `{"data-collections":["DB2INST1.EMPLOYEES","DB2INST1.ORDERS"]}`, + want: []string{"EMPLOYEES", "ORDERS"}, + }, + { + name: "json data-collections bare names", + data: `{"data-collections":["EMPLOYEES"]}`, + want: []string{"EMPLOYEES"}, + }, + { + name: "comma-separated with schema", + data: "DB2INST1.EMPLOYEES, DB2INST1.ORDERS", + want: []string{"EMPLOYEES", "ORDERS"}, + }, + { + name: "comma-separated bare names", + data: "EMPLOYEES,ORDERS", + want: []string{"EMPLOYEES", "ORDERS"}, + }, + { + name: "empty data", + data: "", + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := parseSnapshotSignalTables(tc.data, "DB2INST1", nil) + assert.Equal(t, tc.want, got) + }) + } +} + +// --------------------------------------------------------------------------- +// ackFunc checkpoint semantics +// --------------------------------------------------------------------------- + +// TestAckFuncDoesNotAdvanceCheckpointOnNack verifies the all-or-nothing ackFunc +// semantics: on nack (ackErr != nil) the Capped slot is NOT released, so the +// watermark cannot advance past the failed message. AutoRetryNacks retries by +// calling the same ackFunc with nil; the slot is then released and checkpoint saved. +func TestAckFuncDoesNotAdvanceCheckpointOnNack(t *testing.T) { + t.Parallel() + checkpointSaved := false + d := &db2CDCInput{ + log: testLogger(t), + capped: checkpoint.NewCapped[replication.CSN](10), + } + + csn := replication.NewCSN(100) + resolveFn, err := d.capped.Track(context.Background(), csn, 1) + require.NoError(t, err) + + ackFunc := func(_ context.Context, ackErr error) error { + if ackErr != nil { + // Do not release the Capped slot on failure — AutoRetryNacks will re-deliver. + return nil + } + if highest := resolveFn(); highest != nil { + checkpointSaved = true + } + return nil + } + + _ = ackFunc(context.Background(), errors.New("downstream failure")) + assert.False(t, checkpointSaved, "checkpoint must not advance on nack") + + _ = ackFunc(context.Background(), nil) + assert.True(t, checkpointSaved, "checkpoint must advance on success ack") +} + +// --------------------------------------------------------------------------- +// snapshot_mode: initial_only — ReadBatch must return ErrEndOfInput +// --------------------------------------------------------------------------- + +// TestInitialOnlyTriggersEndOfInput verifies that after snapshot_mode=initial_only +// completes, ReadBatch returns service.ErrEndOfInput instead of blocking forever. +// +// The bug (pre-fix): runCDC returned early after the snapshot without calling +// d.shutSig.TriggerSoftStop(), so ReadBatch blocked indefinitely on d.eventChan. +func TestInitialOnlyTriggersEndOfInput(t *testing.T) { + t.Parallel() + + db := openInputFakeDB(t, &inputFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + // MAX(SYNCHPOINT) on an empty table returns one row with NULL. + // All other queries (snapshot table rows) return empty result sets. + if strings.Contains(q, "MAX(SYNCHPOINT)") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + } + return nil, nil, nil + }, + }) + + d := &db2CDCInput{ + db: db, + snapshotMode: snapshotModeInitialOnly, + snapshotConfig: replication.SnapshotConfig{ + Schemas: []string{"DB2INST1"}, + Tables: []string{}, + BatchSize: 100, + }, + streamConfig: replication.StreamConfig{ + Schemas: []string{"DB2INST1"}, + Tables: []string{}, + PollBatchSize: 100, + }, + eventChan: make(chan replication.ChangeEvent, 64), + errChan: make(chan error, 1), + shutSig: shutdown.NewSignaller(), + log: testLogger(t), + } + + // Run the CDC loop in a background goroutine (mirrors what Connect() does). + d.wg.Add(1) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go d.runCDC(ctx) + + // ReadBatch must unblock and return ErrEndOfInput once the snapshot finishes. + // If the bug is present this call hangs until the test context times out. + _, _, err := d.ReadBatch(ctx) + require.ErrorIs(t, err, service.ErrEndOfInput, + "ReadBatch must return ErrEndOfInput after snapshot_mode=initial_only completes") +} + +// --------------------------------------------------------------------------- +// BLOB/DBCS type mapping — cTypeForSQLType +// --------------------------------------------------------------------------- + +// TestCTypeForSQLType verifies that cTypeForSQLType maps every SQL type to the +// correct ODBC C transfer type, preventing BLOB truncation and DBCS misinterpretation. +func TestCTypeForSQLType(t *testing.T) { + t.Parallel() + + tests := []struct { + sqlType db2cli.SQLSMALLINT + wantCType db2cli.SQLSMALLINT + label string + }{ + // Binary / BLOB → SQL_C_BINARY + {db2cli.SQL_BINARY, db2cli.SQL_C_BINARY, "SQL_BINARY"}, + {db2cli.SQL_VARBINARY, db2cli.SQL_C_BINARY, "SQL_VARBINARY"}, + {db2cli.SQL_LONGVARBINARY, db2cli.SQL_C_BINARY, "SQL_LONGVARBINARY"}, + {db2cli.SQL_BLOB, db2cli.SQL_C_BINARY, "SQL_BLOB"}, + // DBCS / wide-char → SQL_C_WCHAR + {db2cli.SQL_WCHAR, db2cli.SQL_C_WCHAR, "SQL_WCHAR"}, + {db2cli.SQL_WVARCHAR, db2cli.SQL_C_WCHAR, "SQL_WVARCHAR"}, + {db2cli.SQL_WLONGVARCHAR, db2cli.SQL_C_WCHAR, "SQL_WLONGVARCHAR"}, + {db2cli.SQL_GRAPHIC, db2cli.SQL_C_WCHAR, "SQL_GRAPHIC"}, + {db2cli.SQL_VARGRAPHIC, db2cli.SQL_C_WCHAR, "SQL_VARGRAPHIC"}, + {db2cli.SQL_LONGVARGRAPHIC, db2cli.SQL_C_WCHAR, "SQL_LONGVARGRAPHIC"}, + {db2cli.SQL_DBCLOB, db2cli.SQL_C_WCHAR, "SQL_DBCLOB"}, + // Everything else → SQL_C_CHAR + {db2cli.SQL_CHAR, db2cli.SQL_C_CHAR, "SQL_CHAR"}, + {db2cli.SQL_VARCHAR, db2cli.SQL_C_CHAR, "SQL_VARCHAR"}, + {db2cli.SQL_INTEGER, db2cli.SQL_C_CHAR, "SQL_INTEGER"}, + {db2cli.SQL_BIGINT, db2cli.SQL_C_CHAR, "SQL_BIGINT"}, + {db2cli.SQL_BOOLEAN, db2cli.SQL_C_CHAR, "SQL_BOOLEAN"}, + } + for _, tt := range tests { + got := cTypeForSQLType(tt.sqlType) + assert.Equal(t, tt.wantCType, got, + "cTypeForSQLType(%s=%d): got %d, want %d", tt.label, tt.sqlType, got, tt.wantCType) + } +} + +// TestDecodeUTF16LE verifies that decodeUTF16LE correctly converts UTF-16LE +// bytes (as returned by DB2 for GRAPHIC/VARGRAPHIC columns) to UTF-8 strings. +// The round-trip encodes strings to UTF-16LE via golang.org/x/text/encoding/unicode +// (same library used by the production code) then decodes to verify correctness. +func TestDecodeUTF16LE(t *testing.T) { + t.Parallel() + + enc := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewEncoder() + + toUTF16LE := func(s string) []byte { + b, err := enc.Bytes([]byte(s)) + require.NoError(t, err) + return b + } + + tests := []struct { + input string + }{ + {""}, + {"AB"}, + {"Hello, World!"}, + {"中文"}, // CJK characters + {"日本語"}, // Japanese + {"한국어"}, // Korean + } + for _, tt := range tests { + encoded := toUTF16LE(tt.input) + got := decodeUTF16LE(encoded) + assert.Equal(t, tt.input, got, "round-trip for %q", tt.input) + } +} + +// --------------------------------------------------------------------------- +// Idempotency key +// --------------------------------------------------------------------------- + +// TestIdempotencyKeyFormat verifies that idempotencyKey produces stable, +// deterministic strings for both CDC and snapshot events. +func TestIdempotencyKeyFormat(t *testing.T) { + t.Parallel() + + cdcEvent := replication.ChangeEvent{ + Schema: "DB2INST1", + Table: "ORDERS", + CSN: replication.NewCSN(12345), + IntentSeq: 1, + } + key := idempotencyKey(cdcEvent) + // CSN.String() returns "CSN:"; 12345 decimal = 0x3039. + assert.Equal(t, "DB2INST1.ORDERS."+cdcEvent.CSN.String()+".1", key) + + // Snapshot event: null CSN — key derived from sorted PK values. + snapEvent := replication.ChangeEvent{ + Schema: "DB2INST1", + Table: "ORDERS", + CSN: replication.NullCSN(), + PKColumns: []string{"ID"}, + Data: map[string]any{"ID": "42", "NAME": "foo"}, + } + snapKey := idempotencyKey(snapEvent) + assert.True(t, strings.HasPrefix(snapKey, "DB2INST1.ORDERS.snapshot."), + "snapshot key must start with schema.table.snapshot., got %q", snapKey) + assert.Contains(t, snapKey, "ID=42", "snapshot key must contain PK field") +} + +// TestIdempotencyKeyStable verifies the key is deterministic across calls. +func TestIdempotencyKeyStable(t *testing.T) { + t.Parallel() + + event := replication.ChangeEvent{ + Schema: "SCH", + Table: "TBL", + CSN: replication.NewCSN(999), + IntentSeq: 7, + } + k1 := idempotencyKey(event) + k2 := idempotencyKey(event) + assert.Equal(t, k1, k2, "idempotencyKey must be deterministic") +} + +// --------------------------------------------------------------------------- +// snapshot_mode: when_needed +// --------------------------------------------------------------------------- + +// TestSnapshotWhenNeededMode verifies that when_needed triggers a snapshot only +// when the loaded checkpoint CSN is null (no prior state). +func TestSnapshotWhenNeededMode(t *testing.T) { + t.Parallel() + + assert.True(t, shouldTakeSnapshot(snapshotModeWhenNeeded, replication.NullCSN()), + "when_needed + null CSN should trigger snapshot") + assert.False(t, shouldTakeSnapshot(snapshotModeWhenNeeded, replication.NewCSN(42)), + "when_needed + existing CSN should skip snapshot") + assert.True(t, shouldTakeSnapshot(snapshotModeInitial, replication.NullCSN()), + "initial + null CSN should snapshot") + assert.False(t, shouldTakeSnapshot(snapshotModeInitial, replication.NewCSN(42)), + "initial + existing CSN should skip snapshot (already ran once)") + assert.False(t, shouldTakeSnapshot(snapshotModeNever, replication.NullCSN()), + "never + null CSN must not snapshot") + assert.False(t, shouldTakeSnapshot(snapshotModeNever, replication.NewCSN(99)), + "never + existing CSN must not snapshot") +} + +// --------------------------------------------------------------------------- +// Idempotency key — full wire-through tests +// --------------------------------------------------------------------------- + +// TestIdempotencyKeyUnique verifies that distinct CDC events produce distinct keys. +func TestIdempotencyKeyUnique(t *testing.T) { + t.Parallel() + + e1 := replication.ChangeEvent{Schema: "S", Table: "T", CSN: replication.NewCSN(1), IntentSeq: 0} + e2 := replication.ChangeEvent{Schema: "S", Table: "T", CSN: replication.NewCSN(2), IntentSeq: 0} + e3 := replication.ChangeEvent{Schema: "S", Table: "T", CSN: replication.NewCSN(1), IntentSeq: 1} + + assert.NotEqual(t, idempotencyKey(e1), idempotencyKey(e2), "different CSNs must produce different keys") + assert.NotEqual(t, idempotencyKey(e1), idempotencyKey(e3), "different intent seqs must produce different keys") +} + +// TestIdempotencyKeyInMessage verifies that eventToMessage sets the +// db2_idempotency_key metadata on outgoing messages. +func TestIdempotencyKeyInMessage(t *testing.T) { + t.Parallel() + + d := &db2CDCInput{} + event := replication.ChangeEvent{ + Schema: "DB2INST1", + Table: "ORDERS", + Operation: replication.OpTypeInsert, + CSN: replication.NewCSN(555), + IntentSeq: 2, + Data: map[string]any{"ID": "1", "NAME": "test"}, + } + + msg, err := d.eventToMessage(event) + require.NoError(t, err) + + val, ok := msg.MetaGet("db2_idempotency_key") + require.True(t, ok, "db2_idempotency_key metadata must be present") + assert.NotEmpty(t, val) + + // Key must be deterministic across two calls with the same event. + msg2, err := d.eventToMessage(event) + require.NoError(t, err) + val2, _ := msg2.MetaGet("db2_idempotency_key") + assert.Equal(t, val, val2, "idempotency key must be deterministic for the same event") +} + +// TestSnapshotIdempotencyKeyVsCDCKey verifies that snapshot events (null CSN) +// and CDC events for the same row produce distinct keys. +func TestSnapshotIdempotencyKeyVsCDCKey(t *testing.T) { + t.Parallel() + + snapEvent := replication.ChangeEvent{ + Schema: "DB2INST1", + Table: "ORDERS", + CSN: replication.NullCSN(), + Data: map[string]any{"ID": "42"}, + } + cdcEvent := replication.ChangeEvent{ + Schema: "DB2INST1", + Table: "ORDERS", + CSN: replication.NewCSN(42), + IntentSeq: 0, + } + + snapKey := idempotencyKey(snapEvent) + cdcKey := idempotencyKey(cdcEvent) + assert.NotEqual(t, snapKey, cdcKey, "snapshot and CDC keys must be distinct") + assert.Contains(t, snapKey, "snapshot", "snapshot key must contain 'snapshot' marker") +} + +// Compile-time check: sort is used by idempotencyKey. +var _ = sort.Strings diff --git a/internal/impl/db2/constants_sql.go b/internal/impl/db2/constants_sql.go new file mode 100644 index 0000000000..a7410ecfad --- /dev/null +++ b/internal/impl/db2/constants_sql.go @@ -0,0 +1,74 @@ +// Copyright 2025 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2 + +// SQL format strings used in db2CDCInput. All strings embed user-configured +// table names (signalTable, cpCacheTableName, asnCDCSchema) so they cannot be +// compile-time constants. They are computed once in newDB2CDCInput by calling +// fmt.Sprintf with the validated table names and stored in db2CDCInput for +// reuse across all polling iterations. +// +// All table identifiers embedded here are validated via isValidDB2Identifier +// (schema parts) or validateQualifiedIdentifier (schema.table) before any SQL +// is built. Direct string interpolation is safe for these values. +const ( + // Checkpoint table DML (%s = fully-qualified table name, e.g. "RPCN.CDC_CHECKPOINT"). + // + // cpMergeSQLFmt upserts a key/value pair. DB2 MERGE guarantees atomicity. + cpMergeSQLFmt = ` + MERGE INTO %s AS T + USING (VALUES (?, ?)) AS S(CACHE_KEY, CACHE_VAL) + ON T.CACHE_KEY = S.CACHE_KEY + WHEN MATCHED THEN UPDATE SET T.CACHE_VAL = S.CACHE_VAL + WHEN NOT MATCHED THEN INSERT (CACHE_KEY, CACHE_VAL) VALUES (S.CACHE_KEY, S.CACHE_VAL) + ` + // cpSelectSQLFmt reads a single checkpoint value by key. + cpSelectSQLFmt = "SELECT CACHE_VAL FROM %s WHERE CACHE_KEY = ?" + // cpDeleteSQLFmt removes a checkpoint entry by key. + cpDeleteSQLFmt = "DELETE FROM %s WHERE CACHE_KEY = ?" + + // Signal table queries (%[1]s = signal table, %[2]d = signalFetchLimit). + // + // Signal table schema (auto-created if absent): + // CREATE TABLE
( + // ID VARCHAR(255) NOT NULL, + // TYPE VARCHAR(64) NOT NULL, + // DATA VARCHAR(2048), + // PRIMARY KEY (ID) + // ) + // + // Signal types consumed by processSignals: + // 'execute-snapshot' – trigger an incremental snapshot of DATA.data-collections + // 'stop-snapshot' – abort the in-progress incremental snapshot + // 'pause-snapshot' – pause the snapshot mid-run (resumes on 'resume-snapshot') + // 'resume-snapshot' – resume a paused snapshot + sigSelectStopSQLFmt = "SELECT ID FROM %[1]s WHERE TYPE = 'stop-snapshot' FETCH FIRST %[2]d ROWS ONLY" + sigSelectPauseSQLFmt = "SELECT ID FROM %[1]s WHERE TYPE = 'pause-snapshot' FETCH FIRST %[2]d ROWS ONLY" + sigSelectResumeSQLFmt = "SELECT ID FROM %[1]s WHERE TYPE = 'resume-snapshot' FETCH FIRST %[2]d ROWS ONLY" + sigSelectExecSQLFmt = "SELECT ID, DATA FROM %[1]s WHERE TYPE = 'execute-snapshot' FETCH FIRST %[2]d ROWS ONLY" + // sigDeleteSQLFmt deletes any signal row by ID (used for post-snapshot cleanup). + sigDeleteSQLFmt = "DELETE FROM %s WHERE ID = ?" + // sigDeleteExecSQLFmt deletes only 'execute-snapshot' signals to avoid + // accidentally removing control signals that arrived concurrently. + sigDeleteExecSQLFmt = "DELETE FROM %s WHERE ID = ? AND TYPE = 'execute-snapshot'" + // sigAbortPollSQLFmt is polled inside the snapshot goroutine to detect + // stop/pause signals without holding any other lock. + sigAbortPollSQLFmt = "SELECT TYPE FROM %s WHERE TYPE IN ('stop-snapshot','pause-snapshot') FETCH FIRST 1 ROWS ONLY" + + // Schema-change poll query. %[1]s = asnCDCSchema, %[2]d = schemaChangeFetchLimit. + // The caller substitutes the lastHex watermark via a second fmt.Sprintf using %%s. + schemaChangeQueryFmt = ` + SELECT r.SOURCE_OWNER, r.SOURCE_TABLE, r.CD_NEW_SYNCHPOINT + FROM %[1]s.IBMSNAP_REGISTER r + WHERE r.SOURCE_OWNER = ? + AND r.CD_NEW_SYNCHPOINT > X'%%s' + ORDER BY r.CD_NEW_SYNCHPOINT + FETCH FIRST %[2]d ROWS ONLY + ` +) diff --git a/internal/impl/db2/db2cli/constants.go b/internal/impl/db2/db2cli/constants.go new file mode 100644 index 0000000000..5f22a43c2d --- /dev/null +++ b/internal/impl/db2/db2cli/constants.go @@ -0,0 +1,500 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +// Package db2cli implements IBM DB2 Call Level Interface (CLI) bindings via +// purego. Using purego instead of CGO means the DB2 driver compiles without a +// C toolchain and without linking against libdb2.so at build time — the shared +// library is loaded at runtime via dlopen/LoadLibrary. This shifts the IBM DB2 +// client runtime requirement from the build machine to the deployment +// environment, enabling a single pre-built binary that works wherever +// libdb2.so (Linux) or libdb2.dylib (macOS) is installed. +// +// Why purego instead of github.com/ibmdb/go_ibm_db? +// +// The IBM-owned go_ibm_db driver works via CGO, embedding a dependency on +// gcc and the DB2 C++ runtime (libdb2.so / clidriver) at compile time. +// This makes cross-compilation and pre-built binary distribution impossible: +// every build machine needs the full IBM Data Server Client installed. +// purego loads the same shared library at runtime, keeping the build entirely +// pure Go — no C toolchain, no CGO, no build-time linking against IBM code. +// The tradeoff is that the IBM CLI library must be present on the deployment +// host (not the build host), which matches how other Redpanda Connect +// enterprise connectors handle optional native dependencies. +// +// DB2 CLI is IBM's C-based call-level interface that mirrors the ODBC API. +// Every exported function in this package is a thin Go wrapper around the +// corresponding C function loaded from the DB2 client shared library. The +// constant names, numeric values, and semantics are identical to ODBC and to +// the IBM DB2 CLI reference documentation at +// https://www.ibm.com/docs/en/db2/11.5?topic=interfaces-db2-cli +package db2cli + +// DB2 CLI constants used throughout the IBM DB2 CLI API. +// Full constant reference: https://www.ibm.com/docs/en/db2/11.5?topic=reference-cli-constants + +// Handle type constants passed as the first argument to SQLAllocHandle and +// SQLFreeHandle to indicate which kind of handle to allocate or release. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=reference-handle-types +const ( + // SQL_HANDLE_ENV selects an environment handle, which holds global state + // (ODBC version, connection pooling mode) shared across all connections. + SQL_HANDLE_ENV = 1 + + // SQL_HANDLE_DBC selects a connection handle, which represents one + // physical TCP/IP connection to a DB2 instance. + SQL_HANDLE_DBC = 2 + + // SQL_HANDLE_STMT selects a statement handle, which represents one + // prepared or executed SQL statement and its associated cursor. + SQL_HANDLE_STMT = 3 + + // SQL_HANDLE_DESC selects a descriptor handle, which describes the + // structure of a parameter or column (type, length, precision, etc.). + SQL_HANDLE_DESC = 4 +) + +// Return codes returned by every DB2 CLI function call. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=cli-return-codes +// ODBC reference: https://learn.microsoft.com/en-us/sql/odbc/reference/develop-app/return-codes-odbc +const ( + // SQL_SUCCESS indicates the function completed without warnings or errors. + SQL_SUCCESS SQLRETURN = 0 + + // SQL_SUCCESS_WITH_INFO indicates success, but diagnostic records are + // available via SQLGetDiagRec (e.g. string truncation warnings). + SQL_SUCCESS_WITH_INFO SQLRETURN = 1 + + // SQL_NO_DATA indicates that no more rows are available in the result set + // (returned by SQLFetch when the cursor is exhausted) or that no data was + // found for a positioned operation. + SQL_NO_DATA SQLRETURN = 100 + + // SQL_ERROR indicates a non-recoverable failure. Call SQLGetDiagRec with + // the appropriate handle to retrieve the SQLSTATE and message text. + SQL_ERROR SQLRETURN = -1 + + // SQL_INVALID_HANDLE indicates the handle passed to the function is not + // a valid allocated handle. This is a programming error; the handle was + // never allocated, has already been freed, or is the wrong type. + SQL_INVALID_HANDLE SQLRETURN = -2 + + // SQL_STILL_EXECUTING is returned when a function called in asynchronous + // mode has not yet completed. Poll again or call SQLCancel. + SQL_STILL_EXECUTING SQLRETURN = 2 + + // SQL_NEED_DATA indicates that the application must supply parameter data + // at execution time via SQLPutData. Used with SQL_DATA_AT_EXEC. + SQL_NEED_DATA SQLRETURN = 99 +) + +// Null handle sentinels used as the inputHandle argument to SQLAllocHandle +// when allocating the first (environment) handle, and in diagnostic calls +// where no handle is available. +const ( + SQL_NULL_HANDLE SQLHANDLE = 0 + SQL_NULL_HENV SQLHENV = 0 + SQL_NULL_HDBC SQLHDBC = 0 + SQL_NULL_HSTMT SQLHSTMT = 0 + SQL_NULL_HDESC SQLHDESC = 0 +) + +// Special length and indicator values used in SQLBindCol, SQLBindParameter, +// and SQLGetData indicator arguments. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=reference-length-indicator-values +const ( + // SQL_NTS means the corresponding data value is a null-terminated C string; + // its length is determined by strlen(). Pass as the StrLen_or_Ind argument + // when the string buffer ends with a zero byte. + SQL_NTS = -3 + + // SQL_NULL_DATA in an indicator variable signals that the column or + // parameter value is SQL NULL (not an empty string or zero). + SQL_NULL_DATA = -1 + + // SQL_DATA_AT_EXEC in an indicator variable signals that the parameter + // data will be supplied at execution time via SQLPutData. + SQL_DATA_AT_EXEC = -2 + + // SQL_NO_TOTAL is returned in an indicator variable by SQLGetData when the + // total length of a large-object column is not known in advance. + SQL_NO_TOTAL = -4 + + // SQL_DEFAULT_PARAM signals that the default value defined in the procedure + // should be used for this parameter. + SQL_DEFAULT_PARAM = -5 + + // SQL_IGNORE signals that the parameter should be ignored (used with + // bulk operations). + SQL_IGNORE = -6 +) + +// Environment attribute constants for SQLSetEnvAttr / SQLGetEnvAttr. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlsetenvattr +const ( + // SQL_ATTR_ODBC_VERSION sets the ODBC version the application conforms to. + // Must be set on the environment handle before allocating connection handles. + // Use SQL_OV_ODBC3 for ODBC 3.x behaviour (recommended). + SQL_ATTR_ODBC_VERSION = 200 + + // SQL_ATTR_CONNECTION_POOLING controls whether connection pooling is enabled + // at the environment level. Disabled by default; not used in this driver. + SQL_ATTR_CONNECTION_POOLING = 201 + + // SQL_ATTR_CP_MATCH controls how pooled connections are matched to requests. + SQL_ATTR_CP_MATCH = 202 + + // SQL_ATTR_OUTPUT_NTS controls whether character data returned by CLI + // functions is null-terminated. Default is SQL_TRUE. + SQL_ATTR_OUTPUT_NTS = 10001 + + // SQL_OV_ODBC3 is the value for SQL_ATTR_ODBC_VERSION that enables + // ODBC 3.x semantics (recommended for all new applications). + SQL_OV_ODBC3 = 3 + + // SQL_OV_ODBC3_80 is the value for SQL_ATTR_ODBC_VERSION that enables + // ODBC 3.80 semantics (Windows 8 / SQL Server 2012 additions). + SQL_OV_ODBC3_80 = 380 +) + +// Connection attribute constants for SQLSetConnectAttr / SQLGetConnectAttr. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlsetconnectattr +const ( + // SQL_ATTR_ACCESS_MODE controls whether the connection is read-only. + SQL_ATTR_ACCESS_MODE = 101 + + // SQL_ATTR_AUTOCOMMIT controls whether each SQL statement is automatically + // committed. Set to SQL_AUTOCOMMIT_OFF to begin an explicit transaction; + // restore to SQL_AUTOCOMMIT_ON after SQLEndTran. + SQL_ATTR_AUTOCOMMIT = 102 + + // SQL_ATTR_CONNECTION_TIMEOUT sets the number of seconds to wait for any + // request on the connection before returning SQL_ERROR. + SQL_ATTR_CONNECTION_TIMEOUT = 113 + + // SQL_ATTR_CURRENT_CATALOG sets the default schema (catalog) for the connection. + SQL_ATTR_CURRENT_CATALOG = 109 + + // SQL_ATTR_LOGIN_TIMEOUT sets the number of seconds to wait for a login + // (SQLDriverConnect / SQLConnect) to complete. + SQL_ATTR_LOGIN_TIMEOUT = 103 + + // SQL_ATTR_TXN_ISOLATION sets the transaction isolation level. Use the + // SQL_TXN_* constants as the value. + SQL_ATTR_TXN_ISOLATION = 108 + + // SQL_AUTOCOMMIT_OFF disables autocommit; the application must call + // SQLEndTran(SQL_COMMIT) or SQLEndTran(SQL_ROLLBACK) explicitly. + SQL_AUTOCOMMIT_OFF = 0 + + // SQL_AUTOCOMMIT_ON enables autocommit (default). Each SQL statement is + // committed automatically upon successful completion. + SQL_AUTOCOMMIT_ON = 1 +) + +// Statement attribute constants for SQLSetStmtAttr / SQLGetStmtAttr. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlsetstmtattr +const ( + SQL_ATTR_APP_ROW_DESC = 10010 + SQL_ATTR_APP_PARAM_DESC = 10011 + SQL_ATTR_IMP_ROW_DESC = 10012 + SQL_ATTR_IMP_PARAM_DESC = 10013 + SQL_ATTR_CURSOR_SCROLLABLE = -1 + SQL_ATTR_CURSOR_SENSITIVITY = -2 + + // SQL_ATTR_QUERY_TIMEOUT sets the maximum number of seconds a statement + // may execute before it is cancelled and SQL_ERROR is returned. Zero + // disables the timeout (default). + SQL_ATTR_QUERY_TIMEOUT = 0 + + // SQL_ATTR_MAX_ROWS limits the number of rows returned by a SELECT. Zero + // means no limit (default). + SQL_ATTR_MAX_ROWS = 1 + SQL_ATTR_NOSCAN = 2 + SQL_ATTR_MAX_LENGTH = 3 + SQL_ATTR_ASYNC_ENABLE = 4 + SQL_ATTR_ROW_BIND_TYPE = 5 + + // SQL_ATTR_CURSOR_TYPE controls the cursor model. Use SQL_CURSOR_FORWARD_ONLY + // (default, most efficient) for read-once result sets. + SQL_ATTR_CURSOR_TYPE = 6 + SQL_ATTR_CONCURRENCY = 7 + SQL_ATTR_ROW_ARRAY_SIZE = 27 + SQL_ATTR_ROW_STATUS_PTR = 25 + SQL_ATTR_ROWS_FETCHED_PTR = 26 +) + +// Transaction completion type constants passed to SQLEndTran. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlendtran +const ( + // SQL_COMMIT durably commits all changes made in the current transaction. + SQL_COMMIT = 0 + + // SQL_ROLLBACK undoes all changes made in the current transaction. + SQL_ROLLBACK = 1 +) + +// Transaction isolation level constants used with SQL_ATTR_TXN_ISOLATION. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=concepts-isolation-levels +// +// DB2 uses different names than ODBC: Read Uncommitted = UR, Read Committed = CS +// (Cursor Stability), Repeatable Read = RS, Serializable = RR (Repeatable Read). +// Note: DB2's "Repeatable Read" (RR, serializable) is stricter than SQL standard +// REPEATABLE READ — it prevents phantom rows. Use SQL_TXN_SERIALIZABLE for DB2 RR. +const ( + // SQL_TXN_READ_UNCOMMITTED maps to DB2 "Uncommitted Read" (UR). Dirty reads + // are allowed; used only for analytics where consistency is not required. + SQL_TXN_READ_UNCOMMITTED = 1 + + // SQL_TXN_READ_COMMITTED maps to DB2 "Cursor Stability" (CS). A shared lock + // is held only while the cursor is positioned on a row. Default isolation level. + SQL_TXN_READ_COMMITTED = 2 + + // SQL_TXN_REPEATABLE_READ maps to DB2 "Read Stability" (RS). Rows read in + // the transaction cannot be modified by others, but phantom rows are possible. + SQL_TXN_REPEATABLE_READ = 4 + + // SQL_TXN_SERIALIZABLE maps to DB2 "Repeatable Read" (RR). No dirty reads, + // non-repeatable reads, or phantom rows. This is the strongest isolation level + // and is used for snapshot transactions to ensure consistent table reads. + SQL_TXN_SERIALIZABLE = 8 +) + +// C data type constants used in the targetType argument of SQLBindCol and +// SQLGetData to specify how DB2 should convert the column value into the +// application buffer. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=data-c-types +const ( + // SQL_C_CHAR binds the column as a null-terminated char string. All DB2 + // column types can be retrieved as SQL_C_CHAR (with appropriate conversion). + // This is the only target type used by the db2 driver in this package. + SQL_C_CHAR = 1 + + SQL_C_LONG = 4 + SQL_C_SHORT = 5 + SQL_C_FLOAT = 7 + SQL_C_DOUBLE = 8 + SQL_C_NUMERIC = 2 + SQL_C_DEFAULT = 99 + SQL_C_DATE = 9 + SQL_C_TIME = 10 + SQL_C_TIMESTAMP = 11 + + // SQL_C_BINARY retrieves the column as raw bytes. Used for CHAR FOR BIT DATA + // columns (such as IBMSNAP_COMMITSEQ) when exact byte representation matters. + SQL_C_BINARY = -2 + SQL_C_BIT = -7 + SQL_C_SBIGINT = -25 + SQL_C_UBIGINT = -27 + SQL_C_TINYINT = -6 + SQL_C_SLONG = -16 + SQL_C_SSHORT = -15 + SQL_C_STINYINT = -26 + SQL_C_ULONG = -18 + SQL_C_USHORT = -17 + SQL_C_UTINYINT = -28 + SQL_C_WCHAR = -8 +) + +// SQL data type constants that identify the native DB2 column type in +// SQLDescribeCol, SQLBindParameter, and column-type metadata queries. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=data-sql-types +const ( + SQL_CHAR = 1 + SQL_NUMERIC = 2 + SQL_DECIMAL = 3 + SQL_INTEGER = 4 + SQL_SMALLINT = 5 + SQL_FLOAT = 6 + SQL_REAL = 7 + SQL_DOUBLE = 8 + SQL_VARCHAR = 12 + SQL_DATE = 9 + SQL_TIME = 10 + SQL_TIMESTAMP = 11 + SQL_LONGVARCHAR = -1 + + // SQL_BINARY and SQL_VARBINARY map to CHAR FOR BIT DATA / VARCHAR FOR BIT DATA. + // IBMSNAP_COMMITSEQ is typed CHAR(10) FOR BIT DATA; its SQL type is SQL_BINARY. + SQL_BINARY = -2 + SQL_VARBINARY = -3 + SQL_LONGVARBINARY = -4 + SQL_BIGINT = -5 + SQL_TINYINT = -6 + SQL_BIT = -7 + SQL_WCHAR = -8 + SQL_WVARCHAR = -9 + SQL_WLONGVARCHAR = -10 + SQL_GUID = -11 + SQL_TYPE_DATE = 91 + SQL_TYPE_TIME = 92 + SQL_TYPE_TIMESTAMP = 93 + + // SQL_GRAPHIC, SQL_VARGRAPHIC, SQL_LONGVARGRAPHIC are IBM DB2 extensions for + // double-byte character set (DBCS) columns (GRAPHIC, VARGRAPHIC, DBCLOB variants). + // These require SQL_C_WCHAR retrieval to avoid UTF-16LE/UTF-8 misinterpretation. + SQL_GRAPHIC = -95 + SQL_VARGRAPHIC = -96 + SQL_LONGVARGRAPHIC = -97 + + // DB2-specific extended SQL types not present in the ODBC specification. + SQL_BLOB = -98 // DB2 BLOB (Binary Large Object) + SQL_CLOB = -99 // DB2 CLOB (Character Large Object) + SQL_DBCLOB = -350 // DB2 DBCLOB (Double-Byte CLOB) + SQL_XML = -370 // DB2 XML (stored as hierarchical data) + SQL_DECFLOAT = -360 // DB2 DECFLOAT (IEEE 754 decimal floating-point) + SQL_BOOLEAN = 16 // DB2 BOOLEAN (DB2 11.1+) +) + +// Nullable information constants returned by SQLDescribeCol in the nullable +// output argument. +const ( + // SQL_NO_NULLS means the column is defined NOT NULL. + SQL_NO_NULLS = 0 + + // SQL_NULLABLE means the column may contain SQL NULL values. + SQL_NULLABLE = 1 + + // SQL_NULLABLE_UNKNOWN means nullability cannot be determined (e.g. for + // expressions in the select list). + SQL_NULLABLE_UNKNOWN = 2 +) + +// Fetch orientation constants for SQLFetchScroll's fetchOrientation argument. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlfetchscroll +const ( + SQL_FETCH_NEXT = 1 // Advance one row forward (same as SQLFetch) + SQL_FETCH_FIRST = 2 // Move to the first row of the result set + SQL_FETCH_LAST = 3 // Move to the last row + SQL_FETCH_PRIOR = 4 // Move one row backward + SQL_FETCH_ABSOLUTE = 5 // Move to the row at the absolute row number in fetchOffset + SQL_FETCH_RELATIVE = 6 // Move fetchOffset rows relative to current position +) + +// SQLGetInfo information type constants. Pass as the infoType argument to +// retrieve driver and data-source properties. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlgetinfo +const ( + SQL_DBMS_NAME = 17 // Name of the DBMS product (e.g. "DB2/LINUXX8664") + SQL_DBMS_VER = 18 // DBMS version string (e.g. "11.05.0007") + SQL_DATABASE_NAME = 16 // Name of the currently connected database + SQL_DRIVER_NAME = 6 // Name of the CLI driver file + SQL_DRIVER_VER = 7 // Version of the CLI driver + SQL_SERVER_NAME = 13 // Name of the DB2 server + SQL_MAX_CATALOG_NAME_LEN = 34 + SQL_MAX_COLUMN_NAME_LEN = 30 + SQL_MAX_CURSOR_NAME_LEN = 31 + SQL_MAX_SCHEMA_NAME_LEN = 32 + SQL_MAX_TABLE_NAME_LEN = 35 + SQL_TXN_CAPABLE = 46 + SQL_GETDATA_EXTENSIONS = 81 +) + +// Diagnostic field constants for SQLGetDiagField. These identify which field +// of a diagnostic record to retrieve. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlgetdiagfield +const ( + SQL_DIAG_RETURNCODE = 1 // Return code of the function that created the record + SQL_DIAG_NUMBER = 2 // Number of diagnostic records available + SQL_DIAG_ROW_COUNT = 3 // Number of rows affected by INSERT/UPDATE/DELETE + SQL_DIAG_SQLSTATE = 4 // Five-character SQLSTATE string + SQL_DIAG_NATIVE = 5 // Driver-specific native error code + SQL_DIAG_MESSAGE_TEXT = 6 // Human-readable error message + SQL_DIAG_DYNAMIC_FUNCTION = 7 // SQL statement type that produced the record + SQL_DIAG_CLASS_ORIGIN = 8 // Origin of the SQLSTATE class + SQL_DIAG_SUBCLASS_ORIGIN = 9 // Origin of the SQLSTATE subclass + SQL_DIAG_CONNECTION_NAME = 10 // Name of the connection + SQL_DIAG_SERVER_NAME = 11 // Name of the server +) + +// Column descriptor field constants for SQLColAttribute and descriptor handles. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlcolattribute +const ( + SQL_DESC_COUNT = 1001 + SQL_DESC_TYPE = 1002 + SQL_DESC_LENGTH = 1003 + SQL_DESC_OCTET_LENGTH_PTR = 1004 + SQL_DESC_PRECISION = 1005 + SQL_DESC_SCALE = 1006 + SQL_DESC_DATETIME_INTERVAL_CODE = 1007 + SQL_DESC_NULLABLE = 1008 + SQL_DESC_INDICATOR_PTR = 1009 + SQL_DESC_DATA_PTR = 1010 + SQL_DESC_NAME = 1011 + SQL_DESC_UNNAMED = 1012 + SQL_DESC_OCTET_LENGTH = 1013 + SQL_DESC_ALLOC_TYPE = 1099 +) + +// Cursor concurrency constants for SQL_ATTR_CONCURRENCY. Controls whether the +// cursor can modify rows in the result set. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlsetstmtattr +const ( + // SQL_CONCUR_READ_ONLY makes the cursor read-only; no positioned updates or + // deletes are allowed. This is the most efficient concurrency mode and is + // required for the CDC streaming queries in this package. + SQL_CONCUR_READ_ONLY = 1 + SQL_CONCUR_LOCK = 2 // Lowest sufficient locking for updateability + SQL_CONCUR_ROWVER = 3 // Optimistic concurrency using row version + SQL_CONCUR_VALUES = 4 // Optimistic concurrency using row values +) + +// Cursor type constants for SQL_ATTR_CURSOR_TYPE. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlsetstmtattr +const ( + // SQL_CURSOR_FORWARD_ONLY is the default and most efficient cursor type. + // Rows are fetched in order; only SQLFetch (not SQLFetchScroll) is supported. + // All CDC and snapshot queries use forward-only cursors. + SQL_CURSOR_FORWARD_ONLY = 0 + + // SQL_CURSOR_KEYSET_DRIVEN is a scrollable cursor that uses a keyset to + // identify rows. Supports SQLFetchScroll; more expensive than forward-only. + SQL_CURSOR_KEYSET_DRIVEN = 1 + + // SQL_CURSOR_DYNAMIC is a fully scrollable cursor; membership and order can + // change between fetches. Most expensive; not used in this package. + SQL_CURSOR_DYNAMIC = 2 + + // SQL_CURSOR_STATIC is a scrollable cursor with a fixed result set snapshot. + SQL_CURSOR_STATIC = 3 +) + +// Free statement option constants for SQLFreeStmt's option argument. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlfreestmt +const ( + // SQL_CLOSE closes the cursor associated with the statement handle and + // discards any pending results. The statement handle remains allocated. + SQL_CLOSE = 0 + + // SQL_DROP closes the cursor and frees the statement handle. Equivalent to + // calling SQLFreeHandle(SQL_HANDLE_STMT, hstmt). + SQL_DROP = 1 + + // SQL_UNBIND releases all column bindings set by SQLBindCol. + SQL_UNBIND = 2 + + // SQL_RESET_PARAMS releases all parameter bindings set by SQLBindParameter. + SQL_RESET_PARAMS = 3 +) + +// Parameter input/output type constants for the inputOutputType argument of +// SQLBindParameter. +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlbindparameter +const ( + // SQL_PARAM_INPUT marks the parameter as an input-only parameter (value + // flows from application to DB2). This is the only type used by this driver + // since all CDC queries use read-only prepared statements. + SQL_PARAM_INPUT = 1 + + // SQL_PARAM_INPUT_OUTPUT marks the parameter as both input and output. + // Used with stored procedures. + SQL_PARAM_INPUT_OUTPUT = 2 + + // SQL_PARAM_OUTPUT marks the parameter as output-only (value flows from DB2 + // to the application). Used with stored procedure OUT parameters. + SQL_PARAM_OUTPUT = 4 +) diff --git a/internal/impl/db2/db2cli/db2cli.go b/internal/impl/db2/db2cli/db2cli.go new file mode 100644 index 0000000000..4c98ecf367 --- /dev/null +++ b/internal/impl/db2/db2cli/db2cli.go @@ -0,0 +1,179 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2cli + +import ( + "fmt" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + + "github.com/ebitengine/purego" +) + +// DB2 CLI ("Call Level Interface") library handle. "CLI" is IBM's term for +// its C-based ODBC-compatible SQL API — not a command-line tool. The name +// appears in library filenames (libdb2.so, db2cli.dll) and IBM documentation. +var ( + // libHandle is written once inside loadOnce.Do; visible after libLoaded is true. + libHandle uintptr + // libLoaded is set to true after a successful load; provides acquire/release fence. + libLoaded atomic.Bool + loadOnce sync.Once + // loadErr is written once inside loadOnce.Do; safe to read via LoadLibrary return value. + loadErr error +) + +// getLibraryPath returns the platform-specific DB2 CLI shared library name. +// "CLI" here is IBM's Call Level Interface — a C-based ODBC-compatible API +// for executing SQL against DB2, not a command-line tool. +// IBM ships libdb2.so.1 for Linux x86_64 and ppc64le; libdb2.dylib for macOS +// (Intel/ARM via Rosetta). Windows ships db2cli.dll. These are the only +// platforms tested with the IBM DB2 client. Any other GOOS is unsupported and +// panics at init time rather than failing silently at query time. +func getLibraryPath() string { + switch runtime.GOOS { + case "linux": + return "libdb2.so.1" // resolved via LD_LIBRARY_PATH or ldconfig cache + case "darwin": + return "libdb2.dylib" + case "windows": + return "db2cli.dll" + default: + panic("db2cli: unsupported GOOS " + runtime.GOOS + "; IBM DB2 CLI is only available on linux, darwin, and windows") + } +} + +// LoadLibrary loads the IBM DB2 CLI shared library using the platform's +// standard dynamic linker search (LD_LIBRARY_PATH / ldconfig cache on Linux, +// DYLD_LIBRARY_PATH on macOS). The library name is platform-specific: +// - Linux: libdb2.so.1 (IBM Data Server Driver package) +// - macOS: libdb2.dylib (IBM ODBC CLI Driver / clidriver) +// - Windows: db2cli.dll +// +// LoadLibrary is idempotent: subsequent calls return the cached result of the +// first attempt. If the library cannot be found, the error message includes +// installation hints. For tests that extract the library from a Docker container +// use LoadLibraryFromPath to bypass the linker search. +func LoadLibrary() error { + return loadLibrary(getLibraryPath()) +} + +// LoadLibraryFromPath loads the IBM DB2 CLI shared library from an absolute +// file system path, bypassing the platform linker cache. This is used by +// integration tests that extract libdb2.so.1 from the DB2 container image into +// a known directory (see db2test.ensureDB2Library). The load is still idempotent +// — if a previous LoadLibrary or LoadLibraryFromPath call succeeded, this is a +// no-op. Returns an error if the library cannot be opened or if any function +// symbol fails to resolve. +// +// path must be an absolute path. Relative paths and paths with ".." traversal +// components are rejected to prevent unintended library substitution. +func LoadLibraryFromPath(path string) error { + path = filepath.Clean(path) + if !filepath.IsAbs(path) { + return fmt.Errorf("LoadLibraryFromPath requires an absolute path, got %q", path) + } + return loadLibrary(path) +} + +func loadLibrary(libPath string) error { + loadOnce.Do(func() { + // RTLD_LOCAL (not RTLD_GLOBAL) prevents DB2 CLI symbols from leaking into + // the global symbol namespace, which would allow a malicious libdb2.so on + // LD_LIBRARY_PATH to hijack symbols used by other libraries. + libHandle, loadErr = purego.Dlopen(libPath, purego.RTLD_NOW|purego.RTLD_LOCAL) + if loadErr != nil { + loadErr = fmt.Errorf("loading DB2 CLI library %s: %w (ensure DB2 client is installed and library is in PATH/LD_LIBRARY_PATH)", libPath, loadErr) + return + } + + loadErr = registerFunctions() + if loadErr != nil { + loadErr = fmt.Errorf("registering DB2 CLI functions: %w", loadErr) + return + } + // libLoaded.Store is a release fence: any goroutine that reads + // libLoaded.Load() == true is guaranteed to observe the writes to + // libHandle and all registered function pointers above. + libLoaded.Store(true) + }) + + return loadErr +} + +// registerFunctions registers all DB2 CLI function pointers +func registerFunctions() error { + // This will register function pointers defined in functions.go + // Each function will be registered individually using purego.RegisterLibFunc + + if err := registerConnectionFunctions(); err != nil { + return fmt.Errorf("register connection functions: %w", err) + } + + if err := registerStatementFunctions(); err != nil { + return fmt.Errorf("register statement functions: %w", err) + } + + if err := registerResultFunctions(); err != nil { + return fmt.Errorf("register result functions: %w", err) + } + + if err := registerTransactionFunctions(); err != nil { + return fmt.Errorf("register transaction functions: %w", err) + } + + if err := registerDiagnosticFunctions(); err != nil { + return fmt.Errorf("register diagnostic functions: %w", err) + } + + if err := registerMetadataFunctions(); err != nil { + return fmt.Errorf("register metadata functions: %w", err) + } + + if err := registerCursorFunctions(); err != nil { + return fmt.Errorf("register cursor functions: %w", err) + } + + if err := registerDataAtExecFunctions(); err != nil { + return fmt.Errorf("register data-at-exec functions: %w", err) + } + + if err := registerExtendedConnFunctions(); err != nil { + return fmt.Errorf("register extended connection functions: %w", err) + } + + if err := registerDescriptorFunctions(); err != nil { + return fmt.Errorf("register descriptor functions: %w", err) + } + + return nil +} + +// IsLoaded reports whether the DB2 CLI shared library was loaded successfully +// and all function pointers were registered. Returns false if LoadLibrary has +// not been called yet or if the last call returned an error. +func IsLoaded() bool { + // libLoaded.Load() is an acquire fence: if it returns true, all writes made + // by loadLibrary (libHandle, function pointer registrations) are visible. + return libLoaded.Load() +} + +// GetLibraryHandle returns the raw dlopen handle for the loaded DB2 CLI shared +// library. This is an opaque uintptr that can be passed to purego.RegisterLibFunc +// for additional function registrations not covered by this package. Returns 0 +// if the library has not been loaded. +func GetLibraryHandle() uintptr { + // libLoaded.Load() ensures libHandle is visible if the library was loaded. + if !libLoaded.Load() { + return 0 + } + return libHandle +} diff --git a/internal/impl/db2/db2cli/errors.go b/internal/impl/db2/db2cli/errors.go new file mode 100644 index 0000000000..21a4fd528c --- /dev/null +++ b/internal/impl/db2/db2cli/errors.go @@ -0,0 +1,227 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2cli + +import ( + "errors" + "fmt" + "unsafe" +) + +// DB2Error represents a single DB2 CLI diagnostic record retrieved by +// SQLGetDiagRec. It implements the error interface and carries the five-character +// SQLSTATE code, a DB2-specific native error code, and the human-readable message. +// +// SQLSTATE is a standard five-character code defined by the SQL standard and +// extended by IBM. The first two characters identify the error class; the last +// three identify the sub-condition. IBM SQLSTATE reference: +// https://www.ibm.com/docs/en/db2/11.5?topic=messages-sqlstate +type DB2Error struct { + // SQLState is the five-character SQLSTATE code (e.g. "42710", "08006"). + SQLState string + // NativeError is the DB2-specific error code (SQLCODE). Negative values + // are errors; positive values are warnings. + NativeError int32 + // Message is the human-readable error message text from the DB2 message catalog. + Message string + // HandleType and Handle identify the handle on which the error occurred. + // Useful for routing the error to additional diagnostic queries. + HandleType SQLSMALLINT + Handle SQLHANDLE +} + +// Error implements the error interface, returning a formatted string containing +// the SQLSTATE, native error code, and message text. +func (e *DB2Error) Error() string { + return fmt.Sprintf("DB2 Error [%s] (Native: %d): %s", e.SQLState, e.NativeError, e.Message) +} + +// IsConnectionError reports whether this error is a connection exception. +// SQLSTATE class 08 covers all connection-related failures (e.g. 08001 = client +// unable to establish connection, 08006 = connection failure during transaction). +func (e *DB2Error) IsConnectionError() bool { + // SQLSTATE class 08 = Connection Exception + return len(e.SQLState) >= 2 && e.SQLState[:2] == "08" +} + +// IsIntegrityConstraintViolation reports whether this error is an integrity +// constraint violation. SQLSTATE class 23 covers unique-key violations (23505), +// foreign-key violations (23503), and other constraint errors. +func (e *DB2Error) IsIntegrityConstraintViolation() bool { + // SQLSTATE class 23 = Integrity Constraint Violation + return len(e.SQLState) >= 2 && e.SQLState[:2] == "23" +} + +// IsDeadlock reports whether this error is a deadlock or serialization failure. +// SQLSTATE 40001 is the standard code for deadlock detection; DB2 raises it when +// the lock manager detects a cycle and rolls back one of the involved transactions. +func (e *DB2Error) IsDeadlock() bool { + // SQLSTATE 40001 = Serialization failure (deadlock) + return e.SQLState == "40001" +} + +// IsTimeout reports whether this error is a query or connection timeout. +// SQLSTATE HYT00 = query timeout (SQL_ATTR_QUERY_TIMEOUT elapsed). +// SQLSTATE HYT01 = connection timeout (SQL_ATTR_CONNECTION_TIMEOUT elapsed). +func (e *DB2Error) IsTimeout() bool { + return e.SQLState == "HYT00" || e.SQLState == "HYT01" +} + +// GetDiagnostics collects all diagnostic records associated with handle after a +// CLI function failure. It calls SQLGetDiagRec in a loop (up to 64 records) and +// returns the accumulated records as a slice of DB2Error. The caller typically +// calls GetLastError when only the first (most specific) error is needed. +func GetDiagnostics(handleType SQLSMALLINT, handle SQLHANDLE) []DB2Error { + var errors []DB2Error + + for recNum := SQLSMALLINT(1); recNum <= 64; recNum++ { + sqlState := make([]byte, 6) + messageText := make([]byte, 1024) + var nativeError SQLINTEGER + var textLen SQLSMALLINT + + ret := SQLGetDiagRec( + handleType, + handle, + recNum, + (*SQLCHAR)(unsafe.Pointer(&sqlState[0])), + &nativeError, + (*SQLCHAR)(unsafe.Pointer(&messageText[0])), + 1024, + &textLen, + ) + + if ret == SQL_NO_DATA { + break + } + + if ret != SQL_SUCCESS && ret != SQL_SUCCESS_WITH_INFO { + break + } + + sqlStateStr := string(sqlState[:5]) // SQLSTATE is always 5 characters + // Clamp textLen: per ODBC spec textLen may be SQL_NO_TOTAL (-4) or any + // negative value when the driver cannot determine the true length. + // max(0, …) guards against a negative slice index panic. + n := max(0, min(int(textLen), len(messageText))) + messageStr := string(messageText[:n]) + + errors = append(errors, DB2Error{ + SQLState: sqlStateStr, + NativeError: int32(nativeError), + Message: messageStr, + HandleType: handleType, + Handle: handle, + }) + } + + return errors +} + +// GetLastError retrieves the first (primary) diagnostic record for handle and +// returns it as a Go error. When no diagnostic records are available (e.g. if +// the library is not loaded), a generic message is returned instead of nil to +// prevent silent failures. Callers use this after any CLI function returns +// SQL_ERROR to obtain a descriptive error value. +func GetLastError(handleType SQLSMALLINT, handle SQLHANDLE) error { + diags := GetDiagnostics(handleType, handle) + if len(diags) == 0 { + return errors.New("unknown DB2 error (no diagnostics available)") + } + return &diags[0] +} + +// CheckReturn maps a SQLRETURN value to a Go error. SQL_SUCCESS and +// SQL_SUCCESS_WITH_INFO both return nil (warnings are not surfaced as errors). +// SQL_NO_DATA returns nil because callers treat it as end-of-results, not an +// error. For SQL_ERROR and SQL_INVALID_HANDLE the diagnostic records are fetched +// via GetLastError and wrapped with the operation name for context. +func CheckReturn(ret SQLRETURN, handleType SQLSMALLINT, handle SQLHANDLE, operation string) error { + switch ret { + case SQL_SUCCESS: + return nil + case SQL_SUCCESS_WITH_INFO: + // Success with warnings - not an error, but diagnostics may be available + return nil + case SQL_NO_DATA: + // No data is not an error for fetch operations + return nil + case SQL_INVALID_HANDLE: + return fmt.Errorf("%s: invalid handle", operation) + case SQL_ERROR: + err := GetLastError(handleType, handle) + return fmt.Errorf("%s: %w", operation, err) + case SQL_STILL_EXECUTING: + return fmt.Errorf("%s: operation still executing", operation) + default: + return fmt.Errorf("%s: unexpected return code %d", operation, ret) + } +} + +// Common SQLSTATE constants for programmatic error classification. These are +// the five-character codes returned in the sqlState buffer by SQLGetDiagRec. +// The first two characters identify the error class; the last three identify +// the specific sub-condition. IBM SQLSTATE reference: +// https://www.ibm.com/docs/en/db2/11.5?topic=messages-sqlstate +const ( + // SQLSTATE_SUCCESS (class 00) indicates successful completion with no warnings. + SQLSTATE_SUCCESS = "00000" + + // Class 01: Warning + SQLSTATE_WARNING = "01000" + + // Class 02: No Data + SQLSTATE_NO_DATA = "02000" + + // Class 08: Connection Exception + SQLSTATE_CONNECTION_EXCEPTION = "08000" + SQLSTATE_CONNECTION_DOES_NOT_EXIST = "08003" + SQLSTATE_CONNECTION_FAILURE = "08006" + SQLSTATE_CONNECTION_NAME_IN_USE = "08002" + SQLSTATE_TRANSACTION_RESOLUTION = "08007" + + // Class 21: Cardinality Violation + SQLSTATE_CARDINALITY_VIOLATION = "21000" + + // Class 22: Data Exception + SQLSTATE_DATA_EXCEPTION = "22000" + SQLSTATE_STRING_DATA_RIGHT_TRUNC = "22001" + SQLSTATE_NUMERIC_VALUE_OUT_RANGE = "22003" + SQLSTATE_INVALID_DATETIME_FORMAT = "22007" + SQLSTATE_DIVISION_BY_ZERO = "22012" + + // Class 23: Integrity Constraint Violation + SQLSTATE_INTEGRITY_CONSTRAINT_VIOLATION = "23000" + SQLSTATE_UNIQUE_VIOLATION = "23505" + SQLSTATE_FOREIGN_KEY_VIOLATION = "23503" + + // Class 24: Invalid Cursor State + SQLSTATE_INVALID_CURSOR_STATE = "24000" + + // Class 25: Invalid Transaction State + SQLSTATE_INVALID_TRANSACTION_STATE = "25000" + + // Class 40: Transaction Rollback + SQLSTATE_SERIALIZATION_FAILURE = "40001" // Deadlock + SQLSTATE_TRANSACTION_ROLLBACK = "40000" + + // Class 42: Syntax Error or Access Rule Violation + SQLSTATE_SYNTAX_ERROR = "42000" + SQLSTATE_TABLE_OR_VIEW_NOT_FOUND = "42S02" + SQLSTATE_COLUMN_NOT_FOUND = "42S22" + SQLSTATE_DUPLICATE_COLUMN = "42S21" + SQLSTATE_PERMISSION_DENIED = "42501" + + // Class HY: CLI-specific errors + SQLSTATE_GENERAL_ERROR = "HY000" + SQLSTATE_MEMORY_ALLOCATION_ERROR = "HY001" + SQLSTATE_INVALID_HANDLE_TYPE = "HY092" + SQLSTATE_TIMEOUT_EXPIRED = "HYT00" + SQLSTATE_CONNECTION_TIMEOUT = "HYT01" +) diff --git a/internal/impl/db2/db2cli/errors_test.go b/internal/impl/db2/db2cli/errors_test.go new file mode 100644 index 0000000000..5419240ef4 --- /dev/null +++ b/internal/impl/db2/db2cli/errors_test.go @@ -0,0 +1,49 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2cli + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestGetDiagnosticsMessageTextClamp exercises the ODBC buffer-length contract: +// textLen from SQLGetDiagRec is the REQUIRED length before truncation, not how +// much was written. It may exceed the 1024-byte buffer, so we clamp before slicing. +func TestGetDiagnosticsMessageTextClamp(t *testing.T) { + msgBuf := make([]byte, 1024) + for i := range msgBuf { + msgBuf[i] = 'A' + } + textLen := SQLSMALLINT(2000) // server reports 2000 bytes needed; buffer is 1024 + + n := min(int(textLen), len(msgBuf)) + assert.NotPanics(t, func() { _ = string(msgBuf[:n]) }) + assert.Len(t, string(msgBuf[:n]), 1024) +} + +func TestGetDiagnosticsMessageTextExact(t *testing.T) { + // When textLen fits within the buffer the full message is used as-is. + msgBuf := make([]byte, 1024) + copy(msgBuf, "short message") + textLen := SQLSMALLINT(13) + + n := min(int(textLen), len(msgBuf)) + assert.Equal(t, "short message", string(msgBuf[:n])) +} + +func TestGetDiagnosticsMessageTextZero(t *testing.T) { + // textLen == 0 should produce an empty string without panic. + msgBuf := make([]byte, 1024) + textLen := SQLSMALLINT(0) + + n := min(int(textLen), len(msgBuf)) + assert.Empty(t, string(msgBuf[:n])) +} diff --git a/internal/impl/db2/db2cli/functions.go b/internal/impl/db2/db2cli/functions.go new file mode 100644 index 0000000000..de4a709fb7 --- /dev/null +++ b/internal/impl/db2/db2cli/functions.go @@ -0,0 +1,1152 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2cli + +import ( + "runtime" + "unsafe" + + "github.com/ebitengine/purego" +) + +// Function pointers for DB2 CLI functions +// These are registered at library load time + +// Connection Management Functions +var ( + sqlAllocHandle func(handleType SQLSMALLINT, inputHandle SQLHANDLE, outputHandle *SQLHANDLE) SQLRETURN + sqlFreeHandle func(handleType SQLSMALLINT, handle SQLHANDLE) SQLRETURN + sqlConnect func(hdbc SQLHDBC, dsn *SQLCHAR, dsnLen SQLSMALLINT, uid *SQLCHAR, uidLen SQLSMALLINT, pwd *SQLCHAR, pwdLen SQLSMALLINT) SQLRETURN + sqlDriverConnect func(hdbc SQLHDBC, hwnd uintptr, connStr *SQLCHAR, connStrLen SQLSMALLINT, outConnStr *SQLCHAR, outConnStrMax SQLSMALLINT, outConnStrLen *SQLSMALLINT, driverCompletion SQLUSMALLINT) SQLRETURN + sqlDisconnect func(hdbc SQLHDBC) SQLRETURN + sqlSetEnvAttr func(henv SQLHENV, attr SQLINTEGER, value uintptr, strLen SQLINTEGER) SQLRETURN + sqlSetConnectAttr func(hdbc SQLHDBC, attr SQLINTEGER, value uintptr, strLen SQLINTEGER) SQLRETURN + sqlGetConnectAttr func(hdbc SQLHDBC, attr SQLINTEGER, value SQLPOINTER, bufLen SQLINTEGER, strLen *SQLINTEGER) SQLRETURN + sqlGetInfo func(hdbc SQLHDBC, infoType SQLUSMALLINT, infoValue SQLPOINTER, bufLen SQLSMALLINT, strLen *SQLSMALLINT) SQLRETURN +) + +// Statement Functions +var ( + sqlExecDirect func(hstmt SQLHSTMT, sql *SQLCHAR, sqlLen SQLINTEGER) SQLRETURN + sqlPrepare func(hstmt SQLHSTMT, sql *SQLCHAR, sqlLen SQLINTEGER) SQLRETURN + sqlExecute func(hstmt SQLHSTMT) SQLRETURN + sqlFetch func(hstmt SQLHSTMT) SQLRETURN + sqlFetchScroll func(hstmt SQLHSTMT, fetchOrientation SQLSMALLINT, fetchOffset SQLLEN) SQLRETURN + sqlCloseCursor func(hstmt SQLHSTMT) SQLRETURN + sqlFreeStmt func(hstmt SQLHSTMT, option SQLUSMALLINT) SQLRETURN + sqlSetStmtAttr func(hstmt SQLHSTMT, attr SQLINTEGER, value uintptr, strLen SQLINTEGER) SQLRETURN + sqlGetStmtAttr func(hstmt SQLHSTMT, attr SQLINTEGER, value SQLPOINTER, bufLen SQLINTEGER, strLen *SQLINTEGER) SQLRETURN + sqlRowCount func(hstmt SQLHSTMT, rowCount *SQLLEN) SQLRETURN + sqlMoreResults func(hstmt SQLHSTMT) SQLRETURN + sqlCancel func(hstmt SQLHSTMT) SQLRETURN +) + +// Result Set Functions +var ( + sqlNumResultCols func(hstmt SQLHSTMT, colCount *SQLSMALLINT) SQLRETURN + sqlDescribeCol func(hstmt SQLHSTMT, colNum SQLUSMALLINT, colName *SQLCHAR, colNameMax SQLSMALLINT, colNameLen, dataType *SQLSMALLINT, colSize *SQLULEN, decimalDigits, nullable *SQLSMALLINT) SQLRETURN + sqlColAttribute func(hstmt SQLHSTMT, colNum, fieldID SQLUSMALLINT, charAttr SQLPOINTER, bufLen SQLSMALLINT, strLen *SQLSMALLINT, numAttr *SQLLEN) SQLRETURN + sqlBindCol func(hstmt SQLHSTMT, colNum SQLUSMALLINT, targetType SQLSMALLINT, targetValue SQLPOINTER, bufLen SQLLEN, indicator *SQLLEN) SQLRETURN + sqlGetData func(hstmt SQLHSTMT, colNum SQLUSMALLINT, targetType SQLSMALLINT, targetValue SQLPOINTER, bufLen SQLLEN, indicator *SQLLEN) SQLRETURN +) + +// Parameter Functions +var ( + sqlBindParameter func(hstmt SQLHSTMT, paramNum SQLUSMALLINT, inputOutputType, valueType, paramType SQLSMALLINT, colSize SQLULEN, decimalDigits SQLSMALLINT, paramValue SQLPOINTER, bufLen SQLLEN, indicator *SQLLEN) SQLRETURN + sqlNumParams func(hstmt SQLHSTMT, paramCount *SQLSMALLINT) SQLRETURN +) + +// Transaction Functions +var ( + sqlEndTran func(handleType SQLSMALLINT, handle SQLHANDLE, completionType SQLSMALLINT) SQLRETURN +) + +// Diagnostic Functions +var ( + sqlGetDiagRec func(handleType SQLSMALLINT, handle SQLHANDLE, recNum SQLSMALLINT, sqlState *SQLCHAR, nativeError *SQLINTEGER, messageText *SQLCHAR, bufLen SQLSMALLINT, textLen *SQLSMALLINT) SQLRETURN + sqlGetDiagField func(handleType SQLSMALLINT, handle SQLHANDLE, recNum, diagID SQLSMALLINT, diagInfo SQLPOINTER, bufLen SQLSMALLINT, strLen *SQLSMALLINT) SQLRETURN + sqlError func(henv SQLHENV, hdbc SQLHDBC, hstmt SQLHSTMT, sqlState *SQLCHAR, nativeError *SQLINTEGER, messageText *SQLCHAR, bufLen SQLSMALLINT, textLen *SQLSMALLINT) SQLRETURN +) + +// Metadata / Catalog Functions +var ( + sqlTables func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, tableName *SQLCHAR, tableNameLen SQLSMALLINT, tableType *SQLCHAR, tableTypeLen SQLSMALLINT) SQLRETURN + sqlColumns func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, tableName *SQLCHAR, tableNameLen SQLSMALLINT, columnName *SQLCHAR, columnNameLen SQLSMALLINT) SQLRETURN + sqlPrimaryKeys func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, tableName *SQLCHAR, tableNameLen SQLSMALLINT) SQLRETURN + sqlForeignKeys func(hstmt SQLHSTMT, pkCatalogName *SQLCHAR, pkCatalogNameLen SQLSMALLINT, pkSchemaName *SQLCHAR, pkSchemaNameLen SQLSMALLINT, pkTableName *SQLCHAR, pkTableNameLen SQLSMALLINT, fkCatalogName *SQLCHAR, fkCatalogNameLen SQLSMALLINT, fkSchemaName *SQLCHAR, fkSchemaNameLen SQLSMALLINT, fkTableName *SQLCHAR, fkTableNameLen SQLSMALLINT) SQLRETURN + sqlStatistics func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, tableName *SQLCHAR, tableNameLen SQLSMALLINT, unique SQLUSMALLINT, accuracy SQLUSMALLINT) SQLRETURN + sqlSpecialColumns func(hstmt SQLHSTMT, colType SQLUSMALLINT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, tableName *SQLCHAR, tableNameLen SQLSMALLINT, scope SQLUSMALLINT, nullable SQLUSMALLINT) SQLRETURN + sqlTablePrivileges func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, tableName *SQLCHAR, tableNameLen SQLSMALLINT) SQLRETURN + sqlColumnPrivileges func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, tableName *SQLCHAR, tableNameLen SQLSMALLINT, columnName *SQLCHAR, columnNameLen SQLSMALLINT) SQLRETURN + sqlProcedures func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, procName *SQLCHAR, procNameLen SQLSMALLINT) SQLRETURN + sqlProcedureColumns func(hstmt SQLHSTMT, catalogName *SQLCHAR, catalogNameLen SQLSMALLINT, schemaName *SQLCHAR, schemaNameLen SQLSMALLINT, procName *SQLCHAR, procNameLen SQLSMALLINT, columnName *SQLCHAR, columnNameLen SQLSMALLINT) SQLRETURN + sqlGetTypeInfo func(hstmt SQLHSTMT, dataType SQLSMALLINT) SQLRETURN + sqlGetFunctions func(hdbc SQLHDBC, function SQLUSMALLINT, supported *SQLUSMALLINT) SQLRETURN + sqlDataSources func(henv SQLHENV, direction SQLUSMALLINT, serverName *SQLCHAR, serverNameMax SQLSMALLINT, serverNameLen *SQLSMALLINT, description *SQLCHAR, descriptionMax SQLSMALLINT, descriptionLen *SQLSMALLINT) SQLRETURN + sqlDescribeParam func(hstmt SQLHSTMT, paramNum SQLUSMALLINT, dataType *SQLSMALLINT, paramSize *SQLULEN, decimalDigits *SQLSMALLINT, nullable *SQLSMALLINT) SQLRETURN +) + +// Cursor / Extended Fetch Functions +var ( + sqlExtendedFetch func(hstmt SQLHSTMT, fetchType SQLUSMALLINT, irow SQLLEN, rowCountPtr *SQLULEN, rowStatusArray *SQLUSMALLINT) SQLRETURN + sqlSetPos func(hstmt SQLHSTMT, rowNum SQLULEN, operation SQLUSMALLINT, lockType SQLUSMALLINT) SQLRETURN + sqlBulkOperations func(hstmt SQLHSTMT, operation SQLUSMALLINT) SQLRETURN + sqlNextResult func(hstmtSource SQLHSTMT, hstmtTarget SQLHSTMT) SQLRETURN +) + +// Data-at-Execution Functions +var ( + sqlParamData func(hstmt SQLHSTMT, value *SQLPOINTER) SQLRETURN + sqlPutData func(hstmt SQLHSTMT, data SQLPOINTER, strLen SQLLEN) SQLRETURN +) + +// Extended Connection Functions +var ( + sqlBrowseConnect func(hdbc SQLHDBC, connStrIn *SQLCHAR, connStrInLen SQLSMALLINT, connStrOut *SQLCHAR, connStrOutMax SQLSMALLINT, connStrOutLen *SQLSMALLINT) SQLRETURN + sqlNativeSql func(hdbc SQLHDBC, sqlStrIn *SQLCHAR, sqlStrInLen SQLINTEGER, sqlStr *SQLCHAR, sqlStrMax SQLINTEGER, sqlStrLen *SQLINTEGER) SQLRETURN + sqlGetEnvAttr func(henv SQLHENV, attr SQLINTEGER, value SQLPOINTER, bufLen SQLINTEGER, strLen *SQLINTEGER) SQLRETURN +) + +// Cursor Name Functions +var ( + sqlGetCursorName func(hstmt SQLHSTMT, cursor *SQLCHAR, cursorMax SQLSMALLINT, cursorLen *SQLSMALLINT) SQLRETURN + sqlSetCursorName func(hstmt SQLHSTMT, cursor *SQLCHAR, cursorLen SQLSMALLINT) SQLRETURN +) + +// Descriptor Functions +var ( + sqlCopyDesc func(srcDesc SQLHDESC, dstDesc SQLHDESC) SQLRETURN + sqlGetDescField func(descHandle SQLHDESC, recNum SQLSMALLINT, fieldID SQLSMALLINT, value SQLPOINTER, bufLen SQLINTEGER, strLen *SQLINTEGER) SQLRETURN + sqlSetDescField func(descHandle SQLHDESC, recNum SQLSMALLINT, fieldID SQLSMALLINT, value SQLPOINTER, bufLen SQLINTEGER) SQLRETURN + sqlGetDescRec func(descHandle SQLHDESC, recNum SQLSMALLINT, name *SQLCHAR, nameMax SQLSMALLINT, nameLen *SQLSMALLINT, dataType *SQLSMALLINT, subType *SQLSMALLINT, length *SQLLEN, precision *SQLSMALLINT, scale *SQLSMALLINT, nullable *SQLSMALLINT) SQLRETURN + sqlSetDescRec func(descHandle SQLHDESC, recNum SQLSMALLINT, dataType SQLSMALLINT, subType SQLSMALLINT, length SQLLEN, precision SQLSMALLINT, scale SQLSMALLINT, data SQLPOINTER, strLen *SQLLEN, indicator *SQLLEN) SQLRETURN +) + +// registerConnectionFunctions registers connection-related DB2 CLI functions +func registerConnectionFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlAllocHandle, "SQLAllocHandle"}, + {&sqlFreeHandle, "SQLFreeHandle"}, + {&sqlConnect, "SQLConnect"}, + {&sqlDriverConnect, "SQLDriverConnect"}, + {&sqlDisconnect, "SQLDisconnect"}, + {&sqlSetEnvAttr, "SQLSetEnvAttr"}, + {&sqlSetConnectAttr, "SQLSetConnectAttr"}, + {&sqlGetConnectAttr, "SQLGetConnectAttr"}, + {&sqlGetInfo, "SQLGetInfo"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerStatementFunctions registers statement-related DB2 CLI functions +func registerStatementFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlExecDirect, "SQLExecDirect"}, + {&sqlPrepare, "SQLPrepare"}, + {&sqlExecute, "SQLExecute"}, + {&sqlFetch, "SQLFetch"}, + {&sqlFetchScroll, "SQLFetchScroll"}, + {&sqlCloseCursor, "SQLCloseCursor"}, + {&sqlFreeStmt, "SQLFreeStmt"}, + {&sqlSetStmtAttr, "SQLSetStmtAttr"}, + {&sqlGetStmtAttr, "SQLGetStmtAttr"}, + {&sqlRowCount, "SQLRowCount"}, + {&sqlMoreResults, "SQLMoreResults"}, + {&sqlCancel, "SQLCancel"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerResultFunctions registers result set-related DB2 CLI functions +func registerResultFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlNumResultCols, "SQLNumResultCols"}, + {&sqlDescribeCol, "SQLDescribeCol"}, + {&sqlColAttribute, "SQLColAttribute"}, + {&sqlBindCol, "SQLBindCol"}, + {&sqlGetData, "SQLGetData"}, + {&sqlBindParameter, "SQLBindParameter"}, + {&sqlNumParams, "SQLNumParams"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerTransactionFunctions registers transaction-related DB2 CLI functions +func registerTransactionFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlEndTran, "SQLEndTran"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerDiagnosticFunctions registers diagnostic-related DB2 CLI functions +func registerDiagnosticFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlGetDiagRec, "SQLGetDiagRec"}, + {&sqlGetDiagField, "SQLGetDiagField"}, + {&sqlError, "SQLError"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerMetadataFunctions registers catalog/metadata DB2 CLI functions +func registerMetadataFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlTables, "SQLTables"}, + {&sqlColumns, "SQLColumns"}, + {&sqlPrimaryKeys, "SQLPrimaryKeys"}, + {&sqlForeignKeys, "SQLForeignKeys"}, + {&sqlStatistics, "SQLStatistics"}, + {&sqlSpecialColumns, "SQLSpecialColumns"}, + {&sqlTablePrivileges, "SQLTablePrivileges"}, + {&sqlColumnPrivileges, "SQLColumnPrivileges"}, + {&sqlProcedures, "SQLProcedures"}, + {&sqlProcedureColumns, "SQLProcedureColumns"}, + {&sqlGetTypeInfo, "SQLGetTypeInfo"}, + {&sqlGetFunctions, "SQLGetFunctions"}, + {&sqlDataSources, "SQLDataSources"}, + {&sqlDescribeParam, "SQLDescribeParam"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerCursorFunctions registers cursor and extended-fetch DB2 CLI functions +func registerCursorFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlExtendedFetch, "SQLExtendedFetch"}, + {&sqlSetPos, "SQLSetPos"}, + {&sqlBulkOperations, "SQLBulkOperations"}, + {&sqlNextResult, "SQLNextResult"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerDataAtExecFunctions registers data-at-execution DB2 CLI functions +func registerDataAtExecFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlParamData, "SQLParamData"}, + {&sqlPutData, "SQLPutData"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerExtendedConnFunctions registers extended connection and utility functions +func registerExtendedConnFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlBrowseConnect, "SQLBrowseConnect"}, + {&sqlNativeSql, "SQLNativeSql"}, + {&sqlGetEnvAttr, "SQLGetEnvAttr"}, + {&sqlGetCursorName, "SQLGetCursorName"}, + {&sqlSetCursorName, "SQLSetCursorName"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// registerDescriptorFunctions registers descriptor DB2 CLI functions +func registerDescriptorFunctions() error { + fns := []struct { + ptr any + name string + }{ + {&sqlCopyDesc, "SQLCopyDesc"}, + {&sqlGetDescField, "SQLGetDescField"}, + {&sqlSetDescField, "SQLSetDescField"}, + {&sqlGetDescRec, "SQLGetDescRec"}, + {&sqlSetDescRec, "SQLSetDescRec"}, + } + + for _, fn := range fns { + purego.RegisterLibFunc(fn.ptr, libHandle, fn.name) + } + + return nil +} + +// Public wrapper functions that call the registered function pointers. +// Each wrapper loads the DB2 CLI shared library on first call and returns +// SQL_ERROR immediately when the library is unavailable, so callers do not +// need a separate pre-flight check. + +// SQLAllocHandle allocates a new DB2 CLI handle of the type given by handleType +// (SQL_HANDLE_ENV, SQL_HANDLE_DBC, SQL_HANDLE_STMT, or SQL_HANDLE_DESC). +// For SQL_HANDLE_ENV pass SQL_NULL_HANDLE as inputHandle. For SQL_HANDLE_DBC +// pass a valid SQLHENV cast to SQLHANDLE. For SQL_HANDLE_STMT pass a valid +// SQLHDBC cast to SQLHANDLE. The allocated handle is written to *outputHandle. +// Every allocated handle must eventually be released with SQLFreeHandle to +// avoid resource leaks inside the DB2 client library. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlallochandle +// ODBC equivalent: SQLAllocHandle +func SQLAllocHandle(handleType SQLSMALLINT, inputHandle SQLHANDLE, outputHandle *SQLHANDLE) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlAllocHandle(handleType, inputHandle, outputHandle) +} + +// SQLFreeHandle releases a DB2 CLI handle previously allocated by SQLAllocHandle. +// For SQL_HANDLE_STMT it also closes any open cursor and discards pending results. +// For SQL_HANDLE_DBC the connection must already be disconnected via SQLDisconnect. +// For SQL_HANDLE_ENV all child connection handles must already be freed. +// Returns SQL_INVALID_HANDLE if the handle has already been freed or was never allocated. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlfreehandle +// ODBC equivalent: SQLFreeHandle +func SQLFreeHandle(handleType SQLSMALLINT, handle SQLHANDLE) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlFreeHandle(handleType, handle) +} + +// SQLConnect establishes a connection to a DB2 data source identified by dsn +// (a data-source name registered in db2dsdriver.cfg or odbc.ini). uid and pwd +// are the user ID and password. SQLDriverConnect is preferred for +// keyword=value connection strings; SQLConnect requires a pre-configured DSN. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlconnect +// ODBC equivalent: SQLConnect +func SQLConnect(hdbc SQLHDBC, dsn, uid, pwd string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + + dsnBytes := append([]byte(dsn), 0) + uidBytes := append([]byte(uid), 0) + pwdBytes := append([]byte(pwd), 0) + + ret := sqlConnect( + hdbc, + (*SQLCHAR)(unsafe.Pointer(&dsnBytes[0])), SQL_NTS, + (*SQLCHAR)(unsafe.Pointer(&uidBytes[0])), SQL_NTS, + (*SQLCHAR)(unsafe.Pointer(&pwdBytes[0])), SQL_NTS, + ) + runtime.KeepAlive(dsnBytes) + runtime.KeepAlive(uidBytes) + runtime.KeepAlive(pwdBytes) + // Zero credential buffers to shrink the window during which plaintext + // credentials linger as GC-reachable memory. + clear(pwdBytes) + clear(uidBytes) + return ret +} + +// SQLDriverConnect establishes a DB2 connection using a keyword=value connection +// string (e.g. "DATABASE=SAMPLE;HOSTNAME=host;PORT=50000;PROTOCOL=TCPIP;UID=u;PWD=p"). +// It is the preferred connection function because it supports all connection +// parameters without requiring a pre-configured data-source name. The completed +// connection string (with driver-supplied defaults) is returned as the first +// result value and may be longer than the input. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqldriverconnect +// ODBC equivalent: SQLDriverConnect +func SQLDriverConnect(hdbc SQLHDBC, connStr string) (string, SQLRETURN) { + if err := LoadLibrary(); err != nil { + return "", SQL_ERROR + } + + connStrBytes := append([]byte(connStr), 0) + outConnStr := make([]byte, 4096) // 4 KB — connection strings can exceed 1 KB + var outConnStrLen SQLSMALLINT + + ret := sqlDriverConnect( + hdbc, + 0, // No window handle + (*SQLCHAR)(unsafe.Pointer(&connStrBytes[0])), SQL_NTS, + (*SQLCHAR)(unsafe.Pointer(&outConnStr[0])), 4096, + &outConnStrLen, + 0, // SQL_DRIVER_NOPROMPT + ) + runtime.KeepAlive(connStrBytes) + runtime.KeepAlive(outConnStr) + // Zero the input connection string to reduce the window during which + // plaintext PWD= credentials linger as GC-reachable memory. + clear(connStrBytes) + + // Clamp output length: per ODBC spec outConnStrLen is the required length + // before truncation and may exceed buffer capacity. A misbehaving driver + // returning a negative length would cause a runtime panic without the + // max(0, …) guard — see the same pattern in driver.go:Columns(). + n := max(0, min(int(outConnStrLen), len(outConnStr))) + result := string(outConnStr[:n]) + // Zero the output buffer to reduce the window during which the plaintext + // PWD= keyword lingers as GC-reachable memory. Note: result already holds + // a copy of those bytes and will persist until the caller releases it. + clear(outConnStr) + return result, ret +} + +// SQLDisconnect closes an active connection to a DB2 data source. After +// SQLDisconnect returns SQL_SUCCESS, all open statement handles on hdbc are +// invalid and must be freed before the connection handle itself is freed. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqldisconnect +// ODBC equivalent: SQLDisconnect +func SQLDisconnect(hdbc SQLHDBC) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlDisconnect(hdbc) +} + +// SQLSetEnvAttr sets an attribute on an environment handle. The most important +// attribute is SQL_ATTR_ODBC_VERSION, which must be set to SQL_OV_ODBC3 before +// allocating connection handles. strLen is the byte length when value is a +// string pointer; use 0 for integer attributes. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlsetenvattr +// ODBC equivalent: SQLSetEnvAttr +func SQLSetEnvAttr(henv SQLHENV, attr SQLINTEGER, value uintptr, strLen SQLINTEGER) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlSetEnvAttr(henv, attr, value, strLen) +} + +// SQLSetConnectAttr sets an attribute on a connection handle. Commonly used to +// enable or disable autocommit (SQL_ATTR_AUTOCOMMIT) and to set the transaction +// isolation level (SQL_ATTR_TXN_ISOLATION). strLen is ignored for integer +// attributes; pass SQL_NTS for string attributes. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlsetconnectattr +// ODBC equivalent: SQLSetConnectAttr +func SQLSetConnectAttr(hdbc SQLHDBC, attr SQLINTEGER, value uintptr, strLen SQLINTEGER) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlSetConnectAttr(hdbc, attr, value, strLen) +} + +// SQLGetInfo retrieves general information about the connected DB2 driver and +// data source. infoType selects the property (e.g. SQL_DBMS_VER, SQL_DBMS_NAME). +// infoValue is a caller-allocated buffer; strLen receives the actual byte length +// of string results (may exceed bufLen, indicating truncation). +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlgetinfo +// ODBC equivalent: SQLGetInfo +func SQLGetInfo(hdbc SQLHDBC, infoType SQLUSMALLINT, infoValue SQLPOINTER, bufLen SQLSMALLINT, strLen *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetInfo(hdbc, infoType, infoValue, bufLen, strLen) +} + +// SQLExecDirect prepares and executes a SQL statement in a single call. +// Use SQLPrepare + SQLExecute when the same statement will be executed multiple +// times with different parameter values. SQLExecDirect is convenient for +// one-shot DDL (CREATE TABLE, DROP TABLE) where parse overhead is acceptable. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlexecdirect +// ODBC equivalent: SQLExecDirect +func SQLExecDirect(hstmt SQLHSTMT, sql string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + + sqlBytes := append([]byte(sql), 0) + ret := sqlExecDirect(hstmt, (*SQLCHAR)(unsafe.Pointer(&sqlBytes[0])), SQLINTEGER(len(sql))) + runtime.KeepAlive(sqlBytes) + return ret +} + +// SQLPrepare parses and compiles a SQL statement on the DB2 server and stores +// the access plan in hstmt. After SQLPrepare succeeds, call SQLBindParameter +// to bind input parameters (if any), then SQLExecute to run the statement. +// Reusing a prepared statement for repeated executions avoids re-parse overhead. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlprepare +// ODBC equivalent: SQLPrepare +func SQLPrepare(hstmt SQLHSTMT, sql string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + + sqlBytes := append([]byte(sql), 0) + ret := sqlPrepare(hstmt, (*SQLCHAR)(unsafe.Pointer(&sqlBytes[0])), SQLINTEGER(len(sql))) + runtime.KeepAlive(sqlBytes) + return ret +} + +// SQLExecute executes a statement previously prepared by SQLPrepare. All +// input parameters must be bound via SQLBindParameter before this call. +// For SELECT statements, the result set cursor is positioned before the first +// row; call SQLFetch to advance it. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlexecute +// ODBC equivalent: SQLExecute +func SQLExecute(hstmt SQLHSTMT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlExecute(hstmt) +} + +// SQLFetch advances the cursor to the next row in the result set and retrieves +// data for all bound columns (those bound via SQLBindCol). Returns SQL_NO_DATA +// when all rows have been consumed. For unbound columns, call SQLGetData after +// SQLFetch to retrieve each column value individually. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlfetch +// ODBC equivalent: SQLFetch +func SQLFetch(hstmt SQLHSTMT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlFetch(hstmt) +} + +// SQLFetchScroll positions the cursor using fetchOrientation (SQL_FETCH_NEXT, +// SQL_FETCH_FIRST, SQL_FETCH_PRIOR, SQL_FETCH_ABSOLUTE, SQL_FETCH_RELATIVE) +// and retrieves the rowset. Requires a scrollable cursor +// (SQL_ATTR_CURSOR_TYPE != SQL_CURSOR_FORWARD_ONLY). For forward-only cursors +// use SQLFetch instead. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlfetchscroll +// ODBC equivalent: SQLFetchScroll +func SQLFetchScroll(hstmt SQLHSTMT, fetchOrientation SQLSMALLINT, fetchOffset SQLLEN) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlFetchScroll(hstmt, fetchOrientation, fetchOffset) +} + +// SQLCloseCursor closes the cursor associated with hstmt and discards any +// unfetched rows. The statement handle remains allocated and may be re-executed. +// Returns SQLSTATE 24000 (invalid cursor state) if no cursor is open; the +// driver's db2Rows.Close method treats 24000 as a no-op. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlclosecursor +// ODBC equivalent: SQLCloseCursor +func SQLCloseCursor(hstmt SQLHSTMT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlCloseCursor(hstmt) +} + +// SQLFreeStmt performs one of several statement-handle cleanup operations +// selected by option: SQL_CLOSE (close cursor), SQL_DROP (close and free handle), +// SQL_UNBIND (release SQLBindCol bindings), or SQL_RESET_PARAMS (release +// SQLBindParameter bindings). Prefer SQLFreeHandle(SQL_HANDLE_STMT, hstmt) for +// full handle deallocation; SQLFreeStmt(SQL_DROP) is an alias. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlfreestmt +// ODBC equivalent: SQLFreeStmt +func SQLFreeStmt(hstmt SQLHSTMT, option SQLUSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlFreeStmt(hstmt, option) +} + +// SQLNumResultCols returns the number of columns in the result set produced by +// a prepared or executed SELECT statement. *colCount is set to 0 if the +// statement is not a SELECT (INSERT/UPDATE/DELETE). Call after SQLPrepare or +// SQLExecute, before the first SQLFetch. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlnumresultcols +// ODBC equivalent: SQLNumResultCols +func SQLNumResultCols(hstmt SQLHSTMT, colCount *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlNumResultCols(hstmt, colCount) +} + +// SQLDescribeCol returns the name, data type, column size, decimal digits, and +// nullability of column colNum (1-based) in the result set. colName is a +// caller-allocated buffer of colNameMax bytes; colNameLen receives the actual +// name length (may exceed colNameMax, indicating truncation). +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqldescribecol +// ODBC equivalent: SQLDescribeCol +func SQLDescribeCol(hstmt SQLHSTMT, colNum SQLUSMALLINT, colName *SQLCHAR, colNameMax SQLSMALLINT, colNameLen, dataType *SQLSMALLINT, colSize *SQLULEN, decimalDigits, nullable *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlDescribeCol(hstmt, colNum, colName, colNameMax, colNameLen, dataType, colSize, decimalDigits, nullable) +} + +// SQLGetData retrieves the value of column colNum (1-based) in the current row +// into targetValue, converting it to targetType (e.g. SQL_C_CHAR). If the value +// is longer than bufLen, DB2 returns SQL_SUCCESS_WITH_INFO and writes partial data; +// call SQLGetData again to retrieve the remainder. *indicator receives the total +// length of the value (SQL_NULL_DATA if the column is NULL, SQL_NO_TOTAL if the +// length is unknown). The driver's readColumnValue method loops on this behaviour. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlgetdata +// ODBC equivalent: SQLGetData +func SQLGetData(hstmt SQLHSTMT, colNum SQLUSMALLINT, targetType SQLSMALLINT, targetValue SQLPOINTER, bufLen SQLLEN, indicator *SQLLEN) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetData(hstmt, colNum, targetType, targetValue, bufLen, indicator) +} + +// SQLBindCol associates column colNum (1-based) with the application buffer at +// targetValue. On each SQLFetch, DB2 writes the converted column value directly +// into the bound buffer. This package uses SQLGetData exclusively (unbound +// columns) so SQLBindCol is provided for completeness; it is not called in the +// CDC streaming path. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlbindcol +// ODBC equivalent: SQLBindCol +func SQLBindCol(hstmt SQLHSTMT, colNum SQLUSMALLINT, targetType SQLSMALLINT, targetValue SQLPOINTER, bufLen SQLLEN, indicator *SQLLEN) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlBindCol(hstmt, colNum, targetType, targetValue, bufLen, indicator) +} + +// SQLRowCount returns the number of rows affected by the most recent INSERT, +// UPDATE, DELETE, or MERGE statement. For SELECT statements the value is -1 +// (unknown) until all rows have been fetched. *rowCount is populated on +// SQL_SUCCESS. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlrowcount +// ODBC equivalent: SQLRowCount +func SQLRowCount(hstmt SQLHSTMT, rowCount *SQLLEN) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlRowCount(hstmt, rowCount) +} + +// SQLEndTran commits or rolls back a transaction on the given handle. +// Pass SQL_HANDLE_DBC + the connection handle to commit/rollback all statements +// opened on that connection. completionType must be SQL_COMMIT or SQL_ROLLBACK. +// After SQLEndTran, autocommit should be restored via SQLSetConnectAttr if it +// was disabled for the transaction (the db2Tx methods handle this automatically). +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlendtran +// ODBC equivalent: SQLEndTran +func SQLEndTran(handleType SQLSMALLINT, handle SQLHANDLE, completionType SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlEndTran(handleType, handle, completionType) +} + +// SQLGetDiagRec retrieves the recNum-th (1-based) diagnostic record associated +// with handle after a CLI function failure. sqlState receives the five-character +// SQLSTATE string. nativeError receives the DB2-specific error code. messageText +// receives the human-readable message (bufLen bytes max); textLen receives the +// actual message length. Returns SQL_NO_DATA when all records have been read. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlgetdiagrec +// ODBC equivalent: SQLGetDiagRec +func SQLGetDiagRec(handleType SQLSMALLINT, handle SQLHANDLE, recNum SQLSMALLINT, sqlState *SQLCHAR, nativeError *SQLINTEGER, messageText *SQLCHAR, bufLen SQLSMALLINT, textLen *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetDiagRec(handleType, handle, recNum, sqlState, nativeError, messageText, bufLen, textLen) +} + +// SQLBindParameter binds the paramNum-th (1-based) parameter marker in the +// prepared statement to paramValue. inputOutputType is SQL_PARAM_INPUT for +// normal query parameters. valueType and paramType select the C buffer type and +// SQL column type respectively (SQL_C_CHAR / SQL_VARCHAR for all parameters in +// this driver). colSize is the column width; bufLen is the buffer capacity. +// The indicator argument must point to a SQLLEN that is kept alive until after +// SQLExecute returns (see bindParams in driver.go for the KeepAlive pattern). +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlbindparameter +// ODBC equivalent: SQLBindParameter +func SQLBindParameter(hstmt SQLHSTMT, paramNum SQLUSMALLINT, inputOutputType, valueType, paramType SQLSMALLINT, colSize SQLULEN, decimalDigits SQLSMALLINT, paramValue SQLPOINTER, bufLen SQLLEN, indicator *SQLLEN) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlBindParameter(hstmt, paramNum, inputOutputType, valueType, paramType, colSize, decimalDigits, paramValue, bufLen, indicator) +} + +// SQLCancel requests cancellation of a SQL statement that is currently executing +// on hstmt. It is safe to call from a different goroutine while another goroutine +// is blocked inside SQLExecute or SQLFetch — the DB2 CLI library signals an +// interrupt and the blocked call returns SQL_ERROR (SQLSTATE HY008). The +// QueryContext implementation in driver.go uses this to honour context cancellation. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlcancel +// ODBC equivalent: SQLCancel +func SQLCancel(hstmt SQLHSTMT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlCancel(hstmt) +} + +// SQLGetConnectAttr retrieves the current value of a connection attribute set +// by SQLSetConnectAttr. For integer attributes value should point to a uintptr; +// for string attributes it should point to a byte buffer of bufLen bytes. +// strLen receives the actual byte length of string results. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=functions-sqlgetconnectattr +// ODBC equivalent: SQLGetConnectAttr +func SQLGetConnectAttr(hdbc SQLHDBC, attr SQLINTEGER, value SQLPOINTER, bufLen SQLINTEGER, strLen *SQLINTEGER) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetConnectAttr(hdbc, attr, value, bufLen, strLen) +} + +// --- Metadata / Catalog wrappers --- + +// SQLTables returns a list of tables stored in the data source. +func SQLTables(hstmt SQLHSTMT, catalogName, schemaName, tableName, tableType string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + tbl, tblLen, b2 := sqlStrPtr(tableName) + typ, typLen, b3 := sqlStrPtr(tableType) + ret := sqlTables(hstmt, cat, catLen, sch, schLen, tbl, tblLen, typ, typLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + runtime.KeepAlive(b3) + return ret +} + +// SQLColumns returns a list of columns in the specified tables. +func SQLColumns(hstmt SQLHSTMT, catalogName, schemaName, tableName, columnName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + tbl, tblLen, b2 := sqlStrPtr(tableName) + col, colLen, b3 := sqlStrPtr(columnName) + ret := sqlColumns(hstmt, cat, catLen, sch, schLen, tbl, tblLen, col, colLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + runtime.KeepAlive(b3) + return ret +} + +// SQLPrimaryKeys returns the column names that make up the primary key of a table. +func SQLPrimaryKeys(hstmt SQLHSTMT, catalogName, schemaName, tableName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + tbl, tblLen, b2 := sqlStrPtr(tableName) + ret := sqlPrimaryKeys(hstmt, cat, catLen, sch, schLen, tbl, tblLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + return ret +} + +// SQLForeignKeys returns the foreign key columns of a table and the primary key columns they reference. +func SQLForeignKeys(hstmt SQLHSTMT, pkCatalogName, pkSchemaName, pkTableName, fkCatalogName, fkSchemaName, fkTableName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + pkCat, pkCatLen, b0 := sqlStrPtr(pkCatalogName) + pkSch, pkSchLen, b1 := sqlStrPtr(pkSchemaName) + pkTbl, pkTblLen, b2 := sqlStrPtr(pkTableName) + fkCat, fkCatLen, b3 := sqlStrPtr(fkCatalogName) + fkSch, fkSchLen, b4 := sqlStrPtr(fkSchemaName) + fkTbl, fkTblLen, b5 := sqlStrPtr(fkTableName) + ret := sqlForeignKeys(hstmt, pkCat, pkCatLen, pkSch, pkSchLen, pkTbl, pkTblLen, fkCat, fkCatLen, fkSch, fkSchLen, fkTbl, fkTblLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + runtime.KeepAlive(b3) + runtime.KeepAlive(b4) + runtime.KeepAlive(b5) + return ret +} + +// SQLStatistics returns statistics about a table and the indexes on it. +func SQLStatistics(hstmt SQLHSTMT, catalogName, schemaName, tableName string, unique, accuracy SQLUSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + tbl, tblLen, b2 := sqlStrPtr(tableName) + ret := sqlStatistics(hstmt, cat, catLen, sch, schLen, tbl, tblLen, unique, accuracy) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + return ret +} + +// SQLSpecialColumns returns information about the optimal set of columns that uniquely identify a row. +func SQLSpecialColumns(hstmt SQLHSTMT, colType SQLUSMALLINT, catalogName, schemaName, tableName string, scope, nullable SQLUSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + tbl, tblLen, b2 := sqlStrPtr(tableName) + ret := sqlSpecialColumns(hstmt, colType, cat, catLen, sch, schLen, tbl, tblLen, scope, nullable) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + return ret +} + +// SQLTablePrivileges returns the table privileges for one or more tables. +func SQLTablePrivileges(hstmt SQLHSTMT, catalogName, schemaName, tableName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + tbl, tblLen, b2 := sqlStrPtr(tableName) + ret := sqlTablePrivileges(hstmt, cat, catLen, sch, schLen, tbl, tblLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + return ret +} + +// SQLColumnPrivileges returns the column privileges for one or more columns in a table. +func SQLColumnPrivileges(hstmt SQLHSTMT, catalogName, schemaName, tableName, columnName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + tbl, tblLen, b2 := sqlStrPtr(tableName) + col, colLen, b3 := sqlStrPtr(columnName) + ret := sqlColumnPrivileges(hstmt, cat, catLen, sch, schLen, tbl, tblLen, col, colLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + runtime.KeepAlive(b3) + return ret +} + +// SQLProcedures returns the list of procedure names stored in the data source. +func SQLProcedures(hstmt SQLHSTMT, catalogName, schemaName, procName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + prc, prcLen, b2 := sqlStrPtr(procName) + ret := sqlProcedures(hstmt, cat, catLen, sch, schLen, prc, prcLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + return ret +} + +// SQLProcedureColumns returns the list of input and output parameters for a procedure. +func SQLProcedureColumns(hstmt SQLHSTMT, catalogName, schemaName, procName, columnName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + cat, catLen, b0 := sqlStrPtr(catalogName) + sch, schLen, b1 := sqlStrPtr(schemaName) + prc, prcLen, b2 := sqlStrPtr(procName) + col, colLen, b3 := sqlStrPtr(columnName) + ret := sqlProcedureColumns(hstmt, cat, catLen, sch, schLen, prc, prcLen, col, colLen) + runtime.KeepAlive(b0) + runtime.KeepAlive(b1) + runtime.KeepAlive(b2) + runtime.KeepAlive(b3) + return ret +} + +// SQLGetTypeInfo returns information about data types supported by the data source. +func SQLGetTypeInfo(hstmt SQLHSTMT, dataType SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetTypeInfo(hstmt, dataType) +} + +// SQLGetFunctions returns information about whether a driver supports a specific ODBC function. +func SQLGetFunctions(hdbc SQLHDBC, function SQLUSMALLINT, supported *SQLUSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetFunctions(hdbc, function, supported) +} + +// SQLDataSources returns a list of data source names. +func SQLDataSources(henv SQLHENV, direction SQLUSMALLINT, serverName *SQLCHAR, serverNameMax SQLSMALLINT, serverNameLen *SQLSMALLINT, description *SQLCHAR, descriptionMax SQLSMALLINT, descriptionLen *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlDataSources(henv, direction, serverName, serverNameMax, serverNameLen, description, descriptionMax, descriptionLen) +} + +// SQLDescribeParam returns the description of a parameter marker in an SQL statement. +func SQLDescribeParam(hstmt SQLHSTMT, paramNum SQLUSMALLINT, dataType *SQLSMALLINT, paramSize *SQLULEN, decimalDigits *SQLSMALLINT, nullable *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlDescribeParam(hstmt, paramNum, dataType, paramSize, decimalDigits, nullable) +} + +// --- Cursor / Extended Fetch wrappers --- + +// SQLExtendedFetch fetches the specified rowset of data from the result set. +// +// Deprecated: superseded by SQLFetchScroll; retained for legacy code. +func SQLExtendedFetch(hstmt SQLHSTMT, fetchType SQLUSMALLINT, irow SQLLEN, rowCountPtr *SQLULEN, rowStatusArray *SQLUSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlExtendedFetch(hstmt, fetchType, irow, rowCountPtr, rowStatusArray) +} + +// SQLSetPos sets the cursor position within a rowset and allows an application to refresh, update, or delete data. +func SQLSetPos(hstmt SQLHSTMT, rowNum SQLULEN, operation SQLUSMALLINT, lockType SQLUSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlSetPos(hstmt, rowNum, operation, lockType) +} + +// SQLBulkOperations performs bulk insertions and bulk bookmark operations. +func SQLBulkOperations(hstmt SQLHSTMT, operation SQLUSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlBulkOperations(hstmt, operation) +} + +// SQLNextResult moves to the next result set when a stored procedure returns multiple result sets. +func SQLNextResult(hstmtSource SQLHSTMT, hstmtTarget SQLHSTMT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlNextResult(hstmtSource, hstmtTarget) +} + +// --- Data-at-Execution wrappers --- + +// SQLParamData is used in conjunction with SQLPutData to supply parameter data at execution time. +func SQLParamData(hstmt SQLHSTMT, value *SQLPOINTER) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlParamData(hstmt, value) +} + +// SQLPutData allows an application to send data for a parameter or column at statement execution time. +func SQLPutData(hstmt SQLHSTMT, data SQLPOINTER, strLen SQLLEN) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlPutData(hstmt, data, strLen) +} + +// --- Extended Connection / Utility wrappers --- + +// SQLBrowseConnect returns successive levels of connection attributes and valid attribute values. +func SQLBrowseConnect(hdbc SQLHDBC, connStrIn string) (string, SQLRETURN) { + if err := LoadLibrary(); err != nil { + return "", SQL_ERROR + } + connStrInBytes := append([]byte(connStrIn), 0) + outBuf := make([]byte, 4096) + var outLen SQLSMALLINT + ret := sqlBrowseConnect( + hdbc, + (*SQLCHAR)(unsafe.Pointer(&connStrInBytes[0])), SQL_NTS, + (*SQLCHAR)(unsafe.Pointer(&outBuf[0])), 4096, + &outLen, + ) + runtime.KeepAlive(connStrInBytes) + // Zero input connection string (may contain PWD= credentials) and output buffer + // to reduce the window during which plaintext credentials linger as GC-reachable + // memory. Note: result already holds a copy of outBuf bytes and persists until + // the caller releases its reference. + clear(connStrInBytes) + // max(0, …): ODBC spec allows the driver to write SQL_NO_TOTAL (-4) into + // outLen to indicate truncation; guard against a negative slice index. + n := max(0, min(int(outLen), len(outBuf))) + result := string(outBuf[:n]) + clear(outBuf) + return result, ret +} + +// SQLNativeSql translates an SQL string containing escape sequences into the native SQL grammar of the data source. +func SQLNativeSql(hdbc SQLHDBC, sqlStrIn string) (string, SQLRETURN) { + if err := LoadLibrary(); err != nil { + return "", SQL_ERROR + } + inBytes := append([]byte(sqlStrIn), 0) + outBuf := make([]byte, 4096) + var outLen SQLINTEGER + ret := sqlNativeSql( + hdbc, + (*SQLCHAR)(unsafe.Pointer(&inBytes[0])), SQL_NTS, + (*SQLCHAR)(unsafe.Pointer(&outBuf[0])), 4096, + &outLen, + ) + // max(0, …): guard against SQL_NO_TOTAL (-4) or other negative indicator values. + n := max(0, min(int(outLen), len(outBuf))) + return string(outBuf[:n]), ret +} + +// SQLGetEnvAttr returns the current setting of an environment attribute. +func SQLGetEnvAttr(henv SQLHENV, attr SQLINTEGER, value SQLPOINTER, bufLen SQLINTEGER, strLen *SQLINTEGER) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetEnvAttr(henv, attr, value, bufLen, strLen) +} + +// SQLGetCursorName returns the cursor name associated with a statement handle. +func SQLGetCursorName(hstmt SQLHSTMT, cursor *SQLCHAR, cursorMax SQLSMALLINT, cursorLen *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetCursorName(hstmt, cursor, cursorMax, cursorLen) +} + +// SQLSetCursorName associates a cursor name with a statement handle. +func SQLSetCursorName(hstmt SQLHSTMT, cursorName string) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + b := append([]byte(cursorName), 0) + return sqlSetCursorName(hstmt, (*SQLCHAR)(unsafe.Pointer(&b[0])), SQL_NTS) +} + +// --- Descriptor wrappers --- + +// SQLCopyDesc copies descriptor information from one descriptor handle to another. +func SQLCopyDesc(srcDesc SQLHDESC, dstDesc SQLHDESC) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlCopyDesc(srcDesc, dstDesc) +} + +// SQLGetDescField returns the current setting or value of a single field of a descriptor record. +func SQLGetDescField(descHandle SQLHDESC, recNum SQLSMALLINT, fieldID SQLSMALLINT, value SQLPOINTER, bufLen SQLINTEGER, strLen *SQLINTEGER) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetDescField(descHandle, recNum, fieldID, value, bufLen, strLen) +} + +// SQLSetDescField sets the value of a single field of a descriptor record. +func SQLSetDescField(descHandle SQLHDESC, recNum SQLSMALLINT, fieldID SQLSMALLINT, value SQLPOINTER, bufLen SQLINTEGER) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlSetDescField(descHandle, recNum, fieldID, value, bufLen) +} + +// SQLGetDescRec returns the current settings or values of the multiple fields of a descriptor record. +func SQLGetDescRec(descHandle SQLHDESC, recNum SQLSMALLINT, name *SQLCHAR, nameMax SQLSMALLINT, nameLen *SQLSMALLINT, dataType *SQLSMALLINT, subType *SQLSMALLINT, length *SQLLEN, precision *SQLSMALLINT, scale *SQLSMALLINT, nullable *SQLSMALLINT) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlGetDescRec(descHandle, recNum, name, nameMax, nameLen, dataType, subType, length, precision, scale, nullable) +} + +// SQLSetDescRec sets multiple descriptor fields that affect the data type and buffer bound to a column or parameter data. +func SQLSetDescRec(descHandle SQLHDESC, recNum SQLSMALLINT, dataType SQLSMALLINT, subType SQLSMALLINT, length SQLLEN, precision SQLSMALLINT, scale SQLSMALLINT, data SQLPOINTER, strLen *SQLLEN, indicator *SQLLEN) SQLRETURN { + if err := LoadLibrary(); err != nil { + return SQL_ERROR + } + return sqlSetDescRec(descHandle, recNum, dataType, subType, length, precision, scale, data, strLen, indicator) +} + +// sqlStrPtr converts a Go string to a *SQLCHAR pointer, length sentinel, and +// the backing []byte buffer. Callers MUST call runtime.KeepAlive on the returned +// buffer after the native ODBC function returns — the GC does not trace +// unsafe.Pointer and could collect the buffer mid-call without the keepalive. +// An empty string returns (nil, 0, nil) — the ODBC convention for "no filter". +// +// SQL_NTS (-3) is the ODBC sentinel for "null-terminated string": it instructs +// the driver to measure the length itself. This avoids a SQLSMALLINT (16-bit) +// overflow that would silently truncate or sign-flip any string ≥ 32 KiB. +func sqlStrPtr(s string) (*SQLCHAR, SQLSMALLINT, []byte) { + if s == "" { + return nil, 0, nil + } + b := append([]byte(s), 0) + return (*SQLCHAR)(unsafe.Pointer(&b[0])), SQL_NTS, b +} diff --git a/internal/impl/db2/db2cli/functions_test.go b/internal/impl/db2/db2cli/functions_test.go new file mode 100644 index 0000000000..f33cdff4b3 --- /dev/null +++ b/internal/impl/db2/db2cli/functions_test.go @@ -0,0 +1,270 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2cli + +import ( + "runtime" + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +// resetLibraryState resets the package-level load state so individual tests +// can simulate an unloaded library without interference. +func resetLibraryState() { + libHandle = 0 + loadOnce = sync.Once{} + loadErr = nil +} + +// TestNewFunctionsReturnSQLErrorWhenLibraryNotLoaded verifies that every new +// public wrapper returns SQL_ERROR (not panics) when the DB2 library has not +// been loaded. This is the fallback contract for deployment environments where +// libdb2 is not installed. +func TestNewFunctionsReturnSQLErrorWhenLibraryNotLoaded(t *testing.T) { + resetLibraryState() + + t.Run("SQLGetConnectAttr", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLGetConnectAttr(0, 0, nil, 0, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLGetTypeInfo", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLGetTypeInfo(0, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLGetFunctions", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLGetFunctions(0, 0, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLDataSources", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLDataSources(0, 0, nil, 0, nil, nil, 0, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLDescribeParam", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLDescribeParam(0, 0, nil, nil, nil, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLTables", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLTables(0, "", "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLColumns", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLColumns(0, "", "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLPrimaryKeys", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLPrimaryKeys(0, "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLForeignKeys", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLForeignKeys(0, "", "", "", "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLStatistics", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLStatistics(0, "", "", "", 0, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLSpecialColumns", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLSpecialColumns(0, 0, "", "", "", 0, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLTablePrivileges", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLTablePrivileges(0, "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLColumnPrivileges", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLColumnPrivileges(0, "", "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLProcedures", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLProcedures(0, "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLProcedureColumns", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLProcedureColumns(0, "", "", "", "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLExtendedFetch", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLExtendedFetch(0, 0, 0, nil, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLSetPos", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLSetPos(0, 0, 0, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLBulkOperations", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLBulkOperations(0, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLNextResult", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLNextResult(0, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLParamData", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLParamData(0, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLPutData", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLPutData(0, nil, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLBrowseConnect", func(t *testing.T) { + assert.NotPanics(t, func() { + _, ret := SQLBrowseConnect(0, "") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLNativeSql", func(t *testing.T) { + assert.NotPanics(t, func() { + _, ret := SQLNativeSql(0, "SELECT 1") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLGetEnvAttr", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLGetEnvAttr(0, 0, nil, 0, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLGetCursorName", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLGetCursorName(0, nil, 0, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLSetCursorName", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLSetCursorName(0, "my_cursor") + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLCopyDesc", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLCopyDesc(0, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLGetDescField", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLGetDescField(0, 0, 0, nil, 0, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLSetDescField", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLSetDescField(0, 0, 0, nil, 0) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLGetDescRec", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLGetDescRec(0, 0, nil, 0, nil, nil, nil, nil, nil, nil, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) + + t.Run("SQLSetDescRec", func(t *testing.T) { + assert.NotPanics(t, func() { + ret := SQLSetDescRec(0, 0, 0, 0, 0, 0, 0, nil, nil, nil) + assert.Equal(t, SQL_ERROR, ret) + }) + }) +} + +// TestSqlStrPtrEmptyString verifies that sqlStrPtr returns nil for an empty string, +// matching the ODBC convention for "no filter" in catalog functions. +func TestSqlStrPtrEmptyString(t *testing.T) { + ptr, length, buf := sqlStrPtr("") + assert.Nil(t, ptr) + assert.Equal(t, SQLSMALLINT(0), length) + assert.Nil(t, buf) +} + +// TestSqlStrPtrNonEmpty verifies that sqlStrPtr returns a non-nil pointer and +// SQL_NTS as the length sentinel so that the DB2 driver measures the string +// itself, avoiding SQLSMALLINT overflow for strings ≥ 32 KiB. +func TestSqlStrPtrNonEmpty(t *testing.T) { + ptr, length, buf := sqlStrPtr("hello") + assert.NotNil(t, ptr) + assert.Equal(t, SQLSMALLINT(SQL_NTS), length) + assert.NotNil(t, buf) + runtime.KeepAlive(buf) +} diff --git a/internal/impl/db2/db2cli/types.go b/internal/impl/db2/db2cli/types.go new file mode 100644 index 0000000000..e928ede4e1 --- /dev/null +++ b/internal/impl/db2/db2cli/types.go @@ -0,0 +1,174 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2cli + +import "unsafe" + +// DB2 CLI type aliases — Go equivalents of the C types used in the IBM DB2 +// CLI (Call Level Interface) API. All handle types are opaque pointer-sized +// values allocated by SQLAllocHandle and freed by SQLFreeHandle. +// +// IBM DB2 CLI type reference: +// https://www.ibm.com/docs/en/db2/11.5?topic=reference-cli-data-types +// +// ODBC equivalent reference (DB2 CLI mirrors ODBC): +// https://learn.microsoft.com/en-us/sql/odbc/reference/appendixes/c-data-types + +// SQLHANDLE is the generic opaque handle type, equivalent to C SQLHANDLE +// (void*). All specific handle types (SQLHENV, SQLHDBC, SQLHSTMT, SQLHDESC) +// are named subtypes of SQLHANDLE. The zero value represents SQL_NULL_HANDLE. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=reference-handle-types +type SQLHANDLE uintptr + +// SQLHENV is an environment handle allocated by +// SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &henv). It holds global +// state including the ODBC version (SQL_ATTR_ODBC_VERSION) and must be +// allocated before any connection handles. One environment handle per process +// is the typical pattern. Equivalent to C SQLHENV. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=handles-environment +type SQLHENV SQLHANDLE + +// SQLHDBC is a connection handle allocated by +// SQLAllocHandle(SQL_HANDLE_DBC, henv, &hdbc). It represents a single +// physical TCP/IP connection to a DB2 database. Transaction state +// (SQL_ATTR_AUTOCOMMIT, SQL_ATTR_TXN_ISOLATION) is per-connection. Equivalent +// to C SQLHDBC. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=handles-connection +type SQLHDBC SQLHANDLE + +// SQLHSTMT is a statement handle allocated by +// SQLAllocHandle(SQL_HANDLE_STMT, hdbc, &hstmt). It represents one prepared +// or executed SQL statement and owns the associated result-set cursor. Multiple +// statement handles may share one connection handle; each has its own cursor +// state. Equivalent to C SQLHSTMT. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=handles-statement +type SQLHSTMT SQLHANDLE + +// SQLHDESC is a descriptor handle that describes the attributes (type, length, +// precision, nullability) of a parameter or column. Descriptor handles are +// implicitly allocated when a statement handle is created; they can also be +// explicitly allocated for advanced use cases. Equivalent to C SQLHDESC. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=handles-descriptor +type SQLHDESC SQLHANDLE + +// SQLRETURN is the return code type for all DB2 CLI functions. Every function +// returns one of: SQL_SUCCESS, SQL_SUCCESS_WITH_INFO, SQL_NO_DATA, +// SQL_ERROR, SQL_INVALID_HANDLE, SQL_STILL_EXECUTING, or SQL_NEED_DATA. +// Equivalent to C SQLRETURN (signed 16-bit integer). +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=cli-return-codes +type SQLRETURN int16 + +// SQLSMALLINT is a signed 16-bit integer used for handle type selectors, +// column counts, and small integer parameters. Equivalent to C SQLSMALLINT. +type SQLSMALLINT int16 + +// SQLUSMALLINT is an unsigned 16-bit integer used for column numbers (1-based) +// and unsigned small integer parameters. Equivalent to C SQLUSMALLINT. +type SQLUSMALLINT uint16 + +// SQLINTEGER is a signed 32-bit integer used for attribute values, string +// lengths, and large integer parameters. Equivalent to C SQLINTEGER. +type SQLINTEGER int32 + +// SQLUINTEGER is an unsigned 32-bit integer. Equivalent to C SQLUINTEGER. +type SQLUINTEGER uint32 + +// SQLLEN is a signed 64-bit integer on 64-bit platforms (32-bit on 32-bit +// platforms). Used for buffer lengths, row counts, and indicator variables. +// This package targets 64-bit only; the type is always int64. Equivalent to +// C SQLLEN. +type SQLLEN int64 + +// SQLULEN is an unsigned 64-bit integer on 64-bit platforms. Used for column +// sizes and unsigned length values. Equivalent to C SQLULEN. +type SQLULEN uint64 + +// SQLPOINTER is a generic void pointer for buffers passed to SQLGetData, +// SQLBindCol, and SQLBindParameter. Equivalent to C SQLPOINTER (void*). +// The caller is responsible for keeping the underlying allocation alive via +// runtime.KeepAlive until the CLI function returns. +type SQLPOINTER unsafe.Pointer + +// SQLCHAR is a single unsigned byte, equivalent to C SQLCHAR (unsigned char). +// Strings are represented as *SQLCHAR pointing to a null-terminated byte slice. +type SQLCHAR byte + +// SQLWCHAR is a two-byte wide character for Unicode (UCS-2) string parameters. +// Equivalent to C SQLWCHAR. Not used in this package; all parameters use +// SQL_C_CHAR (single-byte) to avoid the complexity of UTF-16 conversion. +type SQLWCHAR uint16 + +// SQLBigInt is a signed 64-bit integer. Equivalent to C SQLBIGINT. +type SQLBigInt int64 + +// SQLUBigInt is an unsigned 64-bit integer. Equivalent to C SQLUBIGINT. +type SQLUBigInt uint64 + +// SQLReal is a 32-bit IEEE 754 single-precision floating-point value. +// Equivalent to C SQLREAL (float). +type SQLReal float32 + +// SQLDouble is a 64-bit IEEE 754 double-precision floating-point value. +// Equivalent to C SQLDOUBLE (double). +type SQLDouble float64 + +// SQLDate represents a DB2 DATE value (year, month, day). The layout matches +// the C DATE_STRUCT used with SQL_C_DATE in SQLBindCol / SQLGetData. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=data-date-time-timestamp-types +type SQLDate struct { + Year SQLSMALLINT // e.g. 2024 + Month SQLUSMALLINT // 1–12 + Day SQLUSMALLINT // 1–31 +} + +// SQLTime represents a DB2 TIME value (hour, minute, second). The layout +// matches the C TIME_STRUCT used with SQL_C_TIME in SQLBindCol / SQLGetData. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=data-date-time-timestamp-types +type SQLTime struct { + Hour SQLUSMALLINT // 0–23 + Minute SQLUSMALLINT // 0–59 + Second SQLUSMALLINT // 0–59 +} + +// SQLTimestamp represents a DB2 TIMESTAMP value. The layout matches the C +// TIMESTAMP_STRUCT used with SQL_C_TIMESTAMP in SQLBindCol / SQLGetData. +// Fraction holds the sub-second part in nanoseconds (0–999,999,999), even +// though DB2 TIMESTAMP has only microsecond precision (0–999,999 µs). +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=data-date-time-timestamp-types +type SQLTimestamp struct { + Year SQLSMALLINT + Month SQLUSMALLINT + Day SQLUSMALLINT + Hour SQLUSMALLINT + Minute SQLUSMALLINT + Second SQLUSMALLINT + Fraction SQLUINTEGER // sub-second fraction in nanoseconds +} + +// SQLNumeric represents a DB2 NUMERIC / DECIMAL value in the binary SQL_C_NUMERIC +// format used by SQLBindCol. Precision is the total number of significant digits; +// Scale is the number of digits to the right of the decimal point; Sign is 1 for +// positive and 0 for negative; Val holds the absolute value in little-endian binary. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=data-numeric-decimal-types +type SQLNumeric struct { + Precision SQLCHAR // total significant digits + Scale SQLCHAR // digits after the decimal point + Sign SQLCHAR // 1 = positive, 0 = negative + Val [16]SQLCHAR // absolute value, little-endian binary +} diff --git a/internal/impl/db2/db2test/db2test.go b/internal/impl/db2/db2test/db2test.go new file mode 100644 index 0000000000..e063fa5f86 --- /dev/null +++ b/internal/impl/db2/db2test/db2test.go @@ -0,0 +1,1916 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +// Package db2test provides helpers for integration tests against a live IBM DB2 +// instance running in a Docker container. +package db2test + +import ( + "archive/tar" + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "testing" + "time" + + dockerclient "github.com/docker/docker/client" + "github.com/moby/moby/api/types/container" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2cli" +) + +const ( + // db2Image is the IBM DB2 Community Edition image. It only publishes + // linux/amd64 manifests, so Apple Silicon hosts must use Rosetta emulation. + db2Image = "icr.io/db2_community/db2:latest" + db2Port = "50000/tcp" + db2Database = "TESTDB" + db2User = "db2inst1" + db2Password = "password" + + // db2BaseDir is the root of the DB2 installation inside the container and on + // the test host. DB2 CLI reads the DB2DIR environment variable (which the test + // runner sets to this value) to locate auxiliary files such as message + // catalogs and the driver configuration. + db2BaseDir = "/opt/ibm/db2/V12.1" + + // db2LibDir is the DB2 CLI library directory inside the container. The entire + // directory is extracted because libdb2.so.1 depends on other IBM-specific + // libraries (libdb2osse.so.1, ICU libs, etc.) that are only present inside + // the DB2 image — not on the test host. + db2LibDir = db2BaseDir + "/lib64" + + // db2LibInstallDir is where extracted DB2 libraries are placed. We use the + // same absolute path as inside the DB2 container because libdb2.so.1 and its + // dependencies are compiled with RPATH=/opt/ibm/db2/V12.1/lib64. If the libs + // are placed in a different directory (e.g. /usr/local/lib), the dynamic + // linker resolves libdb2.so.1 fine but then fails to find its dependencies + // because RPATH still points to the original path. Using the same path keeps + // RPATH resolution correct. + db2LibInstallDir = db2BaseDir + "/lib64" + + // db2MsgDir is the message catalog directory inside the container. DB2 CLI + // loads message catalogs from $DB2DIR/msg/ to format human-readable errors. + // Without these files every CLI error is replaced by SQL10007N, which masks + // the real connection failure. + db2MsgDir = db2BaseDir + "/msg" + + // db2LdConfFile is the ldconfig drop-in that registers db2LibInstallDir so + // that dlopen("libdb2.so.1") works without specifying a full path. + db2LdConfFile = "/etc/ld.so.conf.d/db2.conf" +) + +// TestDB wraps *sql.DB with test-oriented helpers for DB2 integration tests. +// It embeds *sql.DB so all standard database/sql methods are available. The T +// field provides test-failure reporting and the DSN field stores the connection +// string used to create the underlying connection, which is useful when tests +// need to open additional connections. +type TestDB struct { + *sql.DB + T *testing.T + DSN string +} + +// MustExec executes a SQL statement and calls t.Fatalf if it returns an error. +// Use this for DDL and DML statements in test setup/teardown where a failure +// means the test cannot continue meaningfully. +func (db *TestDB) MustExec(query string, args ...any) { + _, err := db.Exec(query, args...) + require.NoError(db.T, err, "MustExec: %s", query) +} + +// MustExecContext executes a SQL statement with a context and calls t.Fatalf +// if it returns an error. Use this variant when the test already has a context +// with a deadline (e.g. from t.Context()) to propagate cancellation. +func (db *TestDB) MustExecContext(ctx context.Context, query string, args ...any) { + _, err := db.ExecContext(ctx, query, args...) + require.NoError(db.T, err, "MustExecContext: %s", query) +} + +// db2LibOnce ensures the DB2 CLI library is extracted from the container and +// loaded into the process exactly once, even when tests run in parallel. +var ( + db2LibOnce sync.Once + db2LibErr error +) + +// sharedContainerOnce guards the package-level container singleton so that all +// integration tests in this package share a single DB2 container. Starting two +// containers sequentially (one per test) each takes 5–10 min and easily exceeds +// the 20-minute test timeout when running under DinD on macOS. +var ( + sharedContainerOnce sync.Once + sharedContainer testcontainers.Container + sharedDSN string + sharedContainerErr error +) + +// darwinContainerName holds the name of the externally-managed DB2 container +// when running on macOS in native mode (DB2_DARWIN_CONTAINER env var). Shell +// commands that would normally run as db2inst1 inside the container are +// dispatched via "docker exec {darwinContainerName} su - db2inst1 -c {cmd}". +var darwinContainerName string + +// db2LogConsumer streams DB2 container stdout/stderr to the test process stdout +// so startup progress is visible during the ~5–8 minute first-time setup. +type db2LogConsumer struct{} + +func (*db2LogConsumer) Accept(l testcontainers.Log) { + fmt.Printf("[db2-container] %s", l.Content) +} + +// findDarwinLib searches for libdb2.dylib (IBM DB2 ODBC CLI Driver / clidriver) on macOS. +// The clidriver connects via TCP/IP only — no SysV IPC, no --ipc=host required. +// Returns the absolute path if found, or an error with installation instructions. +func findDarwinLib() (string, error) { + if p := os.Getenv("DB2_DYLIB_PATH"); p != "" { + if _, err := os.Stat(p); err == nil { + return p, nil + } + return "", fmt.Errorf("DB2_DYLIB_PATH=%q: file not found", p) + } + // ibm_db Python package bundles the clidriver + out, err := exec.Command("python3", "-c", + "import ibm_db, os; print(os.path.dirname(ibm_db.__file__))").Output() + if err == nil { + c := filepath.Join(strings.TrimSpace(string(out)), "clidriver", "lib", "libdb2.dylib") + if _, err := os.Stat(c); err == nil { + return c, nil + } + } + for _, c := range []string{ + "/Library/IBM/SQLLIB/lib64/libdb2.dylib", + "/Library/IBM/SQLLIB/lib/libdb2.dylib", + filepath.Join(os.Getenv("HOME"), "sqllib", "lib64", "libdb2.dylib"), + } { + if _, err := os.Stat(c); err == nil { + return c, nil + } + } + return "", errors.New(`IBM DB2 ODBC CLI Driver (libdb2.dylib) not found. + +Install options: + pip3 install ibm_db # bundles clidriver for macOS + export DB2_DYLIB_PATH=/path/to/libdb2.dylib + +See: https://www.ibm.com/support/pages/db2-odbc-cli-driver-download-and-installation-information`) +} + +// acquireDarwinDB2 loads the IBM clidriver dylib and connects to the DB2 container +// on the host/port given by DB2_DARWIN_HOST / DB2_DARWIN_PORT (defaults: localhost:50000). +// The clidriver is TCP-only — no SysV IPC, so --ipc=host is not needed on macOS. +func acquireDarwinDB2(ctx context.Context) { + dylibPath, err := findDarwinLib() + if err != nil { + sharedContainerErr = err + return + } + if err := db2cli.LoadLibraryFromPath(dylibPath); err != nil { + sharedContainerErr = fmt.Errorf("loading libdb2.dylib from %s: %w", dylibPath, err) + return + } + + host := os.Getenv("DB2_DARWIN_HOST") + if host == "" { + host = "localhost" + } + port := os.Getenv("DB2_DARWIN_PORT") + if port == "" { + port = "50000" + } + + dsn := fmt.Sprintf( + "DATABASE=%s;HOSTNAME=%s;PORT=%s;PROTOCOL=TCPIP;UID=%s;PWD=%s", + db2Database, host, port, db2User, db2Password, + ) + + fmt.Printf("db2test: macOS native mode — connecting via clidriver to %s:%s (up to 6 min)...\n", host, port) + deadline := time.Now().Add(6 * time.Minute) + for { + db, openErr := sql.Open("db2-cli", dsn) + if openErr == nil { + db.SetMaxOpenConns(1) + pingErr := db.PingContext(ctx) + _ = db.Close() + if pingErr == nil { + break + } + fmt.Printf("db2test: DB2 not ready yet (%v), retrying...\n", pingErr) + } + if time.Now().After(deadline) { + sharedContainerErr = errors.New("DB2 did not accept connections within 6 min (macOS native mode)") + return + } + time.Sleep(5 * time.Second) + } + + sharedDSN = dsn + fmt.Printf("db2test: DB2 accepting connections (macOS native, clidriver at %s)\n", dylibPath) + + // Set the container name so that runAsDB2Inst1 / findAsnScript / startAsncapLocal + // dispatch through "docker exec" rather than running shell commands locally. + darwinContainerName = os.Getenv("DB2_DARWIN_CONTAINER") + + tuneDB, openErr := sql.Open("db2-cli", dsn) + if openErr == nil { + defer tuneDB.Close() + tuneDB.SetMaxOpenConns(1) + stmt := fmt.Sprintf("CALL SYSPROC.ADMIN_CMD('update db cfg for %s using LOCKTIMEOUT 30')", db2Database) + if _, err := tuneDB.ExecContext(ctx, stmt); err != nil { + fmt.Printf("db2test: DB tuning warning: %v\n", err) + } else { + fmt.Printf("db2test: LOCKTIMEOUT set to 30 s\n") + } + } + + // Create ASNCDC control tables and start the capture daemon — same as + // acquireLocalDB2. Shell commands are dispatched via docker exec because + // the IBM SQL scripts and asncap binary only exist inside the container. + if darwinContainerName == "" { + fmt.Printf("db2test: DB2_DARWIN_CONTAINER not set — ASNCDC setup skipped (CDC tests will fail)\n") + return + } + setupDB, openErr2 := sql.Open("db2-cli", dsn) + if openErr2 != nil { + fmt.Printf("db2test: ASNCDC setup skipped (open: %v)\n", openErr2) + return + } + defer setupDB.Close() + setupDB.SetMaxOpenConns(1) + + if err := createASNCDCTablesIfNeeded(ctx, setupDB); err != nil { + fmt.Printf("db2test: ASNCDC table setup warning (non-fatal): %v\n", err) + } + if err := startAsncapLocal(); err != nil { + fmt.Printf("db2test: asncap start warning (non-fatal): %v\n", err) + } +} + +// AcquireSharedContainer starts (or returns the already-running) DB2 container. +// Safe to call concurrently; the container is started at most once per process. +// Call TerminateSharedContainer from TestMain after m.Run() to clean up. +// +// If DB2_CONTAINER_NAME is set in the environment the container is given that +// name. The run-db2-macos-integration-tests.sh script sets this to a unique +// per-run name so it can tail logs and clean up by name without touching other +// containers. +// +// If DB2_USE_LOCAL=1 is set the function skips container creation and +// connects directly to a DB2 instance already running at 127.0.0.1:50000 +// (the local DB2 in the container the test binary is executing inside). +// The run-db2-macos-integration-tests.sh script uses this mode by copying the +// test binary into the DB2 container and running it there, which avoids all +// IPC-namespace and library-extraction problems caused by libdb2.so.1 (the +// full server library) trying to initialise a local DB2 engine in a separate +// container where no DB2 instance is running. +// +// On macOS (darwin), the clidriver path is used instead: the test binary runs +// natively on the host and connects via TCP to the mapped port. Set DB2_DYLIB_PATH +// or install ibm_db (pip3 install ibm_db) to provide libdb2.dylib. +func AcquireSharedContainer(ctx context.Context) (testcontainers.Container, string, error) { + sharedContainerOnce.Do(func() { + if runtime.GOOS == "darwin" { + acquireDarwinDB2(ctx) + return + } + if os.Getenv("DB2_USE_LOCAL") == "1" { + acquireLocalDB2(ctx) + return + } + fmt.Printf("db2test: starting DB2 container (first-time setup ~5–8 min)...\n") + opts := []testcontainers.ContainerCustomizer{ + testcontainers.WithImagePlatform("linux/amd64"), + testcontainers.WithEnv(map[string]string{ + "LICENSE": "accept", + "DB2INST1_PASSWORD": db2Password, + "DBNAME": db2Database, + // AUTOCONFIG=false: skip DB2 hardware auto-discovery (saves 2-3 min). + "AUTOCONFIG": "false", + // ARCHIVE_LOGS=true: enable transaction log archiving required for CDC. + // The init script sets LOGARCHMETH1=LOGRETAIN and restarts the DB; the + // "DEACTIVATED: NO" log message fires when that restart completes, which + // is our secondary wait-strategy trigger below. + "ARCHIVE_LOGS": "true", + // SAMPLEDB=false: skip IBM sample database creation (saves ~1 min). + "SAMPLEDB": "false", + // REPODB=false: skip DSM repository database creation (saves ~1 min). + // This is a second full CREATE DATABASE call; we do not use the IBM + // Data Server Manager console in tests. + "REPODB": "false", + // HADR_ENABLED=NO: skip High-Availability Disaster Recovery setup. + "HADR_ENABLED": "NO", + // UPDATEAVAIL=NO: skip update-availability check at startup. + "UPDATEAVAIL": "NO", + }), + testcontainers.WithExposedPorts(db2Port), + testcontainers.WithHostConfigModifier(func(hc *container.HostConfig) { + hc.Privileged = true + // DB2's IPC between db2agent and db2fmp uses shared memory. Docker's + // default /dev/shm of 64 MB is too small; 512 MB prevents transient + // SQL0902C / shared-memory exhaustion errors under load. + hc.ShmSize = 512 * 1024 * 1024 + // Share the host IPC namespace so the test runner container (also + // started with --ipc=host) can attach to the SysV shared memory + // segments that libdb2.so.1 initialises in this container. + // db2systm stores the IPC keys; without a shared namespace the keys + // are invisible from the test runner and every SQLDriverConnect + // returns SQL1042C. + hc.IpcMode = "host" + }), + testcontainers.WithLogConsumers(&db2LogConsumer{}), + // Wait for the DB2 init script to finish. "Setup has completed" is + // printed at the very end of the entrypoint after the full sequence: + // CREATE DATABASE → LOGARCHMETH1=LOGRETAIN → backup → restart. + // The connection-retry loop below handles any remaining delay before + // TCPIP connections are accepted. 20-minute deadline gives ample room + // for slow emulated hardware (Apple Silicon / Docker Desktop). + testcontainers.WithWaitStrategyAndDeadline(20*time.Minute, + wait.ForLog("Setup has completed").WithStartupTimeout(20*time.Minute), + ), + } + if name := os.Getenv("DB2_CONTAINER_NAME"); name != "" { + opts = append(opts, testcontainers.WithName(name)) + } + ctr, err := testcontainers.Run(ctx, db2Image, opts...) + if err != nil { + sharedContainerErr = fmt.Errorf("starting shared DB2 container: %w", err) + return + } + sharedContainer = ctr + fmt.Printf("db2test: container up, extracting DB2 libraries...\n") + + if err := ensureDB2Library(ctx, ctr); err != nil { + sharedContainerErr = fmt.Errorf("loading DB2 library: %w", err) + return + } + + // Prefer a direct container-to-container connection on the Docker bridge + // network (container IP : 50000) over the host.docker.internal:mapped_port + // NAT route. The NAT path causes SQL1042C on macOS Docker Desktop because + // packets go: test-runner → host-gateway → port-mapping → DB2 container, + // and DB2's TCPIP listener returns SQL1042C when the originating IP is the + // gateway rather than a local peer. Direct bridge IP bypasses this. + dsn := containerDirectDSN(ctx, ctr) + if dsn == "" { + // Fallback: use testcontainers-provided host + mapped port. + host, hostErr := ctr.Host(ctx) + if hostErr != nil { + sharedContainerErr = fmt.Errorf("getting container host: %w", hostErr) + return + } + port, portErr := ctr.MappedPort(ctx, db2Port) + if portErr != nil { + sharedContainerErr = fmt.Errorf("getting container port: %w", portErr) + return + } + dsn = fmt.Sprintf( + "DATABASE=%s;HOSTNAME=%s;PORT=%s;PROTOCOL=TCPIP;UID=%s;PWD=%s", + db2Database, host, port.Port(), db2User, db2Password, + ) + fmt.Printf("db2test: using fallback DSN (host.docker.internal): %s\n", host) + } + + // "Setup has completed" fires when the init script finishes, but DB2 + // needs additional time to start accepting TCPIP connections. Verify + // the connection before declaring the container ready so that SetupTest + // never races against this window. + // DB2 returns SQL1042C for several minutes after "Setup has completed" + // while its TCPIP listener finishes initialising post-restart. Allow + // up to 6 minutes; on fast hardware it usually clears in ~2–3 min. + fmt.Printf("db2test: verifying DB2 connection (up to 6 min)...\n") + deadline := time.Now().Add(6 * time.Minute) + for { + db, openErr := sql.Open("db2-cli", dsn) + if openErr == nil { + db.SetMaxOpenConns(1) + pingErr := db.PingContext(ctx) + _ = db.Close() + if pingErr == nil { + break + } + fmt.Printf("db2test: DB2 not ready yet (%v), retrying...\n", pingErr) + } + if time.Now().After(deadline) { + sharedContainerErr = errors.New("DB2 did not accept connections within 6 min") + return + } + time.Sleep(5 * time.Second) + } + + sharedDSN = dsn + fmt.Printf("db2test: DB2 accepting connections\n") + + // Apply test-specific DB tuning. LOCKTIMEOUT prevents tests from hanging + // indefinitely on lock contention; the default is -1 (wait forever). + // SYSPROC.ADMIN_CMD is used because UPDATE DATABASE CONFIGURATION is a + // DB2 command-line command, not an SQL statement. + func() { + tuneDB, openErr := sql.Open("db2-cli", dsn) + if openErr != nil { + fmt.Printf("db2test: DB tuning skipped (open: %v)\n", openErr) + return + } + defer tuneDB.Close() + tuneDB.SetMaxOpenConns(1) + stmt := fmt.Sprintf("CALL SYSPROC.ADMIN_CMD('update db cfg for %s using LOCKTIMEOUT 30')", db2Database) + if _, err := tuneDB.ExecContext(ctx, stmt); err != nil { + fmt.Printf("db2test: DB tuning warning: %v\n", err) + } else { + fmt.Printf("db2test: LOCKTIMEOUT set to 30 s\n") + } + }() + }) + return sharedContainer, sharedDSN, sharedContainerErr +} + +// TerminateSharedContainer stops the shared DB2 container. +// Call from TestMain after m.Run() to clean up. +func TerminateSharedContainer(ctx context.Context) { + if sharedContainer != nil { + _ = sharedContainer.Terminate(ctx) + } +} + +// acquireLocalDB2 sets up the shared DSN for a DB2 instance already running +// locally at 127.0.0.1:50000. Called from AcquireSharedContainer when +// DB2_USE_LOCAL=1 is set (i.e. the test binary is running inside the DB2 +// container via docker exec from run-db2-macos-integration-tests.sh). +func acquireLocalDB2(ctx context.Context) { + fmt.Printf("db2test: local mode — using DB2 instance at 127.0.0.1:50000\n") + + // Library is already present at the standard path inside the container. + if err := ensureDB2Library(ctx, nil); err != nil { + sharedContainerErr = fmt.Errorf("loading DB2 library (local mode): %w", err) + return + } + + dsn := fmt.Sprintf( + "DATABASE=%s;HOSTNAME=127.0.0.1;PORT=50000;PROTOCOL=TCPIP;UID=%s;PWD=%s", + db2Database, db2User, db2Password, + ) + + fmt.Printf("db2test: verifying local DB2 connection (up to 2 min)...\n") + deadline := time.Now().Add(2 * time.Minute) + for { + db, openErr := sql.Open("db2-cli", dsn) + if openErr == nil { + db.SetMaxOpenConns(1) + pingErr := db.PingContext(ctx) + _ = db.Close() + if pingErr == nil { + break + } + fmt.Printf("db2test: DB2 not ready yet (%v), retrying...\n", pingErr) + } + if time.Now().After(deadline) { + sharedContainerErr = errors.New("local DB2 did not accept connections within 2 min") + return + } + time.Sleep(5 * time.Second) + } + + sharedDSN = dsn + fmt.Printf("db2test: DB2 accepting connections (local mode)\n") + + // Tune LOCKTIMEOUT via shell — UPDATE DATABASE CONFIGURATION is a DB2 CLI + // command that SYSPROC.ADMIN_CMD may reject depending on the privilege level. + if out, err := runAsDB2Inst1(fmt.Sprintf( + "db2 update db cfg for %s using LOCKTIMEOUT 30", db2Database, + )); err != nil { + fmt.Printf("db2test: LOCKTIMEOUT warning: %v\n%s\n", err, out) + } else { + fmt.Printf("db2test: LOCKTIMEOUT set to 30 s\n") + } + + // Create ASNCDC control tables and start the capture daemon. This must + // happen here (once, before any test calls EnableASNCDC) because the setup + // involves running IBM SQL scripts and the asncap binary, both of which + // require the db2inst1 environment and must not run in parallel. + setupDB, openErr := sql.Open("db2-cli", dsn) + if openErr != nil { + fmt.Printf("db2test: ASNCDC setup skipped (open: %v)\n", openErr) + return + } + defer setupDB.Close() + setupDB.SetMaxOpenConns(1) + + if err := createASNCDCTablesIfNeeded(ctx, setupDB); err != nil { + fmt.Printf("db2test: ASNCDC table setup warning (non-fatal): %v\n", err) + } + if err := startAsncapLocal(); err != nil { + fmt.Printf("db2test: asncap start warning (non-fatal): %v\n", err) + } +} + +// ensureDB2Library extracts the DB2 CLI shared libraries from the running +// container into /opt/ibm/db2/V12.1/lib64 (the same path used inside the DB2 +// image), writes a drop-in ldconfig config file, runs ldconfig, and calls +// db2cli.LoadLibrary so the purego driver can dlopen them. +// +// We copy the entire lib64 directory (not just libdb2.so.1) because the main +// library depends on IBM-specific shared objects — libdb2osse.so.1, ICU libs, +// etc. — that are only available inside the DB2 container image. +// +// LD_LIBRARY_PATH cannot be used here: in a pure-Go binary (no CGO), +// os.Setenv updates Go's internal env cache but does NOT call the C library's +// setenv(3), so dlopen (a C function) never sees the change. The solution is +// to extract the libs to the SAME path they lived in inside the container +// (/opt/ibm/db2/V12.1/lib64) — libdb2.so.1 is compiled with RPATH pointing +// to that directory, so placing the files there keeps transitive dependency +// resolution correct. A drop-in ldconfig config file then registers the +// directory so dlopen("libdb2.so.1") resolves by name. +// +// The extraction runs exactly once per process (via sync.Once); parallel tests +// share the result safely. +func ensureDB2Library(ctx context.Context, ctr testcontainers.Container) error { + db2LibOnce.Do(func() { + if ctr == nil { + // Local mode: the library is already installed inside the DB2 container + // at the standard path. Register it with ldconfig and dlopen it. + if mkErr := os.MkdirAll("/etc/ld.so.conf.d", 0o755); mkErr == nil { + _ = os.WriteFile(db2LdConfFile, []byte(db2LibInstallDir+"\n"), 0o644) + if out, ldErr := exec.Command("ldconfig").CombinedOutput(); ldErr != nil { + fmt.Printf("db2test: ldconfig warning: %v\n%s\n", ldErr, out) + } + } + libPath := filepath.Join(db2LibInstallDir, "libdb2.so.1") + fmt.Printf("db2test: loading DB2 library from %s (local mode)\n", libPath) + db2LibErr = db2cli.LoadLibraryFromPath(libPath) + if db2LibErr != nil { + return + } + var warmupHenv db2cli.SQLHENV + if r := db2cli.SQLAllocHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(db2cli.SQL_NULL_HENV), (*db2cli.SQLHANDLE)(&warmupHenv)); r == db2cli.SQL_SUCCESS || r == db2cli.SQL_SUCCESS_WITH_INFO { + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(warmupHenv)) + } + fmt.Printf("db2test: DB2 CLI library warm-up complete (local mode)\n") + return + } + + // testcontainers' CopyFileFromContainer is designed for single files only — it + // internally calls tarReader.Next() once, advancing past the directory header + // and leaving the reader at EOF for a directory path. Use the Docker client + // directly to get the raw unmodified tar stream for the lib64 directory. + dc, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv, dockerclient.WithAPIVersionNegotiation()) + if err != nil { + db2LibErr = fmt.Errorf("creating Docker client: %w", err) + return + } + defer dc.Close() + + tarStream, _, err := dc.CopyFromContainer(ctx, ctr.GetContainerID(), db2LibDir) + if err != nil { + db2LibErr = fmt.Errorf("copy %s from container: %w", db2LibDir, err) + return + } + defer tarStream.Close() + + if err := os.MkdirAll(db2LibInstallDir, 0o755); err != nil { + db2LibErr = fmt.Errorf("create %s: %w", db2LibInstallDir, err) + return + } + + // Extract .so files and their symlinks into db2LibInstallDir. + tr := tar.NewReader(tarStream) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + db2LibErr = fmt.Errorf("reading DB2 lib64 tar: %w", err) + return + } + + name := filepath.Base(hdr.Name) + if !strings.Contains(name, ".so") { + continue + } + + destPath := filepath.Join(db2LibInstallDir, name) + + switch hdr.Typeflag { + case tar.TypeSymlink: + // Rewrite the link target to its basename so it resolves within + // the flat install dir rather than pointing into the container path. + target := filepath.Base(hdr.Linkname) + if linkErr := os.Symlink(target, destPath); linkErr != nil && !os.IsExist(linkErr) { + db2LibErr = fmt.Errorf("symlink %s → %s: %w", destPath, target, linkErr) + return + } + case tar.TypeReg: + f, createErr := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)|0o755) + if createErr != nil { + db2LibErr = fmt.Errorf("create %s: %w", destPath, createErr) + return + } + if _, copyErr := io.Copy(f, tr); copyErr != nil { + f.Close() + db2LibErr = fmt.Errorf("write %s: %w", destPath, copyErr) + return + } + f.Close() + } + } + + // Extract message catalog directory ($DB2DIR/msg/) from the container. + // DB2 CLI loads .mo/.cat files from this directory to format error messages. + // Without them every error is replaced with SQL10007N, masking the real cause. + msgTarStream, _, err := dc.CopyFromContainer(ctx, ctr.GetContainerID(), db2MsgDir) + if err != nil { + db2LibErr = fmt.Errorf("copy %s from container: %w", db2MsgDir, err) + return + } + defer msgTarStream.Close() + + msgTR := tar.NewReader(msgTarStream) + for { + hdr, err := msgTR.Next() + if err == io.EOF { + break + } + if err != nil { + db2LibErr = fmt.Errorf("reading DB2 msg tar: %w", err) + return + } + // Tar entries have paths like "msg/en_US.iso88591/db2adm.cat". + // Prepend db2BaseDir to reconstruct the full installation path. + destPath := filepath.Join(db2BaseDir, hdr.Name) + switch hdr.Typeflag { + case tar.TypeDir: + if mkErr := os.MkdirAll(destPath, 0o755); mkErr != nil { + db2LibErr = fmt.Errorf("create dir %s: %w", destPath, mkErr) + return + } + case tar.TypeReg: + if mkErr := os.MkdirAll(filepath.Dir(destPath), 0o755); mkErr != nil { + db2LibErr = fmt.Errorf("create parent for %s: %w", destPath, mkErr) + return + } + f, createErr := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)|0o644) + if createErr != nil { + db2LibErr = fmt.Errorf("create %s: %w", destPath, createErr) + return + } + if _, copyErr := io.Copy(f, msgTR); copyErr != nil { + f.Close() + db2LibErr = fmt.Errorf("write %s: %w", destPath, copyErr) + return + } + f.Close() + case tar.TypeSymlink: + // Catalog files may be versioned symlinks (e.g. db2ca.cat -> db2ca.1.1.cat). + // Preserve them so DB2 CLI can resolve the canonical names. + if mkErr := os.MkdirAll(filepath.Dir(destPath), 0o755); mkErr == nil { + _ = os.Remove(destPath) // overwrite any stale symlink + _ = os.Symlink(hdr.Linkname, destPath) + } + } + } + fmt.Printf("db2test: extracted %s OK\n", db2MsgDir) + + // Create locale symlinks in the msg directory so DB2 CLI can find message + // catalogs regardless of the active LANG setting in the test container. + // The container locale may differ from en_US.iso88591 which is the only + // subdirectory extracted from the DB2 image. + msgInstallDir := filepath.Join(db2BaseDir, "msg") + for _, alias := range []string{"C", "en_US", "en_US.UTF-8", "en_US.utf8"} { + linkPath := filepath.Join(msgInstallDir, alias) + if _, statErr := os.Lstat(linkPath); os.IsNotExist(statErr) { + _ = os.Symlink("en_US.iso88591", linkPath) + } + } + + // Extract DB2 security plugin directory ($DB2DIR/security64/) from the + // container. DB2 CLI needs the security plugins (IBMOSauthclient.so etc.) + // to authenticate via username/password over TCPIP. The instance + // sqllib/security64/plugin/IBM is a symlink to $DB2DIR/security64/plugin/IBM, + // so extracting the global directory is sufficient. + sec64Src := db2BaseDir + "/security64" + if sec64TarStream, _, sec64Err := dc.CopyFromContainer(ctx, ctr.GetContainerID(), sec64Src); sec64Err == nil { + defer sec64TarStream.Close() + sec64TR := tar.NewReader(sec64TarStream) + for { + hdr, err := sec64TR.Next() + if err == io.EOF { + break + } + if err != nil { + db2LibErr = fmt.Errorf("reading security64 tar: %w", err) + return + } + destPath := filepath.Join(db2BaseDir, hdr.Name) + switch hdr.Typeflag { + case tar.TypeDir: + if mkErr := os.MkdirAll(destPath, 0o755); mkErr != nil { + db2LibErr = fmt.Errorf("create dir %s: %w", destPath, mkErr) + return + } + case tar.TypeReg: + if mkErr := os.MkdirAll(filepath.Dir(destPath), 0o755); mkErr != nil { + db2LibErr = fmt.Errorf("create parent for %s: %w", destPath, mkErr) + return + } + f, createErr := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)|0o755) + if createErr != nil { + db2LibErr = fmt.Errorf("create %s: %w", destPath, createErr) + return + } + if _, copyErr := io.Copy(f, sec64TR); copyErr != nil { + f.Close() + db2LibErr = fmt.Errorf("write %s: %w", destPath, copyErr) + return + } + f.Close() + case tar.TypeSymlink: + if mkErr := os.MkdirAll(filepath.Dir(destPath), 0o755); mkErr == nil { + _ = os.Remove(destPath) + _ = os.Symlink(hdr.Linkname, destPath) + } + } + } + fmt.Printf("db2test: extracted %s OK\n", sec64Src) + } else { + fmt.Printf("db2test: security64 not available (non-fatal): %v\n", sec64Err) + } + + // Create a minimal db2dsdriver.cfg so the CLI knows it is operating as a + // remote-only thin client (no local DB2 instance required). + cfgDir := filepath.Join(db2BaseDir, "cfg") + if mkErr := os.MkdirAll(cfgDir, 0o755); mkErr != nil { + db2LibErr = fmt.Errorf("create cfg dir: %w", mkErr) + return + } + dsdriverCfg := filepath.Join(cfgDir, "db2dsdriver.cfg") + if _, statErr := os.Stat(dsdriverCfg); os.IsNotExist(statErr) { + if writeErr := os.WriteFile(dsdriverCfg, []byte("\n\n"), 0o644); writeErr != nil { + db2LibErr = fmt.Errorf("create db2dsdriver.cfg: %w", writeErr) + return + } + } + + // Populate the DB2 instance home that libdb2.so.1 locates via + // getpwnam("db2inst1"). In the icr.io/db2_community/db2 image the + // instance owner's home is /database/config/db2inst1 (not /home/db2inst1). + // The server library requires db2systm and db2nodes.cfg in sqllib/ to + // initialise even for remote TCPIP connections. + instanceHome := "/database/config/db2inst1" + instanceSqllib := instanceHome + "/sqllib" + for _, relPath := range []string{ + "db2systm", + "db2nodes.cfg", + filepath.Join("cfg", "db2cli.ini"), + } { + srcPath := instanceSqllib + "/" + relPath + dstPath := filepath.Join(instanceSqllib, relPath) + if mkErr := os.MkdirAll(filepath.Dir(dstPath), 0o755); mkErr != nil { + db2LibErr = fmt.Errorf("create dir for %s: %w", relPath, mkErr) + return + } + if extractErr := extractSingleFileFromContainer(ctx, dc, ctr.GetContainerID(), srcPath, dstPath); extractErr != nil { + fmt.Printf("db2test: skipping instance file %s (non-fatal): %v\n", relPath, extractErr) + } else { + fmt.Printf("db2test: extracted instance file %s\n", dstPath) + } + } + + // db2nodes.cfg extracted from the DB2 container contains the DB2 + // container's hostname. libdb2.so.1 reads this file and checks that + // the current machine's hostname matches node 0. If they differ, the + // library returns SQL1042C on every connection attempt — even remote + // TCPIP connections. Overwrite with the test runner's own hostname. + nodesCfgPath := filepath.Join(instanceSqllib, "db2nodes.cfg") + if localHost, hostnameErr := os.Hostname(); hostnameErr == nil { + nodesCfg := fmt.Sprintf("0 %s 0\n", localHost) + if writeErr := os.WriteFile(nodesCfgPath, []byte(nodesCfg), 0o644); writeErr == nil { + fmt.Printf("db2test: rewrote db2nodes.cfg with local hostname %q\n", localHost) + } + } + + // Extract instance-level message catalogs — libdb2.so.1 checks + // $INSTHOME/sqllib/msg/ before $DB2DIR/msg/. + instanceMsgSrc := instanceSqllib + "/msg" + instanceMsgDst := filepath.Join(instanceSqllib, "msg") + if instanceMsgTS, _, instanceMsgErr := dc.CopyFromContainer(ctx, ctr.GetContainerID(), instanceMsgSrc); instanceMsgErr == nil { + defer instanceMsgTS.Close() + instanceMsgTR := tar.NewReader(instanceMsgTS) + for { + hdr, err := instanceMsgTR.Next() + if err == io.EOF { + break + } + if err != nil { + break + } + relName := strings.TrimPrefix(strings.TrimPrefix(hdr.Name, "msg"), "/") + if relName == "" { + continue + } + dstPath := filepath.Join(instanceMsgDst, relName) + switch hdr.Typeflag { + case tar.TypeDir: + _ = os.MkdirAll(dstPath, 0o755) + case tar.TypeReg: + _ = os.MkdirAll(filepath.Dir(dstPath), 0o755) + if f, createErr := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)|0o644); createErr == nil { + _, _ = io.Copy(f, instanceMsgTR) + f.Close() + } + case tar.TypeSymlink: + _ = os.Symlink(filepath.Base(hdr.Linkname), dstPath) + } + } + fmt.Printf("db2test: extracted instance msg to %s\n", instanceMsgDst) + } else { + fmt.Printf("db2test: instance msg not available (non-fatal): %v\n", instanceMsgErr) + } + + // Ensure the instance msg directory has locale subdirs pointing to the + // global $DB2DIR/msg locale dirs. In the container the instance sqllib/msg/ + // entries are symlinks → $DB2DIR/msg, so the tar extraction skips them, + // leaving the directory empty. DB2 CLI checks the instance path first and + // fails with SQL10007N if the locale dir is absent. + { + globalMsgBase := filepath.Join(db2BaseDir, "msg") + if err := os.MkdirAll(instanceMsgDst, 0o755); err == nil { + if entries, readErr := os.ReadDir(globalMsgBase); readErr == nil { + for _, e := range entries { + linkPath := filepath.Join(instanceMsgDst, e.Name()) + if _, statErr := os.Lstat(linkPath); os.IsNotExist(statErr) { + _ = os.Symlink(filepath.Join(globalMsgBase, e.Name()), linkPath) + } + } + } + } + } + + // Create instance sqllib/security64 directory structure with IBM symlink. + // In the container, sqllib/security64/plugin/IBM -> $DB2DIR/security64/plugin/IBM. + // Recreating this allows DB2 CLI to find authentication plugins. + { + instanceSec64Plugin := filepath.Join(instanceSqllib, "security64", "plugin") + for _, dir := range []string{"client", "group", "server"} { + _ = os.MkdirAll(filepath.Join(instanceSec64Plugin, dir), 0o755) + } + ibmLink := filepath.Join(instanceSec64Plugin, "IBM") + if _, statErr := os.Lstat(ibmLink); os.IsNotExist(statErr) { + _ = os.Symlink(filepath.Join(db2BaseDir, "security64", "plugin", "IBM"), ibmLink) + } + } + + // Diagnostic: print which critical paths exist vs. are missing. + for _, checkPath := range []string{ + "/database/config/db2inst1/sqllib/db2systm", + "/database/config/db2inst1/sqllib/db2nodes.cfg", + "/database/config/db2inst1/sqllib/cfg/db2cli.ini", + "/opt/ibm/db2/V12.1/msg/en_US.iso88591", + "/database/config/db2inst1/sqllib/msg/en_US.iso88591", + "/opt/ibm/db2/V12.1/security64/plugin/IBM/client/IBMOSauthclient.so", + "/database/config/db2inst1/sqllib/security64/plugin/IBM", + } { + if _, statErr := os.Stat(checkPath); statErr == nil { + fmt.Printf("db2test: EXISTS: %s\n", checkPath) + } else { + fmt.Printf("db2test: MISSING: %s\n", checkPath) + } + } + + // Register db2LibInstallDir with ldconfig so dlopen finds libdb2.so.1 by + // name. db2LibInstallDir is not in ldconfig's default search path, so we + // drop a config file into /etc/ld.so.conf.d/ before running ldconfig. + if err := os.MkdirAll("/etc/ld.so.conf.d", 0o755); err != nil { + db2LibErr = fmt.Errorf("create /etc/ld.so.conf.d: %w", err) + return + } + if err := os.WriteFile(db2LdConfFile, []byte(db2LibInstallDir+"\n"), 0o644); err != nil { + db2LibErr = fmt.Errorf("write %s: %w", db2LdConfFile, err) + return + } + // Verify the primary library file was extracted before attempting to load. + libPath := filepath.Join(db2LibInstallDir, "libdb2.so.1") + if _, statErr := os.Stat(libPath); statErr != nil { + db2LibErr = fmt.Errorf("libdb2.so.1 not found at %s after extraction: %w", libPath, statErr) + return + } + fmt.Printf("db2test: extracted %s OK, loading...\n", libPath) + + // Run ldconfig so the system linker cache includes db2LibInstallDir. + // This is best-effort; the full-path dlopen below is the authoritative load. + if out, ldErr := exec.Command("ldconfig").CombinedOutput(); ldErr != nil { + fmt.Printf("db2test: ldconfig warning: %v\n%s\n", ldErr, out) + } + + // Print runtime environment so we can verify DB2DIR is set and the + // msg directory actually has the expected contents on disk. + fmt.Printf("db2test: DB2DIR=%q LANG=%q\n", os.Getenv("DB2DIR"), os.Getenv("LANG")) + if entries, lsErr := os.ReadDir(filepath.Join(db2BaseDir, "msg")); lsErr == nil { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + fmt.Printf("db2test: msg/ entries: %v\n", names) + } else { + fmt.Printf("db2test: msg/ read error: %v\n", lsErr) + } + if entries, lsErr := os.ReadDir(filepath.Join(db2BaseDir, "msg", "en_US.iso88591")); lsErr == nil { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + fmt.Printf("db2test: msg/en_US.iso88591/ (%d files): %v\n", len(names), names[:min(10, len(names))]) + } else { + fmt.Printf("db2test: msg/en_US.iso88591/ error: %v\n", lsErr) + } + + // Use the full absolute path so dlopen resolves libdb2.so.1 without + // depending on the linker cache. + db2LibErr = db2cli.LoadLibraryFromPath(libPath) + if db2LibErr != nil { + return + } + + // Absorb the first-call SQLAllocHandle(ENV) failure. The DB2 CLI library + // runs one-time global initialization on the very first SQLAllocHandle call; + // that init returns SQL_ERROR even though it partially succeeds, causing all + // subsequent calls to work normally. Calling SQLAllocHandle once here + // consumes that failure so that real connection attempts all succeed. + var warmupHenv db2cli.SQLHENV + if r := db2cli.SQLAllocHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(db2cli.SQL_NULL_HENV), (*db2cli.SQLHANDLE)(&warmupHenv)); r == db2cli.SQL_SUCCESS || r == db2cli.SQL_SUCCESS_WITH_INFO { + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(warmupHenv)) + } + fmt.Printf("db2test: DB2 CLI library warm-up complete\n") + }) + return db2LibErr +} + +// extractSingleFileFromContainer copies one regular file from a running container +// to dstPath. Non-fatal callers should just log the returned error. +func extractSingleFileFromContainer(ctx context.Context, dc *dockerclient.Client, containerID, srcPath, dstPath string) error { + ts, _, err := dc.CopyFromContainer(ctx, containerID, srcPath) + if err != nil { + return err + } + defer ts.Close() + tr := tar.NewReader(ts) + for { + hdr, err := tr.Next() + if err == io.EOF { + return fmt.Errorf("file not found in tar for %s", srcPath) + } + if err != nil { + return fmt.Errorf("reading tar: %w", err) + } + if hdr.Typeflag != tar.TypeReg { + continue + } + f, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)|0o644) + if err != nil { + return fmt.Errorf("create %s: %w", dstPath, err) + } + _, copyErr := io.Copy(f, tr) + f.Close() + return copyErr + } +} + +// containerDirectDSN returns a DSN that connects directly to the DB2 container +// via its bridge network IP and port 50000, bypassing the host.docker.internal +// NAT route. Returns "" if the IP cannot be determined. +func containerDirectDSN(ctx context.Context, ctr testcontainers.Container) string { + dc, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv, dockerclient.WithAPIVersionNegotiation()) + if err != nil { + return "" + } + defer dc.Close() + info, err := dc.ContainerInspect(ctx, ctr.GetContainerID()) + if err != nil { + return "" + } + for _, nw := range info.NetworkSettings.Networks { + if nw.IPAddress != "" { + dsn := fmt.Sprintf( + "DATABASE=%s;HOSTNAME=%s;PORT=50000;PROTOCOL=TCPIP;UID=%s;PWD=%s", + db2Database, nw.IPAddress, db2User, db2Password, + ) + fmt.Printf("db2test: using direct container IP DSN: %s:50000\n", nw.IPAddress) + return dsn + } + } + return "" +} + +// SetupTest returns a TestDB connected to the shared DB2 container for this +// test run. The container is started once per process (via AcquireSharedContainer +// called from TestMain) and torn down after all tests finish. +// +// Each call opens its own *sql.DB so connection-pool state is isolated between +// tests. The returned connection is closed in t.Cleanup. +// +// On macOS the test uses IBM's clidriver (libdb2.dylib) — a TCP-only client that +// needs no SysV IPC. Set DB2_DYLIB_PATH to the dylib path, or install ibm_db via +// pip3 (pip3 install ibm_db bundles the clidriver). The test is skipped when the +// shared container is unavailable (Docker not running, no icr.io credentials, etc.). +// SetupTest acquires the shared DB2 container (starting it if needed), opens a +// new *sql.DB connection, and returns a TestDB ready for use. If the shared +// container is unavailable (e.g. Docker is not running or the container failed +// to start) the test is skipped rather than failed, so the test suite does not +// block CI when DB2 integration tests are not configured. The connection is +// closed automatically via t.Cleanup. +func SetupTest(t *testing.T) *TestDB { + t.Helper() + + ctx := t.Context() + _, dsn, err := AcquireSharedContainer(ctx) + if err != nil { + t.Skipf("skipping DB2 integration test: shared container unavailable: %v", err) + } + + t.Logf("db2test: connecting to DSN=%s", dsn) + + var db *sql.DB + var lastPingErr error + require.Eventually(t, func() bool { + if db != nil { + _ = db.Close() + } + db, err = sql.Open("db2-cli", dsn) + if err != nil { + lastPingErr = fmt.Errorf("sql.Open: %w", err) + t.Logf("db2test: sql.Open failed: %v", lastPingErr) + return false + } + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(5) + db.SetConnMaxLifetime(5 * time.Minute) + lastPingErr = db.PingContext(ctx) + if lastPingErr != nil { + t.Logf("db2test: ping failed: %v", lastPingErr) + } + return lastPingErr == nil + }, 2*time.Minute, 3*time.Second, "DB2 did not become reachable") + + t.Cleanup(func() { + assert.NoError(t, db.Close()) + }) + + return &TestDB{DB: db, T: t, DSN: dsn} +} + +// EnableASNCDC starts the DB2 SQL Replication capture daemon and registers the +// given tables for CDC. Must be called after tables already exist. +// +// Setup sequence mirrors IBM's ASNCDC SQL replication setup: +// 1. Start the capture daemon (retry until ready). +// 2. Call ADDTABLE for each table. +// 3. Set STATE='A' in IBMSNAP_REGISTER for each table. +// 4. Reinit the capture daemon so it picks up the new registrations. +// 5. Wait for SYNCHPOINT to become non-null (daemon has processed the table). +// +// When DB2_USE_LOCAL=1 (test binary running inside the DB2 container), the +// ASNCDC.ASNCDCSERVICES and ASNCDC.ADDTABLE SQL UDFs are not available in +// DB2 Community Edition, so this method uses os/exec shell commands instead. +func (db *TestDB) EnableASNCDC(schema string, tables []string) { + db.T.Helper() + + // DB2 Community Edition does not ship the ASNCDC.ASNCDCSERVICES / ADDTABLE + // UDFs (they are part of the IBM ASNTOOLS package). Use the local-mode path + // (direct shell commands via docker exec on macOS, direct su on Linux) for + // both DB2_USE_LOCAL=1 and macOS native mode. + if os.Getenv("DB2_USE_LOCAL") == "1" || runtime.GOOS == "darwin" { + db.enableASNCDCLocal(schema, tables) + return + } + + ctx := db.T.Context() + + // Start the capture daemon — retry because the first call after the DB + // restarts with LOGARCHMETH1=LOGRETAIN may fail transiently. + require.Eventually(db.T, func() bool { + _, err := db.ExecContext(ctx, "VALUES ASNCDC.ASNCDCSERVICES('start','asncdc')") + return err == nil + }, 2*time.Minute, 5*time.Second, "ASNCDC service did not start") + + for _, table := range tables { + _, err := db.ExecContext(ctx, + fmt.Sprintf("VALUES ASNCDC.ADDTABLE('%s','%s')", schema, table), + ) + require.NoError(db.T, err, "ASNCDC.ADDTABLE(%s, %s)", schema, table) + + // Activate the registration — ADDTABLE creates it with STATE='I' (inactive). + _, err = db.ExecContext(ctx, fmt.Sprintf( + "UPDATE ASNCDC.IBMSNAP_REGISTER SET STATE = 'A' WHERE SOURCE_OWNER = '%s' AND SOURCE_TABLE = '%s'", + schema, table, + )) + require.NoError(db.T, err, "activate IBMSNAP_REGISTER %s.%s", schema, table) + } + + // Reinit the capture daemon so it picks up newly registered tables. + _, err := db.ExecContext(ctx, "VALUES ASNCDC.ASNCDCSERVICES('reinit','asncdc')") + require.NoError(db.T, err, "ASNCDC reinit") + + // Wait for the daemon to process at least one table registration and populate SYNCHPOINT. + require.Eventually(db.T, func() bool { + var count int + _ = db.QueryRowContext(ctx, + fmt.Sprintf( + "SELECT COUNT(*) FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_OWNER = '%s' AND SYNCHPOINT IS NOT NULL", + schema, + ), + ).Scan(&count) + return count > 0 + }, 2*time.Minute, 3*time.Second, "SYNCHPOINT did not become non-null after ASNCDC reinit") +} + +// enableASNCDCLocal is the local-mode implementation of EnableASNCDC used when +// DB2_USE_LOCAL=1. It replaces the ASNCDC.ASNCDCSERVICES and ASNCDC.ADDTABLE +// SQL UDFs (which require the IBM ASNTOOLS package not present in DB2 Community +// Edition) with direct shell commands and SQL DDL. +func (db *TestDB) enableASNCDCLocal(schema string, tables []string) { + db.T.Helper() + ctx := db.T.Context() + + // Set up all table registrations BEFORE starting asncap. If asncap starts + // and finds a registered table without DATA CAPTURE CHANGES it stops + // immediately with ASN0009E. addTableLocal sets DATA CAPTURE CHANGES as + // part of registration, so all tables must be ready before asncap starts. + for _, table := range tables { + if err := addTableLocal(ctx, db.DB, schema, table); err != nil { + db.T.Fatalf("enableASNCDCLocal: addTableLocal(%s.%s): %v", schema, table, err) + } + // addTableLocal now inserts STATE='A' directly with null synchpoints, + // matching IBM's ADDTABLE procedure — no separate UPDATE needed. + } + + // Ensure asncap is running after all tables have DATA CAPTURE CHANGES set. + // All tables must be registered before starting asncap — if asncap encounters + // a registered table without DATA CAPTURE CHANGES it stops immediately. + if err := startAsncapLocal(); err != nil { + db.T.Fatalf("enableASNCDCLocal: startAsncapLocal: %v", err) + } + + // Signal asncap to re-read the registration table and start capturing the + // newly registered tables. + reinitOut, reinitErr := runAsDB2Inst1(fmt.Sprintf( + "asnccmd capture_server=%s capture_schema=ASNCDC reinit", + db2Database, + )) + fmt.Printf("db2test: asnccmd reinit output: %s\n", reinitOut) + if reinitErr != nil { + db.T.Logf("asnccmd reinit warning (non-fatal): %v", reinitErr) + } + + // IBM recommends waiting ~15 seconds after reinit before polling SYNCHPOINT. + // This gives asncap time to process the REINIT, pick up the new registrations, + // and set SYNCHPOINT in IBMSNAP_REGISTER for the new tables. + time.Sleep(15 * time.Second) + + // Verify asncap is still running after reinit. + checkPgrepLocal := func() string { + if darwinContainerName != "" { + out, _ := exec.Command("docker", "exec", darwinContainerName, "pgrep", "-f", "capture_server="+db2Database).Output() + return strings.TrimSpace(string(out)) + } + out, _ := exec.Command("pgrep", "-f", "capture_server="+db2Database).Output() + return strings.TrimSpace(string(out)) + } + if checkPgrepLocal() == "" { + db.T.Logf("db2test: WARNING: asncap is not running after reinit — attempting restart") + if err := startAsncapLocal(); err != nil { + db.T.Logf("db2test: WARNING: asncap restart failed: %v", err) + } + time.Sleep(5 * time.Second) + } + + // Poll until asncap has set SYNCHPOINT for each newly registered table, + // or up to 30s. This ensures the snapshot connector gets a valid starting + // position from captureCurrentCSN rather than CSN(0). + for _, table := range tables { + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + var sp []byte + if err := db.DB.QueryRowContext(ctx, + "SELECT SYNCHPOINT FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_OWNER=? AND SOURCE_TABLE=? AND SYNCHPOINT IS NOT NULL FETCH FIRST 1 ROW ONLY", + schema, table, + ).Scan(&sp); err == nil && len(sp) > 0 { + fmt.Printf("db2test: asncap set SYNCHPOINT for %s.%s: %X\n", schema, table, sp) + break + } + time.Sleep(3 * time.Second) + } + } + + // Print the asncap log tail to help diagnose capture issues. + var logOut []byte + if darwinContainerName != "" { + logOut, _ = exec.Command("docker", "exec", darwinContainerName, "cat", "/tmp/asncap.log").Output() + } else { + logOut, _ = os.ReadFile("/tmp/asncap.log") + } + if len(logOut) > 0 { + tail := logOut + if len(tail) > 3000 { + tail = tail[len(tail)-3000:] + } + fmt.Printf("db2test: asncap log (last 3000 bytes):\n%s\n", tail) + } +} + +// runAsDB2Inst1 executes cmdStr as a shell command under the db2inst1 user. +// The '-' flag loads the full login environment (DB2INSTANCE, DB2DIR, PATH, etc.) +// so that db2, asncap, asnccmd, and other DB2 tools are available. +// +// On macOS (darwin), when DB2_DARWIN_CONTAINER is set, commands are dispatched +// via "docker exec {container} su - db2inst1 -c {cmd}" because the DB2 binaries +// only exist inside the container, not on the macOS host. +func runAsDB2Inst1(cmdStr string) ([]byte, error) { + if darwinContainerName != "" { + cmd := exec.Command("docker", "exec", darwinContainerName, "su", "-", "db2inst1", "-c", cmdStr) + return cmd.CombinedOutput() + } + cmd := exec.Command("su", "-", "db2inst1", "-c", cmdStr) + return cmd.CombinedOutput() +} + +// findAsnScript searches the DB2 installation directory for a named SQL script +// and returns the first match, or empty string if not found. +// +// On macOS, the search runs inside the Docker container via docker exec. +func findAsnScript(name string) string { + var out []byte + var err error + if darwinContainerName != "" { + out, err = exec.Command("docker", "exec", darwinContainerName, "find", "/opt/ibm/db2", "-name", name, "-type", "f").Output() + } else { + out, err = exec.Command("find", "/opt/ibm/db2", "-name", name, "-type", "f").Output() + } + if err != nil || len(out) == 0 { + return "" + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + if len(lines) > 0 && lines[0] != "" { + return lines[0] + } + return "" +} + +// createASNCDCTablesIfNeeded creates the ASNCDC SQL Replication control tables +// required by asncap if they do not already exist. It locates IBM's asnctlw.sql +// script, rewrites all ASN schema references to ASNCDC, and runs it via db2inst1. +// After the base script it also applies any migration scripts (asncap*fp.sql) that +// add columns or tables introduced in later DB2 fix-packs. +func createASNCDCTablesIfNeeded(ctx context.Context, db *sql.DB) error { + // Check whether the ASNCDC schema already exists. + var n int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM SYSCAT.SCHEMATA WHERE SCHEMANAME = 'ASNCDC'", + ).Scan(&n); err != nil { + return fmt.Errorf("checking ASNCDC schema: %w", err) + } + if n > 0 { + fmt.Printf("db2test: ASNCDC schema already exists, checking CAPPARMS...\n") + // Apply any missing columns from newer DB2 fix-packs even on existing schemas. + if err := ensureMissingColumns(ctx, db); err != nil { + fmt.Printf("db2test: ensureMissingColumns warning: %v\n", err) + } + return ensureCAPPARMSRow(ctx, db) + } + + // Find the base control-table creation script. + scriptPath := findAsnScript("asnctlw.sql") + if scriptPath == "" { + return errors.New("asnctlw.sql not found in DB2 installation (required to set up ASNCDC)") + } + fmt.Printf("db2test: found asnctlw.sql at %s\n", scriptPath) + + var content []byte + var err error + if darwinContainerName != "" { + // scriptPath lives inside the container; read it via docker exec cat. + content, err = exec.Command("docker", "exec", darwinContainerName, "cat", scriptPath).Output() + if err != nil { + return fmt.Errorf("reading %s from container %s: %w", scriptPath, darwinContainerName, err) + } + } else { + content, err = os.ReadFile(scriptPath) + if err != nil { + return fmt.Errorf("reading %s: %w", scriptPath, err) + } + } + + // Rewrite all ASN schema references to ASNCDC, then strip multi-partition + // tablespace clauses so the script runs on single-partition DB2 12.1. + modified := rewriteASNSchemaInSQL(string(content), "ASN", "ASNCDC") + modified = stripTablespaceForSinglePartition(modified) + + tmp, err := os.CreateTemp("", "asncdc-*.sql") + if err != nil { + return fmt.Errorf("creating temp SQL file: %w", err) + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + + if _, err := tmp.WriteString(modified); err != nil { + tmp.Close() + return fmt.Errorf("writing temp SQL file: %w", err) + } + tmp.Close() + _ = os.Chmod(tmpPath, 0o644) // readable by db2inst1 + + // asnctlw.sql uses ; as the statement terminator in all DB2 versions. + // Do NOT auto-detect: the file contains ! in comments (e.g. "...tables!!!") + // which would cause false-positive terminator detection. + // + // Use -t (built-in semicolon mode) to avoid passing -td; directly, because + // bash treats ; as a command separator inside su -c "..." strings in some + // DB2 container environments, producing DB21002E. + // + // On macOS, the temp file lives on the host; docker cp it into the container + // so that "db2 -f /path" resolves correctly inside the container. + sqlRunPath := tmpPath + if darwinContainerName != "" { + // Use a /tmp path in the container (the macOS host temp path like + // /var/folders/... does not exist inside the container). + ctrSQLPath := "/tmp/" + filepath.Base(tmpPath) + if cpErr := exec.Command("docker", "cp", tmpPath, darwinContainerName+":"+ctrSQLPath).Run(); cpErr != nil { + fmt.Printf("db2test: docker cp asnctlw.sql warning: %v\n", cpErr) + } else { + sqlRunPath = ctrSQLPath + defer exec.Command("docker", "exec", darwinContainerName, "rm", "-f", ctrSQLPath).Run() //nolint:errcheck + } + } + out, _ := runAsDB2Inst1(fmt.Sprintf( + "db2 connect to %s && db2 -t -f '%s'", + db2Database, sqlRunPath, + )) + fmt.Printf("db2test: asnctlw.sql output:\n%s\n", out) + + // Apply fix-pack migration scripts to add columns / tables introduced in + // DB2 11.4+ (WARNTXSZ, WARNLOGAPI, STALE, etc. in CAPPARMS). + // The script uses ';' as the statement terminator. + // It also uses '!capschema!' and '!captablespace!' as template placeholders + // that must be substituted before running. + for _, fpScript := range []string{ + findAsnScript("asncapluwv1140fp.sql"), + } { + if fpScript == "" { + continue + } + var fpContent []byte + var readErr error + if darwinContainerName != "" { + // fpScript lives inside the container; read it via docker exec cat. + fpContent, readErr = exec.Command("docker", "exec", darwinContainerName, "cat", fpScript).Output() + } else { + fpContent, readErr = os.ReadFile(fpScript) + } + if readErr != nil { + fmt.Printf("db2test: skipping %s (read error: %v)\n", fpScript, readErr) + continue + } + fpModified := rewriteASNSchemaInSQL(string(fpContent), "ASN", "ASNCDC") + // Replace template placeholders — the script ships with !capschema! and + // !captablespace! tokens (both upper and lower case) that must be swapped + // before execution. + fpModified = strings.ReplaceAll(fpModified, "!CAPSCHEMA!", "ASNCDC") + fpModified = strings.ReplaceAll(fpModified, "!capschema!", "ASNCDC") + fpModified = strings.ReplaceAll(fpModified, "!CAPTABLESPACE!", "USERSPACE1") + fpModified = strings.ReplaceAll(fpModified, "!captablespace!", "USERSPACE1") + fpTmp, tmpErr := os.CreateTemp("", "asncdc-fp-*.sql") + if tmpErr != nil { + continue + } + fpTmpPath := fpTmp.Name() + _, _ = fpTmp.WriteString(fpModified) + fpTmp.Close() + _ = os.Chmod(fpTmpPath, 0o644) + defer os.Remove(fpTmpPath) + + fpRunPath := fpTmpPath + if darwinContainerName != "" { + ctrFPPath := "/tmp/" + filepath.Base(fpTmpPath) + if cpErr := exec.Command("docker", "cp", fpTmpPath, darwinContainerName+":"+ctrFPPath).Run(); cpErr != nil { + fmt.Printf("db2test: docker cp fp script warning: %v\n", cpErr) + } else { + fpRunPath = ctrFPPath + defer exec.Command("docker", "exec", darwinContainerName, "rm", "-f", ctrFPPath).Run() //nolint:errcheck + } + } + fpOut, _ := runAsDB2Inst1(fmt.Sprintf( + "db2 connect to %s && db2 -t -f '%s'", + db2Database, fpRunPath, + )) + fmt.Printf("db2test: %s output:\n%s\n", filepath.Base(fpScript), fpOut) + } + + if err := ensureMissingColumns(ctx, db); err != nil { + fmt.Printf("db2test: ensureMissingColumns warning: %v\n", err) + } + + // Ensure IBMQREP_CAPCMD exists so asnccmd can send reinit signals. + if _, createErr := db.ExecContext(ctx, ` + CREATE TABLE ASNCDC.IBMQREP_CAPCMD ( + QMGR VARCHAR(48), + RECVQ VARCHAR(48), + IGNERR CHAR(1) NOT NULL WITH DEFAULT 'N', + COMMAND VARCHAR(2048) NOT NULL, + STATUS CHAR(1) WITH DEFAULT 'P', + CAPSCHEMA VARCHAR(128) NOT NULL, + CMD_TIME TIMESTAMP NOT NULL WITH DEFAULT CURRENT TIMESTAMP + ) + `); createErr != nil && !isAlreadyExistsErr(createErr) { + fmt.Printf("db2test: IBMQREP_CAPCMD creation warning: %v\n", createErr) + } + + return ensureCAPPARMSRow(ctx, db) +} + +// ensureMissingColumns adds columns that are present in newer DB2 fix-packs but +// absent from the 11.4.0 init scripts we execute. Safe to call on every +// startup: isAlreadyExistsErr suppresses SQL0612N duplicate-column errors. +func ensureMissingColumns(ctx context.Context, db *sql.DB) error { + // IBMSNAP_CAPPARMS columns added in 11.4.0 fix-packs. + for _, stmt := range []string{ + "ALTER TABLE ASNCDC.IBMSNAP_CAPPARMS ADD COLUMN STALE INTEGER NOT NULL WITH DEFAULT 3600", + "ALTER TABLE ASNCDC.IBMSNAP_CAPPARMS ADD COLUMN WARNTXSZ INTEGER NOT NULL WITH DEFAULT 0", + "ALTER TABLE ASNCDC.IBMSNAP_CAPPARMS ADD COLUMN WARNLOGAPI INTEGER NOT NULL WITH DEFAULT 0", + } { + if _, err := db.ExecContext(ctx, stmt); err != nil && !isAlreadyExistsErr(err) { + fmt.Printf("db2test: CAPPARMS column warning: %v\n", err) + } + } + // IBMQREP_COLVERSION.COORDINATETYPE added in DB2 12.1 Q Capture migration. + // asncap 12.1.4.0 crashes with SQL0206N if this column is absent. + if _, err := db.ExecContext(ctx, + "ALTER TABLE ASNCDC.IBMQREP_COLVERSION ADD COLUMN COORDINATETYPE VARCHAR(8) WITH DEFAULT NULL", + ); err != nil && !isAlreadyExistsErr(err) { + fmt.Printf("db2test: IBMQREP_COLVERSION COORDINATETYPE warning: %v\n", err) + } + return nil +} + +// rewriteASNSchemaInSQL replaces all occurrences of fromSchema (used as a SQL +// identifier) with toSchema in a DB2 SQL script. The replacement is +// word-boundary–aware to avoid partial matches like "ORASN.TABLE". +func rewriteASNSchemaInSQL(script, fromSchema, toSchema string) string { + q := regexp.QuoteMeta(fromSchema) + // Qualified names: "ASN.TABLE" → "ASNCDC.TABLE" + result := regexp.MustCompile(`\b`+q+`\.`).ReplaceAllString(script, toSchema+".") + // CREATE/USE SCHEMA declarations: "SCHEMA ASN" → "SCHEMA ASNCDC" + result = regexp.MustCompile(`\bSCHEMA\s+`+q+`\b`).ReplaceAllString(result, "SCHEMA "+toSchema) + return result +} + +// stripTablespaceForSinglePartition removes multi-partition tablespace clauses from +// IBM's asnctlw.sql so it works on single-partition DB2 12.1 containers. +// The script was written for DB2 9.x/10.x with multi-partition syntax. +// We strip: +// - Entire "CREATE TABLESPACE … ;" blocks (we don't need separate tablespaces) +// - "IN TSASNCA;", "IN TSASNUOW;", "IN TSASNAA;" lines (table placement clauses) +// - "IN TSASNCA" without semicolon (inline placement clauses) +// +// Tables land in the default tablespace, which is fine for tests. +func stripTablespaceForSinglePartition(sql string) string { + lines := strings.Split(sql, "\n") + var out []string + skipUntilSemicolon := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Start of a CREATE TABLESPACE block — skip until we see the closing ");". + if strings.HasPrefix(trimmed, "CREATE") && strings.Contains(trimmed, "TABLESPACE") { + skipUntilSemicolon = true + } + if skipUntilSemicolon { + if strings.HasSuffix(trimmed, ");") || trimmed == ");" { + skipUntilSemicolon = false + } + continue + } + + // Strip "IN TSASN*" table-placement lines. When the line ends with ';' + // (the statement terminator), emit just ';' so the CREATE TABLE statement + // stays properly terminated. Without this, removing "IN TSASNCA;" leaves + // the enclosing CREATE TABLE without a terminator and db2 runs statements together. + if regexp.MustCompile(`(?i)^\s*IN\s+TSASN\w*\s*;?\s*$`).MatchString(line) { + if strings.Contains(line, ";") { + out = append(out, ";") + } + continue + } + out = append(out, line) + } + return strings.Join(out, "\n") +} + +// ensureCAPPARMSRow inserts a CAPPARMS row if the table is empty. +// asncap will not start unless IBMSNAP_CAPPARMS has exactly one row. +func ensureCAPPARMSRow(ctx context.Context, db *sql.DB) error { + var n int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM ASNCDC.IBMSNAP_CAPPARMS", + ).Scan(&n); err != nil { + return fmt.Errorf("checking CAPPARMS: %w", err) + } + if n > 0 { + return nil + } + + // Preferred: copy the IBM-supplied defaults from the ASN schema. + if _, err := db.ExecContext(ctx, ` + INSERT INTO ASNCDC.IBMSNAP_CAPPARMS + SELECT * FROM ASN.IBMSNAP_CAPPARMS FETCH FIRST 1 ROW ONLY + `); err == nil { + fmt.Printf("db2test: CAPPARMS row copied from ASN schema\n") + return nil + } + + // Fallback 1: all-defaults row (works when every column has a DEFAULT). + if _, err := db.ExecContext(ctx, "INSERT INTO ASNCDC.IBMSNAP_CAPPARMS DEFAULT VALUES"); err == nil { + fmt.Printf("db2test: CAPPARMS DEFAULT VALUES row inserted\n") + return nil + } + + // Fallback 2: explicit minimal row with the columns we know about. + if _, err := db.ExecContext(ctx, + "INSERT INTO ASNCDC.IBMSNAP_CAPPARMS (STARTMODE, ARCH_LEVEL) VALUES ('WARMSI', '1018')", + ); err != nil { + return fmt.Errorf("inserting minimal CAPPARMS row: %w", err) + } + fmt.Printf("db2test: CAPPARMS minimal row inserted\n") + return nil +} + +// isAlreadyExistsErr returns true when a DB2 error indicates the object already +// exists (SQL0601N) or a unique index is already defined (SQL0603N). +func isAlreadyExistsErr(err error) bool { + if err == nil { + return false + } + s := err.Error() + // SQL0601N — object already exists (table, index, etc.) + // SQL0603N — unique index already defined + // SQL0612N — duplicate column name (ADD COLUMN for already-existing column) + return strings.Contains(s, "SQL0601N") || strings.Contains(s, "already exists") || + strings.Contains(s, "SQL0603N") || strings.Contains(s, "SQL0612N") +} + +// startAsncapLocal starts the asncap SQL Replication capture daemon if it is +// not already running. asncap must run as the db2inst1 instance owner. +// +// On macOS (darwin), process checks and log reads are routed through docker exec +// because asncap runs inside the DB2 container, not on the macOS host. +// +// Before starting, any active (STATE='A') registrations for tables that no +// longer have the DATA CAPTURE CHANGES attribute are deactivated to STATE='I'. +// asncap stops immediately with ASN0009E if it encounters such a registration, +// which happens when a source table was dropped-and-recreated between runs in +// a persistent test container. +func startAsncapLocal() error { + // Deactivate stale registrations that lack DATA CAPTURE CHANGES. + // These would cause asncap to crash on startup with ASN0009E. + // We use a best-effort DB2 shell call; errors are non-fatal. + deactivateSQL := "UPDATE ASNCDC.IBMSNAP_REGISTER SET STATE = 'I' WHERE STATE = 'A' AND NOT EXISTS (SELECT 1 FROM SYSCAT.TABLES T WHERE T.TABSCHEMA = ASNCDC.IBMSNAP_REGISTER.SOURCE_OWNER AND T.TABNAME = ASNCDC.IBMSNAP_REGISTER.SOURCE_TABLE AND T.DATACAPTURE <> 'N')" + deactivateScript := fmt.Sprintf(`db2 connect to %s && db2 "%s"`, db2Database, deactivateSQL) + if out, err := runAsDB2Inst1(deactivateScript); err != nil { + fmt.Printf("db2test: deactivate stale registrations warning (non-fatal): %v\n%s\n", err, out) + } + + // checkPgrep returns the pids of asncap processes for TESTDB, empty if none. + checkPgrep := func() string { + if darwinContainerName != "" { + out, _ := exec.Command("docker", "exec", darwinContainerName, "pgrep", "-f", "capture_server="+db2Database).Output() + return strings.TrimSpace(string(out)) + } + out, _ := exec.Command("pgrep", "-f", "capture_server="+db2Database).Output() + return strings.TrimSpace(string(out)) + } + // readAsncapLog returns the contents of /tmp/asncap.log (in container or host). + readAsncapLog := func() []byte { + if darwinContainerName != "" { + out, _ := exec.Command("docker", "exec", darwinContainerName, "cat", "/tmp/asncap.log").Output() + return out + } + out, _ := os.ReadFile("/tmp/asncap.log") + return out + } + + // Check if a capture process for TESTDB is already running. + if checkPgrep() != "" { + fmt.Printf("db2test: asncap already running (pid: %s)\n", checkPgrep()) + return nil + } + + fmt.Printf("db2test: starting asncap capture daemon for %s/ASNCDC...\n", db2Database) + out, err := runAsDB2Inst1(fmt.Sprintf( + "nohup asncap capture_server=%s capture_schema=ASNCDC > /tmp/asncap.log 2>&1 &", + db2Database, + )) + if err != nil { + return fmt.Errorf("launching asncap: %v: %s", err, out) + } + + // Wait up to 30 s for the process to appear, then wait for ASN0100I + // (successfully initialized) before returning. Returning as soon as the + // PID appears is too early: asncap needs another ~2 s to read the DB2 log + // and finish init. If we send "asnccmd reinit" during that window the + // signal arrives before asncap is ready and is silently dropped. + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + if pids := checkPgrep(); pids != "" { + fmt.Printf("db2test: asncap started (pid: %s)\n", pids) + // Wait for the "initialized successfully" log entry before returning + // so that a subsequent asnccmd reinit is processed correctly. + initDeadline := time.Now().Add(30 * time.Second) + for time.Now().Before(initDeadline) { + if logOut := readAsncapLog(); len(logOut) > 0 && + (strings.Contains(string(logOut), "ASN0100I") || strings.Contains(string(logOut), "ASN0109I")) { + fmt.Printf("db2test: asncap fully initialized\n") + return nil + } + time.Sleep(1 * time.Second) + } + fmt.Printf("db2test: WARNING: asncap started but initialization log not seen within 30s\n") + return nil + } + time.Sleep(2 * time.Second) + } + // Log the asncap output to help diagnose startup failures. + if logOut := readAsncapLog(); len(logOut) > 0 { + fmt.Printf("db2test: asncap startup log:\n%s\n", logOut) + } + return errors.New("asncap did not start within 30 s (see /tmp/asncap.log)") +} + +// addTableLocal creates the CD (Change Data) table for a source table and +// registers it in ASNCDC.IBMSNAP_REGISTER. This is the local-mode replacement +// for the ASNCDC.ADDTABLE SQL UDF which requires the IBM ASNTOOLS package. +// +// This function mirrors IBM's ASNCDC.ADDTABLE stored procedure: +// 1. ALTER TABLE … DATA CAPTURE CHANGES +// 2. CREATE TABLE ASNCDC."CDC__
" with VARCHAR FOR BIT DATA cols, +// UNIQUE INDEX, and VOLATILE CARDINALITY +// 3. INSERT INTO ASNCDC.IBMSNAP_REGISTER with STATE='A', NULL synchpoints, +// ARCH_LEVEL='0801', OPTION_FLAGS='NNNN' +// 4. INSERT INTO ASNCDC.IBMSNAP_PRUNCNTL (subscriber entry required by asncap) +func addTableLocal(ctx context.Context, db *sql.DB, sourceSchema, sourceTable string) error { + // IBM's ADDTABLE names the CD table "CDC__
". + cdTable := "CDC_" + sourceSchema + "_" + sourceTable + + // Check which parts of the setup are already in place. + var cdExists, regExists int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM SYSCAT.TABLES WHERE TABSCHEMA='ASNCDC' AND TABNAME=?", + cdTable, + ).Scan(&cdExists); err != nil { + return fmt.Errorf("checking CD table existence: %w", err) + } + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_OWNER=? AND SOURCE_TABLE=?", + sourceSchema, sourceTable, + ).Scan(®Exists); err != nil { + return fmt.Errorf("checking IBMSNAP_REGISTER for %s.%s: %w", sourceSchema, sourceTable, err) + } + + // Both already present — check that the column types are correct (DB2 12 + // requires VARCHAR(16) FOR BIT DATA for IBMSNAP_COMMITSEQ; old containers + // may have CHAR(10) from a previous version of this code). If the types + // are wrong, drop and recreate the CD table so asncap can INSERT 16-byte + // commit sequence numbers without hitting SQLCODE -311. + if cdExists > 0 && regExists > 0 { + var commitseqType string + var commitseqLen int + _ = db.QueryRowContext(ctx, + "SELECT TYPENAME, LENGTH FROM SYSCAT.COLUMNS WHERE TABSCHEMA='ASNCDC' AND TABNAME=? AND COLNAME='IBMSNAP_COMMITSEQ'", + cdTable, + ).Scan(&commitseqType, &commitseqLen) + + if commitseqType == "VARCHAR" && commitseqLen == 16 { + fmt.Printf("db2test: CD table ASNCDC.%s and registration already exist (correct schema)\n", cdTable) + if _, err := db.ExecContext(ctx, + fmt.Sprintf("ALTER TABLE %s.%s DATA CAPTURE CHANGES", sourceSchema, sourceTable), + ); err != nil { + return fmt.Errorf("re-enabling DATA CAPTURE CHANGES on %s.%s: %w", sourceSchema, sourceTable, err) + } + return nil + } + + // Wrong column type — drop and recreate. + fmt.Printf("db2test: CD table ASNCDC.%s has wrong IBMSNAP_COMMITSEQ type (%s(%d)); dropping to recreate with VARCHAR(16)\n", + cdTable, commitseqType, commitseqLen) + _, _ = db.ExecContext(ctx, "DROP TABLE ASNCDC."+cdTable) + _, _ = db.ExecContext(ctx, + "DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_OWNER=? AND SOURCE_TABLE=?", + sourceSchema, sourceTable, + ) + cdExists = 0 + regExists = 0 + } + + // Stale REGISTER row without a corresponding CD table: delete it so we can + // recreate a consistent pair below. + if regExists > 0 && cdExists == 0 { + if _, err := db.ExecContext(ctx, + "DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_OWNER=? AND SOURCE_TABLE=?", + sourceSchema, sourceTable, + ); err != nil { + return fmt.Errorf("removing stale IBMSNAP_REGISTER row for %s.%s: %w", sourceSchema, sourceTable, err) + } + fmt.Printf("db2test: removed stale IBMSNAP_REGISTER row for %s.%s\n", sourceSchema, sourceTable) + regExists = 0 + } + + // Create CD table if missing. + if cdExists == 0 { + // Discover source table columns to mirror in the CD table. + rows, err := db.QueryContext(ctx, ` + SELECT COLNAME, TYPENAME, LENGTH, SCALE + FROM SYSCAT.COLUMNS + WHERE TABSCHEMA = ? AND TABNAME = ? + ORDER BY COLNO + `, sourceSchema, sourceTable) + if err != nil { + return fmt.Errorf("querying source table columns for %s.%s: %w", sourceSchema, sourceTable, err) + } + defer rows.Close() + + type colDef struct { + Name, TypeName string + Len, Scale int + } + var cols []colDef + for rows.Next() { + var c colDef + if err := rows.Scan(&c.Name, &c.TypeName, &c.Len, &c.Scale); err != nil { + return err + } + cols = append(cols, c) + } + if err := rows.Err(); err != nil { + return err + } + if len(cols) == 0 { + return fmt.Errorf("source table %s.%s has no columns or does not exist", sourceSchema, sourceTable) + } + + // Build the CREATE TABLE DDL. All source columns are nullable in the CD + // table because asncap writes only changed columns for UPDATE operations. + var sb strings.Builder + fmt.Fprintf(&sb, "CREATE TABLE ASNCDC.%s (\n", cdTable) + sb.WriteString(" IBMSNAP_OPERATION CHAR(1) NOT NULL,\n") + // DB2 12 uses VARCHAR(16) FOR BIT DATA for the 16-byte log LSN; + // CHAR(10) causes SQLCODE -311 when asncap tries to INSERT. + sb.WriteString(" IBMSNAP_COMMITSEQ VARCHAR(16) FOR BIT DATA NOT NULL,\n") + sb.WriteString(" IBMSNAP_INTENTSEQ VARCHAR(16) FOR BIT DATA NOT NULL,\n") + sb.WriteString(" IBMSNAP_LOGMARKER TIMESTAMP") + for _, c := range cols { + fmt.Fprintf(&sb, ",\n %s %s", strings.TrimSpace(c.Name), db2TypeDef(c.TypeName, c.Len, c.Scale)) + } + sb.WriteString("\n)") + ddl := sb.String() + fmt.Printf("db2test: creating CD table:\n%s\n", ddl) + + if _, err := db.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("creating CD table ASNCDC.%s: %w", cdTable, err) + } + } + + // Enable DATA CAPTURE CHANGES on the source table. asncap requires this + // attribute; without it it logs ASN0009E and stops. The ALTER is idempotent. + if _, err := db.ExecContext(ctx, + fmt.Sprintf("ALTER TABLE %s.%s DATA CAPTURE CHANGES", sourceSchema, sourceTable), + ); err != nil { + return fmt.Errorf("enabling DATA CAPTURE CHANGES on %s.%s: %w", sourceSchema, sourceTable, err) + } + fmt.Printf("db2test: DATA CAPTURE CHANGES enabled on %s.%s\n", sourceSchema, sourceTable) + + // Create the unique index on (COMMITSEQ, INTENTSEQ) — required by IBM's ADDTABLE procedure. + // Idempotent: isAlreadyExistsErr suppresses SQL0601N if the index already exists. + indexSQL := fmt.Sprintf( + `CREATE UNIQUE INDEX ASNCDC."IXCDC_%s_%s" ON ASNCDC."%s" (IBMSNAP_COMMITSEQ ASC, IBMSNAP_INTENTSEQ ASC) PCTFREE 0 MINPCTUSED 0`, + sourceSchema, sourceTable, cdTable, + ) + if _, err := db.ExecContext(ctx, indexSQL); err != nil && !isAlreadyExistsErr(err) { + return fmt.Errorf("creating unique index on CD table ASNCDC.%s: %w", cdTable, err) + } + + // VOLATILE CARDINALITY helps the DB2 optimizer use index scans on the CT table. + volatileSQL := fmt.Sprintf(`ALTER TABLE ASNCDC."%s" VOLATILE CARDINALITY`, cdTable) + if _, err := db.ExecContext(ctx, volatileSQL); err != nil { + fmt.Printf("db2test: VOLATILE CARDINALITY warning (non-fatal): %v\n", err) + } + + // Insert REGISTER row if missing. + if regExists == 0 { + // Key values required by asncap: + // ARCH_LEVEL = '0801' — the value asncap recognises regardless of DB2 version. + // OPTION_FLAGS = 'NNNN' — required; '0000' causes asncap to skip capture. + // CD_OLD_SYNCHPOINT = null, CD_NEW_SYNCHPOINT = null, SYNCHPOINT = null + // — asncap sets these on reinit. Seeding with a stale + // log position caused asncap to silently skip capture. + // STATE = 'A' — Active at insert time; no separate UPDATE needed. + // CONFLICT_LEVEL = '0', CHG_UPD_TO_DEL_INS = 'Y', CHGONLY = 'N', + // RECAPTURE = 'Y' — standard CDC capture flags. + regSQL := fmt.Sprintf(` + INSERT INTO ASNCDC.IBMSNAP_REGISTER ( + SOURCE_OWNER, SOURCE_TABLE, SOURCE_VIEW_QUAL, GLOBAL_RECORD, + SOURCE_STRUCTURE, SOURCE_CONDENSED, SOURCE_COMPLETE, + CD_OWNER, CD_TABLE, PHYS_CHANGE_OWNER, PHYS_CHANGE_TABLE, + CD_OLD_SYNCHPOINT, CD_NEW_SYNCHPOINT, + DISABLE_REFRESH, + ARCH_LEVEL, CONFLICT_LEVEL, CHG_UPD_TO_DEL_INS, + CHGONLY, RECAPTURE, OPTION_FLAGS, + STOP_ON_ERROR, STATE, STATE_INFO + ) VALUES ( + '%s', '%s', 0, 'N', + 1, 'Y', 'Y', + 'ASNCDC', '%s', 'ASNCDC', '%s', + null, null, + 0, + '0801', '0', 'Y', + 'N', 'Y', 'NNNN', + 'Y', 'A', null + ) + `, sourceSchema, sourceTable, cdTable, cdTable) + + if _, err := db.ExecContext(ctx, regSQL); err != nil { + return fmt.Errorf("inserting IBMSNAP_REGISTER row for %s.%s: %w", sourceSchema, sourceTable, err) + } + } + fmt.Printf("db2test: registered %s.%s → ASNCDC.%s (STATE=A, ARCH_LEVEL=0801, OPTION_FLAGS=NNNN)\n", + sourceSchema, sourceTable, cdTable) + + // Step 4: Insert the IBMSNAP_PRUNCNTL subscriber row. + // + // asncap requires at least one PRUNCNTL entry for each registered source table + // to know that a subscriber exists. Without this row asncap silently skips + // capture for the table even though IBMSNAP_REGISTER STATE='A'. + pruncntlSQL := fmt.Sprintf(` + INSERT INTO ASNCDC.IBMSNAP_PRUNCNTL ( + TARGET_SERVER, TARGET_OWNER, TARGET_TABLE, + SYNCHTIME, SYNCHPOINT, + SOURCE_OWNER, SOURCE_TABLE, SOURCE_VIEW_QUAL, + APPLY_QUAL, SET_NAME, + CNTL_SERVER, TARGET_STRUCTURE, CNTL_ALIAS, + PHYS_CHANGE_OWNER, PHYS_CHANGE_TABLE, + MAP_ID + ) VALUES ( + 'KAFKA', '%s', '%s', + NULL, NULL, + '%s', '%s', 0, + 'KAFKAQUAL', 'SET001', + (SELECT CURRENT SERVER FROM SYSIBM.SYSDUMMY1), + 8, + (SELECT CURRENT SERVER FROM SYSIBM.SYSDUMMY1), + 'ASNCDC', '%s', + (SELECT CAST(COALESCE(MAX(CAST(MAP_ID AS INT)), 0) + 1 AS VARCHAR(10)) + FROM ASNCDC.IBMSNAP_PRUNCNTL) + ) + `, sourceSchema, sourceTable, sourceSchema, sourceTable, cdTable) + + if _, err := db.ExecContext(ctx, pruncntlSQL); err != nil { + return fmt.Errorf("inserting IBMSNAP_PRUNCNTL row for %s.%s: %w", sourceSchema, sourceTable, err) + } + fmt.Printf("db2test: inserted IBMSNAP_PRUNCNTL subscriber row for %s.%s\n", sourceSchema, sourceTable) + + return nil +} + +// db2TypeDef converts SYSCAT.COLUMNS metadata to a SQL type definition string. +func db2TypeDef(typename string, length, scale int) string { + switch strings.ToUpper(strings.TrimSpace(typename)) { + case "INTEGER", "INT", "BIGINT", "SMALLINT", "REAL", "DOUBLE", "DATE", "TIME", "TIMESTAMP", "BOOLEAN": + return strings.ToUpper(strings.TrimSpace(typename)) + case "CHARACTER", "CHAR": + return fmt.Sprintf("CHAR(%d)", length) + case "VARCHAR", "CHARACTER VARYING": + return fmt.Sprintf("VARCHAR(%d)", length) + case "DECIMAL", "NUMERIC": + return fmt.Sprintf("DECIMAL(%d,%d)", length, scale) + case "FLOAT": + return fmt.Sprintf("FLOAT(%d)", length) + case "CLOB": + return fmt.Sprintf("CLOB(%d)", length) + case "BLOB": + return fmt.Sprintf("BLOB(%d)", length) + default: + return strings.TrimSpace(typename) + } +} diff --git a/internal/impl/db2/driver.go b/internal/impl/db2/driver.go new file mode 100644 index 0000000000..14a37a1b73 --- /dev/null +++ b/internal/impl/db2/driver.go @@ -0,0 +1,719 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +// Package db2 provides a database/sql driver for IBM DB2 implemented without +// CGO. The driver is named "db2-cli" and registered via database/sql.Register +// in the package's init function. +// +// The driver loads the IBM DB2 CLI shared library at runtime using +// github.com/ebitengine/purego (dlopen/LoadLibrary), so no C compiler or CGO +// toolchain is required at build time. The CLI (ODBC-compatible C API) call +// sequence is: SQLAllocHandle → SQLDriverConnect → SQLPrepare → +// SQLBindParameter → SQLExecute → SQLFetch → SQLGetData → SQLFreeHandle. +// +// Parameter binding uses SQL_C_CHAR / SQL_VARCHAR for all types. CDC stream +// queries embed binary CSN values as hex literals (X'...') to avoid needing +// SQL_C_BINARY for CHAR FOR BIT DATA columns. +// +// Memory safety: bindParams returns both the data buffers (bufs) and the +// indicator lengths (inds). Both must be kept alive via runtime.KeepAlive until +// after SQLExecute returns. The Go GC does not track C-side pointer references, +// so any local variable whose address is passed to SQLBindParameter must be +// heap-allocated (via a slice) and explicitly retained. +package db2 + +import ( + "context" + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "regexp" + "runtime" + "strconv" + "strings" + "unsafe" + + "golang.org/x/text/encoding/unicode" + + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2cli" +) + +// utf16LE is cached at package level so decodeUTF16LE avoids reconstructing +// the encoding descriptor on every GRAPHIC/VARGRAPHIC column value. +var utf16LE = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) + +// connSecretRE matches credential keywords in DB2 connection-string fragments +// that some driver versions include verbatim in error messages. Matches empty +// values (PWD=;) and the UID keyword in addition to PWD. +var connSecretRE = regexp.MustCompile(`(?i)\b(PWD|PASSWORD|UID|USERNAME|SECURITY)=[^\s;]*`) + +// sanitizeConnErr replaces any credential keyword values in a DB2 connection +// error message with "***" before the error is wrapped and propagated to logs. +func sanitizeConnErr(err error) error { + if err == nil { + return nil + } + sanitized := connSecretRE.ReplaceAllStringFunc(err.Error(), func(m string) string { + // Preserve the keyword, replace only the value. + idx := strings.IndexByte(m, '=') + return m[:idx+1] + "***" + }) + if sanitized == err.Error() { + return err + } + return errors.New(sanitized) +} + +// cTypeForSQLType returns the ODBC C transfer type for a given SQL type. +// Using the wrong C type causes silent data corruption: +// - SQL_C_CHAR on BLOB columns truncates at the first zero byte. +// - SQL_C_CHAR on VARGRAPHIC/GRAPHIC returns raw UTF-16LE bytes that look +// like garbage when interpreted as UTF-8. +func cTypeForSQLType(sqlType db2cli.SQLSMALLINT) db2cli.SQLSMALLINT { + switch sqlType { + case db2cli.SQL_BINARY, db2cli.SQL_VARBINARY, db2cli.SQL_LONGVARBINARY, db2cli.SQL_BLOB: + return db2cli.SQL_C_BINARY + case db2cli.SQL_WCHAR, db2cli.SQL_WVARCHAR, db2cli.SQL_WLONGVARCHAR, + db2cli.SQL_GRAPHIC, db2cli.SQL_VARGRAPHIC, db2cli.SQL_LONGVARGRAPHIC, db2cli.SQL_DBCLOB: + return db2cli.SQL_C_WCHAR + default: + return db2cli.SQL_C_CHAR + } +} + +// db2Driver implements database/sql/driver.Driver using purego DB2 CLI bindings. +type db2Driver struct{} + +func init() { + sql.Register("db2-cli", &db2Driver{}) +} + +func (*db2Driver) Open(dsn string) (driver.Conn, error) { + if err := db2cli.LoadLibrary(); err != nil { + return nil, fmt.Errorf("loading DB2 CLI library: %w", err) + } + + var henv db2cli.SQLHENV + ret := db2cli.SQLAllocHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(db2cli.SQL_NULL_HENV), (*db2cli.SQLHANDLE)(&henv)) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + return nil, errors.New("allocating environment handle") + } + + ret = db2cli.SQLSetEnvAttr(henv, db2cli.SQL_ATTR_ODBC_VERSION, db2cli.SQL_OV_ODBC3, 0) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(henv)) + return nil, errors.New("setting ODBC version") + } + + var hdbc db2cli.SQLHDBC + ret = db2cli.SQLAllocHandle(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(henv), (*db2cli.SQLHANDLE)(&hdbc)) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(henv)) + return nil, errors.New("allocating connection handle") + } + + _, ret = db2cli.SQLDriverConnect(hdbc, dsn) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + err := db2cli.GetLastError(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(hdbc)) + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(hdbc)) + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(henv)) + return nil, fmt.Errorf("connecting: %w", sanitizeConnErr(err)) + } + + return &db2Conn{henv: henv, hdbc: hdbc}, nil +} + +// db2Conn implements driver.Conn. +type db2Conn struct { + henv db2cli.SQLHENV + hdbc db2cli.SQLHDBC +} + +func (c *db2Conn) Prepare(query string) (driver.Stmt, error) { + var hstmt db2cli.SQLHSTMT + ret := db2cli.SQLAllocHandle(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(c.hdbc), (*db2cli.SQLHANDLE)(&hstmt)) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + return nil, errors.New("allocating statement handle") + } + + ret = db2cli.SQLPrepare(hstmt, query) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + err := db2cli.GetLastError(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(hstmt)) + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(hstmt)) + return nil, fmt.Errorf("preparing statement: %w", err) + } + + return &db2Stmt{hstmt: hstmt, conn: c}, nil +} + +func (c *db2Conn) Close() error { + if c.hdbc != 0 { + db2cli.SQLDisconnect(c.hdbc) + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(c.hdbc)) + c.hdbc = 0 + } + if c.henv != 0 { + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_ENV, db2cli.SQLHANDLE(c.henv)) + c.henv = 0 + } + return nil +} + +func (c *db2Conn) Begin() (driver.Tx, error) { + ret := db2cli.SQLSetConnectAttr(c.hdbc, db2cli.SQL_ATTR_AUTOCOMMIT, db2cli.SQL_AUTOCOMMIT_OFF, 0) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + return nil, errors.New("beginning transaction") + } + return &db2Tx{conn: c}, nil +} + +// BeginTx implements driver.ConnBeginTx so that sql.TxOptions.Isolation is honoured. +// Without this, database/sql falls back to Begin() and silently drops the isolation level. +func (c *db2Conn) BeginTx(_ context.Context, opts driver.TxOptions) (driver.Tx, error) { + if opts.Isolation != driver.IsolationLevel(sql.LevelDefault) { + isoConst, err := isolationLevelToDB2(sql.IsolationLevel(opts.Isolation)) + if err != nil { + return nil, err + } + ret := db2cli.SQLSetConnectAttr(c.hdbc, db2cli.SQL_ATTR_TXN_ISOLATION, isoConst, 0) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + return nil, fmt.Errorf("setting transaction isolation level: %w", + db2cli.GetLastError(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(c.hdbc))) + } + } + ret := db2cli.SQLSetConnectAttr(c.hdbc, db2cli.SQL_ATTR_AUTOCOMMIT, db2cli.SQL_AUTOCOMMIT_OFF, 0) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + return nil, errors.New("beginning transaction") + } + return &db2Tx{conn: c}, nil +} + +// QueryContext implements driver.QueryerContext so that context cancellation +// interrupts an in-flight SQL query via SQLCancel. Without this, a blocked +// SQLExecute cannot be interrupted, causing goroutine leaks on context cancel. +func (c *db2Conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + vals := make([]driver.Value, len(args)) + for i, a := range args { + vals[i] = a.Value + } + stmt, err := c.Prepare(query) + if err != nil { + return nil, err + } + db2s := stmt.(*db2Stmt) + + type result struct { + rows driver.Rows + err error + } + done := make(chan result, 1) + go func() { + rows, e := db2s.Query(vals) + done <- result{rows, e} + }() + + select { + case res := <-done: + if res.err != nil { + _ = stmt.Close() + } + return res.rows, res.err + case <-ctx.Done(): + db2cli.SQLCancel(db2s.hstmt) + // Drain the result so the goroutine can exit, then discard any + // successfully-produced rows to avoid leaking the 32 KB colBuf. + res := <-done + if res.rows != nil { + _ = res.rows.Close() + } + _ = stmt.Close() + return nil, ctx.Err() + } +} + +// ExecContext implements driver.ExecerContext so that context cancellation +// interrupts an in-flight DML/DDL statement via SQLCancel. Without this, +// database/sql falls back to Prepare+Exec which blocks without cancel support. +func (c *db2Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + vals := make([]driver.Value, len(args)) + for i, a := range args { + vals[i] = a.Value + } + stmt, err := c.Prepare(query) + if err != nil { + return nil, err + } + db2s := stmt.(*db2Stmt) + + type result struct { + res driver.Result + err error + } + done := make(chan result, 1) + go func() { + res, e := db2s.Exec(vals) + done <- result{res, e} + }() + + select { + case res := <-done: + // Always free the statement handle — db2Result holds no reference to + // the stmt, so without this call the SQLHSTMT leaks until connection + // close. Under high-frequency ExecContext call patterns (e.g. bulk INSERTs) + // this exhausts the CLI driver's handle pool (HY014 CLI0129E). + _ = stmt.Close() + return res.res, res.err + case <-ctx.Done(): + db2cli.SQLCancel(db2s.hstmt) + <-done // wait for goroutine to finish after cancel + _ = stmt.Close() + return nil, ctx.Err() + } +} + +// isolationLevelToDB2 maps a sql.IsolationLevel to the corresponding DB2 CLI constant. +func isolationLevelToDB2(level sql.IsolationLevel) (uintptr, error) { + switch level { + case sql.LevelReadUncommitted: + return db2cli.SQL_TXN_READ_UNCOMMITTED, nil + case sql.LevelReadCommitted: + return db2cli.SQL_TXN_READ_COMMITTED, nil + case sql.LevelRepeatableRead: + return db2cli.SQL_TXN_REPEATABLE_READ, nil + case sql.LevelSerializable: + return db2cli.SQL_TXN_SERIALIZABLE, nil + default: + return 0, fmt.Errorf("unsupported transaction isolation level: %v", level) + } +} + +// db2Tx implements driver.Tx. +type db2Tx struct { + conn *db2Conn +} + +func (tx *db2Tx) Commit() error { + ret := db2cli.SQLEndTran(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(tx.conn.hdbc), db2cli.SQL_COMMIT) + // Capture the commit error BEFORE calling SQLSetConnectAttr: the subsequent + // ODBC call overwrites the connection's diagnostic record, so GetLastError + // would return the wrong error if called after the autocommit restore. + var commitErr error + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + commitErr = db2cli.GetLastError(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(tx.conn.hdbc)) + } + // Always restore AUTOCOMMIT regardless of SQLEndTran outcome (MI-10): + // if SQLEndTran fails the connection is in an unknown state but must still + // be usable for future queries — leaving AUTOCOMMIT off would corrupt it. + restoreRet := db2cli.SQLSetConnectAttr(tx.conn.hdbc, db2cli.SQL_ATTR_AUTOCOMMIT, db2cli.SQL_AUTOCOMMIT_ON, 0) + if commitErr != nil { + return fmt.Errorf("committing transaction: %w", commitErr) + } + if restoreRet != db2cli.SQL_SUCCESS && restoreRet != db2cli.SQL_SUCCESS_WITH_INFO { + return fmt.Errorf("re-enabling autocommit after commit: %w", + db2cli.GetLastError(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(tx.conn.hdbc))) + } + return nil +} + +func (tx *db2Tx) Rollback() error { + ret := db2cli.SQLEndTran(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(tx.conn.hdbc), db2cli.SQL_ROLLBACK) + // Capture rollback error before autocommit restore (see Commit for rationale). + var rollbackErr error + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + rollbackErr = db2cli.GetLastError(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(tx.conn.hdbc)) + } + // Always restore AUTOCOMMIT regardless of SQLEndTran outcome (MI-10). + restoreRet := db2cli.SQLSetConnectAttr(tx.conn.hdbc, db2cli.SQL_ATTR_AUTOCOMMIT, db2cli.SQL_AUTOCOMMIT_ON, 0) + if rollbackErr != nil { + return fmt.Errorf("rolling back transaction: %w", rollbackErr) + } + if restoreRet != db2cli.SQL_SUCCESS && restoreRet != db2cli.SQL_SUCCESS_WITH_INFO { + return fmt.Errorf("re-enabling autocommit after rollback: %w", + db2cli.GetLastError(db2cli.SQL_HANDLE_DBC, db2cli.SQLHANDLE(tx.conn.hdbc))) + } + return nil +} + +// db2Stmt implements driver.Stmt. +type db2Stmt struct { + hstmt db2cli.SQLHSTMT + conn *db2Conn +} + +func (s *db2Stmt) Close() error { + if s.hstmt != 0 { + db2cli.SQLFreeHandle(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(s.hstmt)) + s.hstmt = 0 + } + return nil +} + +func (*db2Stmt) NumInput() int { + return -1 +} + +// driverValueToString converts a driver.Value to its SQL string representation. +// +// Special-casing float64 is necessary because JSON unmarshal produces float64 +// for all JSON numbers, so primary-key values restored from the incremental +// snapshot checkpoint (LastEmittedPK) arrive as float64. fmt.Sprintf("%v", +// float64(1234567890)) produces "1.23456789e+09" which is not a valid integer +// literal for a SQL WHERE clause. Instead: if the float64 value is a whole +// number and within the int64 range, format it as a plain decimal integer. +func driverValueToString(v driver.Value) string { + switch val := v.(type) { + case json.Number: + return val.String() + case float64: + if val == math.Trunc(val) && val >= math.MinInt64 && val <= math.MaxInt64 { + return strconv.FormatInt(int64(val), 10) + } + return strconv.FormatFloat(val, 'f', -1, 64) + case bool: + if val { + return "1" + } + return "0" + case []byte: + // DECIMAL/NUMERIC and CHAR/VARCHAR columns arrive from the DB2 ODBC driver as + // []byte. Return the raw string so keyset-pagination cursors round-trip + // correctly — fmt.Sprintf("%v", []byte{49,50}) would produce "[49 50]", not "12". + return string(val) + case string: + return val + default: + return fmt.Sprintf("%v", v) + } +} + +// bindParams binds each driver.Value to the prepared statement as SQL_C_CHAR. +// Returns (bufs, inds): both slices must stay alive via runtime.KeepAlive until +// after SQLExecute returns. bufs holds the null-terminated data buffers; +// inds holds the length indicators. The Go GC does not track C-side pointer +// references, so local variables whose addresses are passed to SQLBindParameter +// must live in heap-allocated slices that are explicitly retained. +func (s *db2Stmt) bindParams(args []driver.Value) (bufs [][]byte, inds []db2cli.SQLLEN, err error) { + bufs = make([][]byte, len(args)) + inds = make([]db2cli.SQLLEN, len(args)) + + for i, arg := range args { + pos := db2cli.SQLUSMALLINT(i + 1) + + if arg == nil { + inds[i] = db2cli.SQLLEN(db2cli.SQL_NULL_DATA) + ret := db2cli.SQLBindParameter( + s.hstmt, pos, + db2cli.SQL_PARAM_INPUT, db2cli.SQL_C_CHAR, db2cli.SQL_VARCHAR, + 0, 0, nil, 0, &inds[i], + ) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + return nil, nil, fmt.Errorf("binding NULL parameter %d: %w", i+1, + db2cli.GetLastError(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(s.hstmt))) + } + continue + } + + strVal := driverValueToString(arg) + // Null-terminate so DB2 CLI can treat it as a C string. + // Pre-allocate len+1 to avoid the two-allocation pattern of + // append([]byte(strVal), 0) which copies then grows. + buf := make([]byte, len(strVal)+1) + copy(buf, strVal) + bufs[i] = buf + + inds[i] = db2cli.SQLLEN(len(strVal)) + ret := db2cli.SQLBindParameter( + s.hstmt, pos, + db2cli.SQL_PARAM_INPUT, db2cli.SQL_C_CHAR, db2cli.SQL_VARCHAR, + db2cli.SQLULEN(len(strVal)), 0, + db2cli.SQLPOINTER(unsafe.Pointer(&buf[0])), + db2cli.SQLLEN(len(buf)), + &inds[i], + ) + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + return nil, nil, fmt.Errorf("binding parameter %d: %w", i+1, + db2cli.GetLastError(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(s.hstmt))) + } + } + + return bufs, inds, nil +} + +func (s *db2Stmt) Exec(args []driver.Value) (driver.Result, error) { + bufs, inds, err := s.bindParams(args) + if err != nil { + return nil, fmt.Errorf("binding parameters: %w", err) + } + + ret := db2cli.SQLExecute(s.hstmt) + runtime.KeepAlive(bufs) // keep data buffers alive until SQLExecute returns + runtime.KeepAlive(inds) // keep indicator lengths alive until SQLExecute returns + + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + err := db2cli.GetLastError(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(s.hstmt)) + return nil, fmt.Errorf("executing statement: %w", err) + } + + var rowCount db2cli.SQLLEN + if r := db2cli.SQLRowCount(s.hstmt, &rowCount); r != db2cli.SQL_SUCCESS { + rowCount = 0 + } + + return &db2Result{rowsAffected: int64(rowCount)}, nil +} + +func (s *db2Stmt) Query(args []driver.Value) (driver.Rows, error) { + bufs, inds, err := s.bindParams(args) + if err != nil { + return nil, fmt.Errorf("binding parameters: %w", err) + } + + ret := db2cli.SQLExecute(s.hstmt) + runtime.KeepAlive(bufs) // keep data buffers alive until SQLExecute returns + runtime.KeepAlive(inds) // keep indicator lengths alive until SQLExecute returns + + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + err := db2cli.GetLastError(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(s.hstmt)) + return nil, fmt.Errorf("executing query: %w", err) + } + + var colCount db2cli.SQLSMALLINT + if r := db2cli.SQLNumResultCols(s.hstmt, &colCount); r != db2cli.SQL_SUCCESS { + return nil, errors.New("getting column count") + } + + return &db2Rows{stmt: s, colCount: int(colCount)}, nil +} + +// db2Result implements driver.Result. +type db2Result struct { + rowsAffected int64 +} + +func (*db2Result) LastInsertId() (int64, error) { + return 0, errors.New("LastInsertId not supported") +} + +func (r *db2Result) RowsAffected() (int64, error) { + return r.rowsAffected, nil +} + +// db2Rows implements driver.Rows. +type db2Rows struct { + stmt *db2Stmt + colCount int + cols []string // cached column names, populated on first Columns() call + colDataTypes []db2cli.SQLSMALLINT // SQL type per column, populated by Columns() + colBuf [32 * 1024]byte // reused across column reads; one allocation per rows-object +} + +func (r *db2Rows) Columns() []string { + if r.cols != nil { + return r.cols + } + cols := make([]string, r.colCount) + colDataTypes := make([]db2cli.SQLSMALLINT, r.colCount) + for i := 0; i < r.colCount; i++ { + colName := make([]byte, 256) + var nameLen, dataType, decimalDigits, nullable db2cli.SQLSMALLINT + var colSize db2cli.SQLULEN + + ret := db2cli.SQLDescribeCol( + r.stmt.hstmt, + db2cli.SQLUSMALLINT(i+1), + (*db2cli.SQLCHAR)(&colName[0]), + 256, + &nameLen, + &dataType, + &colSize, + &decimalDigits, + &nullable, + ) + if ret == db2cli.SQL_SUCCESS || ret == db2cli.SQL_SUCCESS_WITH_INFO { + // Clamp nameLen to [0, bufsize]: per ODBC spec nameLen is the required + // length before truncation; non-conformant drivers may return negative. + n := max(0, min(int(nameLen), len(colName))) + cols[i] = string(colName[:n]) + } else { + cols[i] = fmt.Sprintf("col%d", i+1) + } + colDataTypes[i] = dataType + } + r.cols = cols + r.colDataTypes = colDataTypes + return r.cols +} + +func (r *db2Rows) Close() error { + ret := db2cli.SQLCloseCursor(r.stmt.hstmt) + var closeErr error + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + // SQLSTATE 24000 = "invalid cursor state": cursor was never opened or is + // already closed. This is normal after a zero-row result set; treat as no-op. + diags := db2cli.GetDiagnostics(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(r.stmt.hstmt)) + var invalidCursor bool + for _, d := range diags { + if d.SQLState == db2cli.SQLSTATE_INVALID_CURSOR_STATE { + invalidCursor = true + break + } + } + if !invalidCursor { + closeErr = errors.New("closing cursor") + } + } + // Always free the statement handle so SQLHSTMT resources are returned to DB2 + // immediately rather than accumulating until connection close. + _ = r.stmt.Close() + return closeErr +} + +func (r *db2Rows) Next(dest []driver.Value) error { + ret := db2cli.SQLFetch(r.stmt.hstmt) + if ret == db2cli.SQL_NO_DATA { + return io.EOF + } + if ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO { + err := db2cli.GetLastError(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(r.stmt.hstmt)) + return fmt.Errorf("fetching row: %w", err) + } + + // Ensure colDataTypes is populated (Columns() may not have been called yet). + if r.colDataTypes == nil { + r.Columns() + } + + for i := 0; i < r.colCount; i++ { + cType := cTypeForSQLType(r.colDataTypes[i]) + val, isNull, err := r.readColumnValue(db2cli.SQLUSMALLINT(i+1), cType) + if err != nil { + return fmt.Errorf("reading column %d: %w", i+1, err) + } + if isNull { + dest[i] = nil + } else { + dest[i] = val + } + } + + return nil +} + +// readColumnValue drains one column by calling SQLGetData in a loop. +// cType controls the ODBC transfer type: +// - SQL_C_CHAR — text columns: returns string +// - SQL_C_BINARY — BLOB/BINARY columns: returns []byte (preserves zero bytes) +// - SQL_C_WCHAR — DBCS/VARGRAPHIC columns: returns string decoded from UTF-16LE +// +// When a value exceeds the initial buffer DB2 returns SQL_SUCCESS_WITH_INFO and +// sets indicator > bufSize; subsequent calls retrieve the remainder. This avoids +// both silent truncation and buf[:indicator] out-of-bounds panics for large columns. +// +// maxColumnValueBytes caps LOB accumulation to prevent OOM when a BLOB/CLOB column +// contains an unusually large value. DB2 BLOBs can theoretically reach 2 GB. +const maxColumnValueBytes = 64 * 1024 * 1024 // 64 MiB + +func (r *db2Rows) readColumnValue(colIdx db2cli.SQLUSMALLINT, cType db2cli.SQLSMALLINT) (driver.Value, bool, error) { + buf := r.colBuf[:] // reuse the per-rows buffer; avoids 32 KB heap alloc per column per row + var acc []byte // accumulator for multi-chunk reads + + for { + var indicator db2cli.SQLLEN + ret := db2cli.SQLGetData( + r.stmt.hstmt, colIdx, + cType, + db2cli.SQLPOINTER(unsafe.Pointer(&buf[0])), + db2cli.SQLLEN(len(buf)), + &indicator, + ) + // Keep buf alive across each FFI call: the GC must not relocate the backing + // array while DB2 CLI holds the unsafe.Pointer reference. + runtime.KeepAlive(buf) + + switch { + case ret == db2cli.SQL_NO_DATA: + // All data was drained in a previous iteration. + return valueFromAccumulator(acc, cType), false, nil + case ret != db2cli.SQL_SUCCESS && ret != db2cli.SQL_SUCCESS_WITH_INFO: + return nil, false, db2cli.GetLastError(db2cli.SQL_HANDLE_STMT, db2cli.SQLHANDLE(r.stmt.hstmt)) + case indicator == db2cli.SQL_NULL_DATA: + return nil, true, nil + } + + // Determine how many bytes DB2 wrote into buf. + // If indicator >= len(buf) the value was truncated; DB2 wrote exactly len(buf) bytes + // (possibly including a null-terminator at buf[len(buf)-1]). + // If indicator < 0 (SQL_NO_TOTAL) consume until the null-terminator. + var written int + if indicator >= 0 && int(indicator) < len(buf) { + written = int(indicator) + } else if cType == db2cli.SQL_C_WCHAR { + // UTF-16LE: each codepoint is 2 bytes; stripping single 0x00 bytes would + // corrupt the last character. Strip only aligned 2-byte null pairs instead. + written = len(buf) + for written >= 2 && buf[written-1] == 0 && buf[written-2] == 0 { + written -= 2 + } + } else if cType == db2cli.SQL_C_BINARY { + // Binary data has no null-terminator convention; all bytes in buf are + // significant (e.g. BLOB payloads legitimately end with 0x00). + written = len(buf) + } else { + // Character types: scan back past the null-terminator DB2 appends. + written = len(buf) + for written > 0 && buf[written-1] == 0 { + written-- + } + } + if len(acc)+written > maxColumnValueBytes { + return nil, false, fmt.Errorf("column value exceeds maximum size limit of %d bytes (LOB truncation safeguard)", maxColumnValueBytes) + } + acc = append(acc, buf[:written]...) + + if ret == db2cli.SQL_SUCCESS { + // Full value retrieved in one shot. + return valueFromAccumulator(acc, cType), false, nil + } + // SQL_SUCCESS_WITH_INFO → more data pending; loop. + } +} + +// valueFromAccumulator converts accumulated raw bytes to the appropriate driver.Value. +func valueFromAccumulator(acc []byte, cType db2cli.SQLSMALLINT) driver.Value { + switch cType { + case db2cli.SQL_C_BINARY: + // Return a defensive copy so the caller cannot mutate colBuf via the slice. + out := make([]byte, len(acc)) + copy(out, acc) + return out + case db2cli.SQL_C_WCHAR: + return decodeUTF16LE(acc) + default: + return string(acc) + } +} + +// decodeUTF16LE converts a UTF-16LE byte slice to a UTF-8 Go string. +// DB2 returns GRAPHIC/VARGRAPHIC/DBCLOB data as UTF-16LE when fetched with SQL_C_WCHAR. +// Uses golang.org/x/text/encoding/unicode (already a project dependency) so we +// don't need to roll our own byte-to-uint16 conversion. +func decodeUTF16LE(b []byte) string { + if len(b) == 0 { + return "" + } + out, err := utf16LE.NewDecoder().Bytes(b) + if err != nil { + // Fallback: return raw bytes as a best-effort string rather than silently dropping data. + return string(b) + } + return string(out) +} diff --git a/internal/impl/db2/driver_test.go b/internal/impl/db2/driver_test.go new file mode 100644 index 0000000000..eead0d34dc --- /dev/null +++ b/internal/impl/db2/driver_test.go @@ -0,0 +1,30 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2 + +import ( + "database/sql/driver" + "testing" +) + +// Compile-time assertions: db2Conn must implement both context-aware interfaces +// so that database/sql routes QueryContext and ExecContext through our +// SQLCancel-based cancel path instead of falling back to blocking Prepare+Exec. +var ( + _ driver.QueryerContext = (*db2Conn)(nil) + _ driver.ExecerContext = (*db2Conn)(nil) // fails to compile until ExecContext is added +) + +// TestDB2ConnImplementsContextInterfaces is a placeholder that documents the +// compile-time assertions above; the real test is whether the package builds. +func TestDB2ConnImplementsContextInterfaces(t *testing.T) { + t.Parallel() + // Both assertions are verified at compile time via the var block above. + // If db2Conn is missing QueryContext or ExecContext the build fails here. +} diff --git a/internal/impl/db2/fakedb_test.go b/internal/impl/db2/fakedb_test.go new file mode 100644 index 0000000000..fd0fea2a45 --- /dev/null +++ b/internal/impl/db2/fakedb_test.go @@ -0,0 +1,131 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2 + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "sync" + "sync/atomic" + "testing" +) + +// Fake SQL driver for unit tests in the db2 package. +// Registered once as "fake-db2-input" to avoid conflicts with the real "db2-cli" driver. + +var ( + inputFakeDrvOnce sync.Once + inputFakeConns sync.Map + inputFakeConnSeq atomic.Int64 +) + +type inputFakeHandlers struct { + query func(query string, args []driver.Value) (columns []string, rows [][]driver.Value, err error) + exec func(query string, args []driver.Value) error +} + +func openInputFakeDB(t *testing.T, h *inputFakeHandlers) *sql.DB { + t.Helper() + inputFakeDrvOnce.Do(func() { sql.Register("fake-db2-input", &inputFakeDriver{}) }) + dsn := fmt.Sprintf("input-fake-%d", inputFakeConnSeq.Add(1)) + inputFakeConns.Store(dsn, h) + db, err := sql.Open("fake-db2-input", dsn) + if err != nil { + t.Fatalf("openInputFakeDB: %v", err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + t.Cleanup(func() { + _ = db.Close() + inputFakeConns.Delete(dsn) + }) + return db +} + +type inputFakeDriver struct{} + +func (*inputFakeDriver) Open(dsn string) (driver.Conn, error) { + v, ok := inputFakeConns.Load(dsn) + if !ok { + return nil, fmt.Errorf("fake-db2-input: no handlers for DSN %q", dsn) + } + return &inputFakeConn{h: v.(*inputFakeHandlers)}, nil +} + +type inputFakeConn struct{ h *inputFakeHandlers } + +func (c *inputFakeConn) Prepare(query string) (driver.Stmt, error) { + return &inputFakeStmt{c: c, q: query}, nil +} +func (*inputFakeConn) Close() error { return nil } +func (*inputFakeConn) Begin() (driver.Tx, error) { return &inputFakeTx{}, nil } +func (*inputFakeConn) BeginTx(_ context.Context, _ driver.TxOptions) (driver.Tx, error) { + return &inputFakeTx{}, nil +} + +type inputFakeTx struct{} + +func (*inputFakeTx) Commit() error { return nil } +func (*inputFakeTx) Rollback() error { return nil } + +type inputFakeStmt struct { + c *inputFakeConn + q string +} + +func (*inputFakeStmt) Close() error { return nil } +func (*inputFakeStmt) NumInput() int { return -1 } + +func (s *inputFakeStmt) Exec(args []driver.Value) (driver.Result, error) { + if s.c.h.exec != nil { + return &inputFakeResult{}, s.c.h.exec(s.q, args) + } + return &inputFakeResult{}, nil +} + +func (s *inputFakeStmt) Query(args []driver.Value) (driver.Rows, error) { + if s.c.h.query == nil { + return &inputFakeRows{}, nil + } + cols, data, err := s.c.h.query(s.q, args) + if err != nil { + return nil, err + } + return &inputFakeRows{cols: cols, data: data}, nil +} + +type inputFakeRows struct { + cols []string + data [][]driver.Value + pos int +} + +func (r *inputFakeRows) Columns() []string { return r.cols } +func (*inputFakeRows) Close() error { return nil } + +func (r *inputFakeRows) Next(dest []driver.Value) error { + if r.pos >= len(r.data) { + return io.EOF + } + for i, v := range r.data[r.pos] { + if i < len(dest) { + dest[i] = v + } + } + r.pos++ + return nil +} + +type inputFakeResult struct{} + +func (*inputFakeResult) LastInsertId() (int64, error) { return 0, nil } +func (*inputFakeResult) RowsAffected() (int64, error) { return 1, nil } diff --git a/internal/impl/db2/input_db2_cdc.go b/internal/impl/db2/input_db2_cdc.go new file mode 100644 index 0000000000..6b8d7d5d21 --- /dev/null +++ b/internal/impl/db2/input_db2_cdc.go @@ -0,0 +1,2764 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2 + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "hash/fnv" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/Jeffail/checkpoint" + "github.com/Jeffail/shutdown" + + "github.com/redpanda-data/benthos/v4/public/service" + + "github.com/redpanda-data/connect/v4/internal/confx" + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2cli" + "github.com/redpanda-data/connect/v4/internal/impl/db2/replication" + "github.com/redpanda-data/connect/v4/internal/license" +) + +const ( + db2CDCFieldDSN = "dsn" + db2CDCFieldSchema = "schema" + db2CDCFieldTables = "tables" + db2CDCFieldTableIncludeRegex = "table_include_regex" + db2CDCFieldTableExcludeRegex = "table_exclude_regex" + db2CDCFieldCDCSchema = "cdc_schema" + db2CDCFieldSnapshotMode = "snapshot_mode" + db2CDCFieldSnapshotMaxBatchSize = "snapshot_max_batch_size" + db2CDCFieldSnapshotParallelism = "snapshot_parallelism" + db2CDCFieldSnapshotIsolationMode = "snapshot_isolation_mode" + db2CDCFieldCheckpointMode = "checkpoint_mode" + db2CDCFieldCheckpointCache = "checkpoint_cache" + db2CDCFieldCheckpointCacheKey = "checkpoint_cache_key" + db2CDCFieldCheckpointCacheTableName = "checkpoint_cache_table_name" + db2CDCFieldCheckpointLimit = "checkpoint_limit" + db2CDCFieldStreamBackoffInterval = "stream_backoff_interval" + db2CDCFieldPollBatchSize = "poll_batch_size" + db2CDCFieldHeartbeatInterval = "heartbeat_interval" + db2CDCFieldEmitSchemaChanges = "emit_schema_changes" + db2CDCFieldSignalTable = "signal_table" +) + +// snapshotMode is Go's equivalent of an enum: a named string type with an +// exhaustive const block. The YAML config field accepts the string values below. +// Default: snapshotModeInitial ("initial"). +type snapshotMode string + +const ( + snapshotModeInitial snapshotMode = "initial" // snapshot on first start only + snapshotModeAlways snapshotMode = "always" // snapshot on every start + snapshotModeNever snapshotMode = "never" // skip snapshot, stream only + snapshotModeInitialOnly snapshotMode = "initial_only" // snapshot then stop (no streaming) + snapshotModeWhenNeeded snapshotMode = "when_needed" // snapshot if no checkpoint exists + snapshotModeNoData snapshotMode = "no_data" // Debezium alias for "never" +) + +const ( + // signalFetchLimit is the max rows returned per signal-table poll. + // Signals are rare control messages; 100 is generous and bounds memory. + signalFetchLimit = 100 + // schemaChangeFetchLimit caps the schema-change watermark scan to bound + // memory when many tables are added to replication in a single cycle. + schemaChangeFetchLimit = 1000 +) + +// errSignalCleanupRetry is returned by runIncrementalSnapshotWithContext when +// signal DELETE fails after a successful snapshot. It signals the caller to +// preserve pendingIncrementalCtx so the next processSignals tick retries +// deletion via the SignalIDs stored in the checkpoint rather than triggering +// a duplicate full snapshot via the un-deleted execute-snapshot signal row. +var errSignalCleanupRetry = errors.New("signal cleanup needs retry") + +// shouldTakeSnapshot returns true if a full snapshot should be performed given +// the connector's snapshot mode and the current checkpoint CSN. +func shouldTakeSnapshot(mode snapshotMode, checkpoint replication.CSN) bool { + switch mode { + case snapshotModeInitial, snapshotModeWhenNeeded: + // Snapshot only when there is no prior checkpoint (first start or wiped state). + return checkpoint.IsNull() + case snapshotModeAlways, snapshotModeInitialOnly: + return true + default: // snapshotModeNever, snapshotModeNoData + return false + } +} + +func db2CDCConfigSpec() *service.ConfigSpec { + return service.NewConfigSpec(). + Beta(). + Version("4.92.0"). + Categories("Services"). + Summary("Consumes change data capture (CDC) events from IBM DB2 using SQL Replication."). + Description(`Streams change events from IBM DB2 tables using SQL Replication (the ASNCDC schema). + +This connector uses *pure Go* with `+"`github.com/ebitengine/purego`"+` for DB2 CLI bindings — no CGO or C toolchain required at build time. + +== How it works + +DB2 SQL Replication maintains a set of *change tables* (also called CD tables) alongside your source tables. A capture daemon reads the DB2 transaction log and writes INSERT, UPDATE, and DELETE operations into those tables. This input polls those change tables and converts each row into a Redpanda Connect message. + +The connector operates in two phases: + +1. *Snapshot* (when `+"`snapshot_mode: initial`"+` or `+"`always`"+`): reads all existing rows from each monitored table inside a single `+"`REPEATABLE READ`"+` read-only transaction. The CDC position (CSN) is captured *before* the first row is read, so no changes are missed while the snapshot is in progress. +2. *Streaming*: polls each change table for new rows with `+"`IBMSNAP_COMMITSEQ`"+` greater than the last checkpoint. The maximum `+"`SYNCHPOINT`"+` from `+"`ASNCDC.IBMSNAP_REGISTER`"+` is used as an upper bound on every poll to avoid reading uncommitted rows. + +== UPDATE representation + +DB2 LUW SQL Replication encodes UPDATE operations as a DELETE record followed by an INSERT record that share the same `+"`IBMSNAP_COMMITSEQ`"+` value. This connector detects these pairs using LEAD/LAG window functions (ported from the Debezium DB2 connector) and merges them into a single `+"`op: u`"+` (update) event. The `+"`before`"+` field contains the old row data and `+"`after`"+` contains the new row data. + +== Message format (Debezium-compatible) + +The message body matches the https://debezium.io/documentation/reference/stable/connectors/db2.html[Debezium DB2 connector^] envelope format, making this connector a drop-in replacement for existing Debezium consumers. + +`+"```json"+` +{ + "before": null, + "after": { "ID": 42, "NAME": "Alice", "SALARY": 75000 }, + "source": { + "connector": "redpanda.db2", + "name": "redpanda.db2_cdc", + "schema": "DB2ADMIN", + "table": "EMPLOYEES", + "commit_lsn": "CSN:000000000000C350", + "change_lsn": null, + "snapshot": "false", + "ts_ms": 1705315845000 + }, + "op": "c", + "ts_ms": 1705315845000 +} +`+"```"+` + +`+"**`op` codes**"+` match Debezium: `+"`c`"+` = create/insert, `+"`u`"+` = update, `+"`d`"+` = delete, `+"`r`"+` = read (snapshot), `+"`hb`"+` = heartbeat, `+"`schema_change`"+` = schema change. + +`+"**`before` field**"+`: Populated per Debezium semantics. For `+"`op: u`"+` (update) events, `+"`before`"+` contains the old row data and `+"`after`"+` contains the new row data. For `+"`op: d`"+` (delete) events, `+"`before`"+` contains the deleted row and `+"`after`"+` is `+"`null`"+`. For `+"`op: c`"+` (insert) and `+"`op: r`"+` (snapshot read) events, `+"`before`"+` is `+"`null`"+`. + +`+"**`change_lsn`**"+`: DB2 LUW SQL Replication exposes only `+"`IBMSNAP_COMMITSEQ`"+` (the commit LSN). The intra-transaction LSN (`+"`change_lsn`"+` in Debezium) is not available and is always `+"`null`"+`. + +The following metadata keys are set on every message: + +- `+"`db2_schema`"+`: source table schema +- `+"`db2_table`"+`: source table name +- `+"`db2_operation`"+`: human-readable op — `+"`read`"+`, `+"`insert`"+`, `+"`update`"+`, `+"`delete`"+`, `+"`heartbeat`"+`, or `+"`schema_change`"+` +- `+"`db2_op`"+`: Debezium op code — `+"`r`"+`, `+"`c`"+`, `+"`u`"+`, `+"`d`"+`, `+"`hb`"+`, or `+"`schema_change`"+` +- `+"`db2_csn`"+`: commit sequence number string (empty for snapshot events; backward-compat alias for `+"`db2_commit_lsn`"+`) +- `+"`db2_commit_lsn`"+`: same as `+"`db2_csn`"+` (Debezium field name) +- `+"`db2_connector`"+`: always `+"`redpanda.db2`"+` +- `+"`db2_snapshot`"+`: `+"`true`"+` for snapshot rows, `+"`false`"+` for streaming rows +- `+"`db2_timestamp`"+`: `+"`IBMSNAP_LOGMARKER`"+` timestamp from the change table (RFC3339Nano; omitted for snapshot events) + +== Prerequisites + +1. IBM DB2 10.1 or later with SQL Replication installed. +2. The DB2 CLI shared library must be present at runtime: + - Linux: `+"`libdb2.so.1`"+` (from the IBM Data Server Driver package) + - macOS: `+"`libdb2.dylib`"+` + - Windows: `+"`db2cli.dll`"+` + +=== Installing on Debian/Ubuntu + +Download the IBM Data Server Driver Package (dsdriver) from the IBM support site and run: + +`+"```sh"+` +tar xzf ibm_data_server_driver_package_linuxx64.tar.gz +cd dsdriver && bash installDSDriver +export LD_LIBRARY_PATH=/opt/ibm/dsdriver/lib:$LD_LIBRARY_PATH +`+"```"+` + +=== Installing on RHEL/CentOS + +`+"```sh"+` +rpm -ivh ibm-datasrvrmgr-*.rpm +export LD_LIBRARY_PATH=/opt/ibm/dsdriver/lib:$LD_LIBRARY_PATH +`+"```"+` + +=== Kubernetes init container + +`+"```yaml"+` +initContainers: + - name: install-db2-driver + image: ibmcom/db2:11.5.9.0 + command: ["/bin/sh", "-c", "cp /opt/ibm/db2/V11.5/lib64/libdb2.so.1 /shared/lib/"] + volumeMounts: + - name: db2-lib + mountPath: /shared/lib +volumes: + - name: db2-lib + emptyDir: {} +`+"```"+` + +3. SQL Replication must be configured and the capture daemon running: +`+"```sql"+` +-- Start the CDC capture daemon +CALL ASNCDC.ASNCDCSERVICES('start', 'asncdc'); + +-- Register each table you want to capture (repeat for each table) +CALL ASNCDC.ADDTABLE('MYSCHEMA', 'EMPLOYEES'); +CALL ASNCDC.ADDTABLE('MYSCHEMA', 'ORDERS'); +`+"```"+` + +== Minimal configuration + +`+"```yaml"+` +input: + db2_cdc: + dsn: "DATABASE=SAMPLE;HOSTNAME=localhost;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" + schema: "DB2ADMIN" + tables: ["EMPLOYEES", "ORDERS"] + # snapshot_mode defaults to "initial": snapshot on first start, stream continuously thereafter. + # Set to "never" to skip the snapshot and stream from the current log position only. +`+"```"+` + +== Checkpoint persistence + +The connector tracks the highest processed `+"`IBMSNAP_COMMITSEQ`"+` so it can resume after a restart without replaying already-delivered events. By default a `+"`RPCN.CDC_CHECKPOINT`"+` table is created in DB2 (the `+"`RPCN`"+` schema must already exist). Set `+"`checkpoint_cache`"+` to use an external https://www.docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] instead. +`). + Fields( + service.NewStringField(db2CDCFieldDSN). + Description("DB2 connection string in the keyword=value format accepted by SQLDriverConnect. "+ + "Common keywords: `DATABASE` (database alias), `HOSTNAME`, `PORT` (default 50000), "+ + "`PROTOCOL` (`TCPIP`), `UID`, `PWD`. The DB2 CLI shared library must be installed "+ + "separately — this connector does not bundle it."). + Secret(). + Example("DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret"), + + service.NewStringField(db2CDCFieldSchema). + Description("DB2 schema (TABSCHEMA) that owns the monitored tables. "+ + "Maps to SOURCE_OWNER in ASNCDC.IBMSNAP_REGISTER. "+ + "Case-insensitive — automatically normalized to uppercase."). + Example("DB2ADMIN"), + + service.NewStringListField(db2CDCFieldTables). + Description("List of table names (without schema prefix) to capture changes from. "+ + "Each table must already be registered with SQL Replication via ASNCDC.ADDTABLE before the connector starts. "+ + "The connector will fail at startup if any listed table is missing from ASNCDC.IBMSNAP_REGISTER. "+ + "When empty, all CDC-registered tables in the schema are discovered dynamically from ASNCDC.IBMSNAP_REGISTER."). + Example([]string{"EMPLOYEES", "ORDERS"}). + Optional(), + + service.NewStringListField(db2CDCFieldTableIncludeRegex). + Description("Optional list of regular expressions; only tables whose names match at least one pattern are captured. "+ + "Applied after `tables`. When `tables` is empty, all CDC-registered tables in the schema are discovered first and this filter narrows the set. "+ + "Patterns are matched against the bare table name (without schema prefix)."). + Example([]string{"^EMP", "^ORDER"}). + Optional(). + Advanced(), + + service.NewStringListField(db2CDCFieldTableExcludeRegex). + Description("Optional list of regular expressions; tables whose names match any pattern are excluded from capture. "+ + "Applied after `table_include_regex`."). + Example([]string{"_TEST$", "_STAGING$"}). + Optional(). + Advanced(), + + service.NewStringField(db2CDCFieldCDCSchema). + Description("Schema that owns the SQL Replication control tables (IBMSNAP_REGISTER and the generated change tables). "+ + "Defaults to `ASNCDC`, which is the standard installation schema. "+ + "Change this only if SQL Replication was installed under a custom schema."). + Default("ASNCDC"). + Advanced(), + + service.NewStringEnumField(db2CDCFieldSnapshotMode, + string(snapshotModeInitial), + string(snapshotModeAlways), + string(snapshotModeNever), + string(snapshotModeInitialOnly), + string(snapshotModeWhenNeeded), + string(snapshotModeNoData)). + Description("Controls when an initial full-table snapshot is performed:\n\n"+ + "- `initial` (default): snapshot only on first start (when no checkpoint exists). Skipped on restart.\n"+ + "- `always`: snapshot on every start, regardless of existing checkpoint.\n"+ + "- `never`: skip snapshot; start streaming from the current log position.\n"+ + "- `initial_only`: snapshot then stop (no streaming phase).\n"+ + "- `when_needed`: snapshot if and only if no prior checkpoint exists; equivalent to `initial` for fresh deployments but useful when the checkpoint table may have been wiped.\n"+ + "- `no_data`: Debezium-compatible alias for `never`; no rows are read during snapshot."). + Default(string(snapshotModeInitial)), + + service.NewIntField(db2CDCFieldSnapshotMaxBatchSize). + Description("Number of rows fetched per round-trip during the initial snapshot. "+ + "Rows are retrieved using keyset pagination (ordered by primary key), so reducing this value "+ + "lowers memory usage at the cost of more DB2 round-trips. "+ + "Increase for large tables with small rows to improve snapshot throughput."). + Default(1024). + Advanced(), + + service.NewIntField(db2CDCFieldSnapshotParallelism). + Description("Number of tables to snapshot concurrently during the initial snapshot phase. "+ + "Currently fixed at 1 (sequential): all workers share a single read-only transaction "+ + "and the underlying db2cli driver uses a 1-connection pool, so concurrency > 1 would "+ + "serialize anyway and risk statement-handle data races. "+ + "Values greater than 1 are accepted and stored but have no effect until "+ + "per-worker connection support is added in a future release."). + Default(1). + Advanced(), + + service.NewStringEnumField(db2CDCFieldSnapshotIsolationMode, + "repeatable_read", "read_committed"). + Description("SQL transaction isolation level used during the snapshot phase. "+ + "`repeatable_read` (default) uses DB2 Repeatable Read (RR) — no phantom reads, strongest consistency. "+ + "`read_committed` uses DB2 Cursor Stability (CS) — lower lock overhead but snapshot consistency is not guaranteed across tables."). + Default("repeatable_read"). + Advanced(), + + service.NewStringField(db2CDCFieldCheckpointMode). + Description("Checkpoint tracking mode. `auto` (default) detects the DB2 version and "+ + "selects CSN-based checkpointing on DB2 10.1+. Set to `csn` to require CSN "+ + "mode explicitly and fail fast on older DB2 versions."). + Default("auto"). + Advanced(), + + service.NewStringField(db2CDCFieldCheckpointCache). + Description("Name of a https://www.docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] to use for storing the checkpoint CSN. "+ + "If not set, the connector automatically creates and manages a `"+db2CDCFieldCheckpointCacheTableName+"` table in DB2. "+ + "Use an external cache (e.g. Redis) when you want checkpoint state to survive a DB2 rebuild "+ + "or when the connector user lacks DDL privileges."). + Example("my_redis_cache"). + Optional(). + Advanced(), + + service.NewStringField(db2CDCFieldCheckpointCacheKey). + Description("Key used to store and retrieve the checkpoint in the cache or DB2 checkpoint table. "+ + "Change this value if you run multiple DB2 CDC connectors that share the same cache resource "+ + "or the same DB2 instance, to avoid checkpoint collisions."). + Default("db2_cdc_checkpoint"). + Advanced(), + + service.NewStringField(db2CDCFieldCheckpointCacheTableName). + Description("Fully-qualified DB2 table name for the built-in checkpoint store. "+ + "The table is created automatically if it does not exist (SQLSTATE 42710 is silently ignored). "+ + "The connector will attempt to CREATE the schema if it does not already exist; "+ + "if the connector user lacks CREATE SCHEMA privileges, the schema must be created manually beforehand. "+ + "Only used when `"+db2CDCFieldCheckpointCache+"` is not set."). + Default("RPCN.CDC_CHECKPOINT"). + Advanced(), + + service.NewIntField(db2CDCFieldCheckpointLimit). + Description("Maximum number of messages that can be in-flight (sent but not yet acknowledged) at any one time. "+ + "The CSN checkpoint is only advanced once all messages at a given position are acknowledged, "+ + "preserving at-least-once delivery guarantees. "+ + "Increase this value to allow larger output batches and higher throughput; "+ + "decrease it to reduce the number of re-delivered messages after a restart."). + Default(4096). + Advanced(), + + service.NewDurationField(db2CDCFieldStreamBackoffInterval). + Description("How long to wait between change-table polls when no new rows are found. "+ + "Note that the DB2 capture daemon itself introduces latency — reducing this below the capture "+ + "daemon's commit interval has no benefit. "+ + "Increase for low-traffic tables to reduce unnecessary DB2 queries."). + Default("1s"). + Advanced(), + + service.NewIntField(db2CDCFieldPollBatchSize). + Description("Maximum number of change rows fetched per change-table query. "+ + "Each poll issues one query per monitored table. "+ + "Reduce this value if individual change events are large (e.g. wide rows). "+ + "Increase it to improve throughput for high-velocity tables."). + Default(2048). + Advanced(), + + service.NewDurationField(db2CDCFieldHeartbeatInterval). + Description("Interval at which heartbeat messages (`op=hb`) are emitted when no CDC changes are available. "+ + "Heartbeats keep downstream consumers alive on low-traffic tables and advance the checkpoint on idle connectors. "+ + "The default of 30s matches the Debezium DB2 connector recommended setting. "+ + "Set to 0s to disable."). + Default("30s"). + Advanced(), + + service.NewBoolField(db2CDCFieldEmitSchemaChanges). + Description("When true, emit a schema change event (`op=schema_change`) whenever a new table is added to SQL Replication "+ + "(i.e., when `ASNCDC.ADDTABLE` is called while the connector is running). "+ + "The connector polls `ASNCDC.IBMSNAP_REGISTER` for new entries on each backoff interval. "+ + "Defaults to true to match Debezium behaviour; set to false to suppress schema change events."). + Default(true). + Advanced(), + + service.NewStringField(db2CDCFieldSignalTable). + Description("Fully-qualified DB2 table name for the signal channel (e.g. `DB2INST1.CDC_SIGNALS`). "+ + "When set, the connector polls this table for `execute-snapshot` signals. "+ + "An `execute-snapshot` signal triggers an ad-hoc snapshot of the specified tables without restarting the connector. "+ + "Create the table with: "+ + "`CREATE TABLE .
(ID VARCHAR(255) NOT NULL, TYPE VARCHAR(64) NOT NULL, DATA VARCHAR(2048), PRIMARY KEY (ID))`"). + Optional(). + Advanced(), + + service.NewAutoRetryNacksToggleField(), + service.NewBatchPolicyField("batching"), + ) +} + +func init() { + service.MustRegisterBatchInput("db2_cdc", db2CDCConfigSpec(), newDB2CDCInput) +} + +//------------------------------------------------------------------------------ + +// db2CDCInput streams change data capture events from IBM DB2 tables via SQL Replication. +// +// Lifecycle: +// +// Connect() → runCDC (goroutine) → runSnapshot + runStreaming → ReadBatch (polling loop) +// Close() → shutSig.TriggerSoftStop() → wg.Wait() → db.Close() +// +// See db2CDCConfigSpec() for the full list of user-visible config fields and their defaults. +type db2CDCInput struct { + // ── DB connections ──────────────────────────────────────────────────────── + // db is the primary *sql.DB (single-connection pool). All streaming reads + // and snapshot queries go through this connection. Nil after Close() returns. + // Guarded by mu. + db *sql.DB + // auxDB is a dedicated single-connection pool for control-path operations: + // signal polling, schema-change queries, and incremental snapshot queries. + // Keeping it separate from db prevents control-path DDL from starving the + // streaming poll and allows signal processing to run concurrently with acks. + // Nil after Close() returns. Guarded by mu. + auxDB *sql.DB + // streamer is the SQL Replication poller, initialized by runStreaming. + // processSignals reads streamer (under mu) to get the current stream CSN for + // open-window deduplication during incremental snapshots. + streamer *replication.Streamer + + // ── Static configuration (set in newDB2CDCInput, immutable thereafter) ──── + dsn string // DB2 SQLDriverConnect keyword=value connection string + schema string // source table schema (upper-cased, alphanumeric+underscore) + tables []string // explicit table list; empty = auto-discover from IBMSNAP_REGISTER + tableFilter *confx.RegexpFilter // include/exclude regex filter applied after tables list + asnCDCSchema string // SQL Replication control schema, default "ASNCDC" + snapshotMode snapshotMode // when to run the initial full-table snapshot + checkpointMode string // "auto" (detect DB2 version) or "csn" (require 10.1+) + + // ── Checkpoint store ────────────────────────────────────────────────────── + checkpointCacheKey string // map key for the streaming-CSN checkpoint entry + checkpointLimit int // max unacked messages before ReadBatch blocks (backpressure) + // cpCacheName is the name of an external service.Cache resource. When empty, + // the connector creates and manages its own checkpoint table in DB2 (cpCacheTableName). + cpCacheName string + cpCacheTableName string // fully-qualified table, e.g. "RPCN.CDC_CHECKPOINT" + // Pre-computed DML for the checkpoint table. + // + // Same rationale as the sigSelect* fields: each query embeds cpCacheTableName + // (e.g. "RPCN.CDC_CHECKPOINT"), so they cannot be package-level constants. + // The format strings live in constants_sql.go as cpMergeSQLFmt, cpSelectSQLFmt, + // cpDeleteSQLFmt. These are the fully-instantiated versions, computed once in + // newDB2CDCInput so the hot-path saveCheckpointKV / loadCheckpointKV methods + // never call fmt.Sprintf on every ack or poll tick. + cpMergeSQL string // MERGE INTO … WHEN MATCHED UPDATE / WHEN NOT MATCHED INSERT + cpSelectSQL string // SELECT CACHE_VAL FROM … WHERE CACHE_KEY = ? + cpDeleteSQL string // DELETE FROM … WHERE CACHE_KEY = ? + + // ── Streaming behaviour ─────────────────────────────────────────────────── + heartbeatInterval time.Duration // interval between synthetic heartbeat events; 0 = disabled + emitSchemaChanges bool // true = emit schema_change events on table re-registration + // lastSeenSynchCSN is updated on every poll from ASNCDC.IBMSNAP_REGISTER and + // used as the upper-bound watermark for CDC streaming. This prevents reading + // uncommitted rows by bounding queries to CSNs that are fully captured. + // Guarded by mu. + lastSeenSynchCSN replication.CSN + signalTable string // fully-qualified table for execute-snapshot / stop-snapshot signals + + // Pre-computed query strings for signal-table operations. + // + // Why instance fields rather than package-level constants? + // Each query embeds the user-configured signal_table name (e.g. + // "DB2INST1.CDC_SIGNALS"), so they cannot be compile-time constants. + // The corresponding format strings (with %[1]s/%[2]d placeholders) live in + // constants_sql.go as sigSelectStopSQLFmt, sigSelectPauseSQLFmt, etc. + // Here we store the fully-instantiated versions, computed once in + // newDB2CDCInput from the immutable signalTable field, so processSignals + // (invoked on every BackoffInterval tick) never calls fmt.Sprintf. + // + // Signal table schema (auto-created if missing): + // CREATE TABLE ( + // ID VARCHAR(255) NOT NULL, + // TYPE VARCHAR(64) NOT NULL, + // DATA VARCHAR(2048), + // PRIMARY KEY (ID) + // ) + sigSelectStopSQL string // SELECT … WHERE TYPE = 'stop-snapshot' FETCH FIRST N ROWS ONLY + sigSelectPauseSQL string // SELECT … WHERE TYPE = 'pause-snapshot' + sigSelectResumeSQL string // SELECT … WHERE TYPE = 'resume-snapshot' + sigSelectExecSQL string // SELECT … WHERE TYPE = 'execute-snapshot' + sigDeleteSQL string // DELETE all processed signal rows + sigDeleteExecSQL string // DELETE WHERE ID = ? AND TYPE = 'execute-snapshot' + sigAbortPollSQL string // poll for 'stop-snapshot' while a snapshot is running + + // schemaChangeQueryTemplate is pre-computed with a single %s placeholder for + // the hex-encoded last-seen CSN. All other query parts (schema name, FETCH + // FIRST N) are baked in at construction to avoid rebuilding the multi-line + // string on every BackoffInterval tick. + schemaChangeQueryTemplate string + + // ── Replication engine config ───────────────────────────────────────────── + snapshotConfig replication.SnapshotConfig + streamConfig replication.StreamConfig + + // ── Internal queues ─────────────────────────────────────────────────────── + capped *checkpoint.Capped[replication.CSN] // tracks in-flight message watermarks for acking + eventChan chan replication.ChangeEvent // CDC goroutine → ReadBatch delivery channel + errChan chan error // fatal errors from CDC goroutines → ReadBatch + + // ── Resources and observability ─────────────────────────────────────────── + res *service.Resources + lagGauge *service.MetricGauge // db2_cdc_lag_ms gauge: time.Since(event.Timestamp) in ms + version replication.Version // detected DB2 server version (set in Connect) + log *service.Logger + + // ── Synchronization primitives ──────────────────────────────────────────── + // Invariants: + // mu – primary short-held lock for state updates. + // Guards: db, auxDB, closed, streamer, lastSeenSynchCSN, + // pendingIncrementalCtx, cdcCancel. + // shutSig – two-phase shutdown (soft-stop → drain, hard-stop → abort). + // Reset to a fresh signaller at the top of each Connect() call + // so goroutines from a prior Close() don't bleed into the next + // connect cycle. + // cdcCancel – cancels the CDC context on hard-stop (Close timeout path). + // Written under mu; safe to read outside mu after goroutines exit. + // wg – counts goroutines launched in Connect(). Close() calls + // wg.Wait() before closing any DB handle to prevent use-after-close. + // stopSnapshotMu / stopSnapshotCh – abort-snapshot channel and its guard. + // Kept separate from mu to prevent a deadlock where the snapshot + // goroutine holds mu while the signal-poller goroutine tries to + // close stopSnapshotCh; the two goroutines can make independent progress. + shutSig *shutdown.Signaller + cdcCancel context.CancelFunc + wg sync.WaitGroup + mu sync.Mutex + closed bool + + // pendingIncrementalCtx is loaded from the checkpoint store in Connect() when + // a partial incremental snapshot was interrupted by a prior restart. The first + // processSignals tick resumes from this context instead of starting fresh. + // Nil when no incomplete snapshot is pending. Guarded by mu. + pendingIncrementalCtx *replication.IncrementalSnapshotContext + + // stopSnapshotCh is closed by the signal-poller goroutine when a 'stop-snapshot' + // signal arrives. runIncrementalSnapshotWithContext selects on this channel to + // abort the in-progress snapshot. A new channel is allocated immediately after + // closing so the next snapshot starts with a fresh, un-closed channel. + // Guarded by stopSnapshotMu. + stopSnapshotMu sync.Mutex + stopSnapshotCh chan struct{} +} + +func newDB2CDCInput(conf *service.ParsedConfig, mgr *service.Resources) (service.BatchInput, error) { + if err := license.CheckRunningEnterprise(mgr); err != nil { + return nil, err + } + + dsn, err := conf.FieldString(db2CDCFieldDSN) + if err != nil { + return nil, err + } + + schema, err := conf.FieldString(db2CDCFieldSchema) + if err != nil { + return nil, err + } + schema = strings.ToUpper(schema) + if !isValidDB2Identifier(schema) { + return nil, fmt.Errorf("schema %q contains invalid characters: only uppercase letters, digits, and underscores are allowed", schema) + } + + var tables []string + if conf.Contains(db2CDCFieldTables) { + if tables, err = conf.FieldStringList(db2CDCFieldTables); err != nil { + return nil, err + } + } + for i, t := range tables { + tables[i] = strings.ToUpper(t) + if !isValidDB2Identifier(tables[i]) { + return nil, fmt.Errorf("tables[%d] %q contains invalid characters: only uppercase letters, digits, and underscores are allowed", i, t) + } + } + + var includePatterns, excludePatterns []string + if conf.Contains(db2CDCFieldTableIncludeRegex) { + if includePatterns, err = conf.FieldStringList(db2CDCFieldTableIncludeRegex); err != nil { + return nil, err + } + } + if conf.Contains(db2CDCFieldTableExcludeRegex) { + if excludePatterns, err = conf.FieldStringList(db2CDCFieldTableExcludeRegex); err != nil { + return nil, err + } + } + + tableIncludes, err := confx.ParseRegexpPatterns(includePatterns) + if err != nil { + return nil, fmt.Errorf("table_include_regex: %w", err) + } + tableExcludes, err := confx.ParseRegexpPatterns(excludePatterns) + if err != nil { + return nil, fmt.Errorf("table_exclude_regex: %w", err) + } + + tableFilter := &confx.RegexpFilter{Include: tableIncludes, Exclude: tableExcludes} + + asnCDCSchema, err := conf.FieldString(db2CDCFieldCDCSchema) + if err != nil { + return nil, err + } + asnCDCSchema = strings.ToUpper(asnCDCSchema) + if !isValidDB2Identifier(asnCDCSchema) { + return nil, fmt.Errorf("cdc_schema %q contains invalid characters: only uppercase letters, digits, and underscores are allowed", asnCDCSchema) + } + + snapshotModeStr, err := conf.FieldString(db2CDCFieldSnapshotMode) + if err != nil { + return nil, err + } + mode := snapshotMode(snapshotModeStr) + + snapshotMaxBatchSize, err := conf.FieldInt(db2CDCFieldSnapshotMaxBatchSize) + if err != nil { + return nil, err + } + if snapshotMaxBatchSize < 1 { + return nil, errors.New("snapshot_max_batch_size must be at least 1") + } + + snapshotParallelism, err := conf.FieldInt(db2CDCFieldSnapshotParallelism) + if err != nil { + return nil, err + } + + snapshotIsolationModeStr, err := conf.FieldString(db2CDCFieldSnapshotIsolationMode) + if err != nil { + return nil, err + } + var snapshotIsolationLevel sql.IsolationLevel + switch snapshotIsolationModeStr { + case "repeatable_read": + snapshotIsolationLevel = sql.LevelSerializable // DB2 RR = sql Serializable + case "read_committed": + snapshotIsolationLevel = sql.LevelReadCommitted + default: + return nil, fmt.Errorf("snapshot_isolation_mode %q: must be 'repeatable_read' or 'read_committed'", snapshotIsolationModeStr) + } + + checkpointMode, err := conf.FieldString(db2CDCFieldCheckpointMode) + if err != nil { + return nil, err + } + + var cpCacheName string + if conf.Contains(db2CDCFieldCheckpointCache) { + if cpCacheName, err = conf.FieldString(db2CDCFieldCheckpointCache); err != nil { + return nil, err + } + } + + checkpointCacheKey, err := conf.FieldString(db2CDCFieldCheckpointCacheKey) + if err != nil { + return nil, err + } + if len(checkpointCacheKey) == 0 { + return nil, errors.New("checkpoint_cache_key must not be empty") + } + if len(checkpointCacheKey) > 255 { + return nil, fmt.Errorf("checkpoint_cache_key length %d exceeds maximum 255 characters", len(checkpointCacheKey)) + } + + cpCacheTableName, err := conf.FieldString(db2CDCFieldCheckpointCacheTableName) + if err != nil { + return nil, err + } + cpCacheTableName = strings.ToUpper(cpCacheTableName) + if err := validateQualifiedIdentifier(cpCacheTableName); err != nil { + return nil, fmt.Errorf("checkpoint_cache_table_name %q: %w", cpCacheTableName, err) + } + + checkpointLimit, err := conf.FieldInt(db2CDCFieldCheckpointLimit) + if err != nil { + return nil, err + } + + streamBackoffInterval, err := conf.FieldDuration(db2CDCFieldStreamBackoffInterval) + if err != nil { + return nil, err + } + if streamBackoffInterval <= 0 { + return nil, fmt.Errorf("stream_backoff_interval must be positive, got %v", streamBackoffInterval) + } + + pollBatchSize, err := conf.FieldInt(db2CDCFieldPollBatchSize) + if err != nil { + return nil, err + } + if pollBatchSize < 1 { + return nil, errors.New("poll_batch_size must be at least 1") + } + + heartbeatInterval, err := conf.FieldDuration(db2CDCFieldHeartbeatInterval) + if err != nil { + return nil, err + } + + emitSchemaChanges, err := conf.FieldBool(db2CDCFieldEmitSchemaChanges) + if err != nil { + return nil, err + } + + var signalTable string + if conf.Contains(db2CDCFieldSignalTable) { + if signalTable, err = conf.FieldString(db2CDCFieldSignalTable); err != nil { + return nil, err + } + signalTable = strings.ToUpper(signalTable) + if err := validateQualifiedIdentifier(signalTable); err != nil { + return nil, fmt.Errorf("signal_table %q: %w", signalTable, err) + } + } + + tableFilterFn := tableFilter.Matches + + d := &db2CDCInput{ + dsn: dsn, + schema: schema, + tables: tables, + tableFilter: tableFilter, + heartbeatInterval: heartbeatInterval, + emitSchemaChanges: emitSchemaChanges, + lastSeenSynchCSN: replication.NullCSN(), + signalTable: signalTable, + asnCDCSchema: asnCDCSchema, + snapshotMode: mode, + checkpointMode: checkpointMode, + checkpointCacheKey: checkpointCacheKey, + checkpointLimit: checkpointLimit, + cpCacheName: cpCacheName, + cpCacheTableName: cpCacheTableName, + snapshotConfig: replication.SnapshotConfig{ + Schemas: []string{schema}, + Tables: tables, + AsnCDCSchema: asnCDCSchema, + BatchSize: snapshotMaxBatchSize, + TableFilter: tableFilterFn, + Parallelism: snapshotParallelism, + IsolationLevel: snapshotIsolationLevel, + }, + streamConfig: replication.StreamConfig{ + Schemas: []string{schema}, + Tables: tables, + AsnCDCSchema: asnCDCSchema, + BackoffInterval: streamBackoffInterval, + PollBatchSize: pollBatchSize, + StartingCSN: replication.NullCSN(), + TableFilter: tableFilterFn, + }, + capped: checkpoint.NewCapped[replication.CSN](int64(checkpointLimit)), + eventChan: make(chan replication.ChangeEvent, 2*pollBatchSize), // 2× so a full batch never stalls the streamer + errChan: make(chan error, 2), + shutSig: shutdown.NewSignaller(), + res: mgr, + lagGauge: mgr.Metrics().NewGauge("db2_cdc_lag_ms"), + log: mgr.Logger(), + stopSnapshotCh: make(chan struct{}), + } + + // Pre-compute invariant SQL strings at construction time. cpCacheTableName and + // signalTable are immutable after this point, so these strings never change. + // Computing them here eliminates the data race between initCheckpointTable / + // initSignalTable (called from Connect) and ack funcs that read them concurrently. + d.cpMergeSQL = fmt.Sprintf(cpMergeSQLFmt, cpCacheTableName) + d.cpSelectSQL = fmt.Sprintf(cpSelectSQLFmt, cpCacheTableName) + d.cpDeleteSQL = fmt.Sprintf(cpDeleteSQLFmt, cpCacheTableName) + if signalTable != "" { + d.sigSelectStopSQL = fmt.Sprintf(sigSelectStopSQLFmt, signalTable, signalFetchLimit) + d.sigSelectPauseSQL = fmt.Sprintf(sigSelectPauseSQLFmt, signalTable, signalFetchLimit) + d.sigSelectResumeSQL = fmt.Sprintf(sigSelectResumeSQLFmt, signalTable, signalFetchLimit) + d.sigSelectExecSQL = fmt.Sprintf(sigSelectExecSQLFmt, signalTable, signalFetchLimit) + d.sigDeleteSQL = fmt.Sprintf(sigDeleteSQLFmt, signalTable) + d.sigDeleteExecSQL = fmt.Sprintf(sigDeleteExecSQLFmt, signalTable) + d.sigAbortPollSQL = fmt.Sprintf(sigAbortPollSQLFmt, signalTable) + } + // Pre-compute the schema-change poll query template. Only the lastHex watermark + // (hex-only, safe to embed) varies per call; everything else is construction-time constant. + d.schemaChangeQueryTemplate = fmt.Sprintf(schemaChangeQueryFmt, asnCDCSchema, schemaChangeFetchLimit) + + batchInput, err := service.AutoRetryNacksBatchedToggled(conf, d) + if err != nil { + return nil, err + } + + return conf.WrapBatchInputExtractTracingSpanMapping("db2_cdc", batchInput) +} + +func (d *db2CDCInput) Connect(ctx context.Context) error { + // Phase 1: hold d.mu only long enough to check/reset the connection state and + // register the initial non-nil d.db pointer. Releasing the lock here is + // required because loadCheckpoint (called in Phase 2) calls loadCheckpointKV, + // which itself acquires d.mu to read d.db — keeping the lock across that call + // would deadlock the calling goroutine. + d.mu.Lock() + if d.db != nil { + d.mu.Unlock() + return nil + } + + // Reset shutdown state so goroutines launched below start with a fresh + // signaller. Without this, a prior Close() (which triggers shutSig) would + // leave the new goroutine's context already cancelled, causing it to exit + // immediately and making reconnection silently fail. + d.shutSig = shutdown.NewSignaller() + d.closed = false + d.mu.Unlock() + + // Drain any leftover errors from a previous session so ReadBatch does not + // see stale fatal errors on reconnect. errChan has capacity 2, so loop. + for { + select { + case <-d.errChan: + default: + goto drainedErrChan + } + } +drainedErrChan: + + // Drain stale buffered events from the previous session. Without this, + // ReadBatch would deliver pre-reconnect events that the new CDC loop will + // also re-produce from the saved checkpoint, causing duplicates. + for { + select { + case <-d.eventChan: + default: + goto eventChanDrained + } + } +eventChanDrained: + + // Reset the checkpoint tracker so a high pending-count from the old session + // does not immediately block new events from being tracked. + d.mu.Lock() + d.capped = checkpoint.NewCapped[replication.CSN](int64(d.checkpointLimit)) + d.mu.Unlock() + + // Helper: tear down everything that was opened and return the given error. + // Called on any failure path after d.db is set. + cleanup := func(err error) error { + d.mu.Lock() + if d.auxDB != nil { + d.auxDB.Close() + d.auxDB = nil + } + if d.db != nil { + d.db.Close() + d.db = nil + } + d.mu.Unlock() + return err + } + + // "db2-cli" is the Redpanda-internal driver name for our purego DB2 CLI + // adapter (internal/impl/db2/db2cli). We use purego instead of + // github.com/ibmdb/go_ibm_db because go_ibm_db requires CGO and the IBM + // C++ runtime at build time, making cross-compilation and pre-built binary + // distribution impossible. purego loads libdb2.so / libdb2.dylib at + // runtime, keeping the binary pure Go. + db, err := sql.Open("db2-cli", d.dsn) + if err != nil { + return fmt.Errorf("opening DB2 connection: %w", sanitizeConnErr(err)) + } + + // Limit to a single connection — CDC logic assumes stable connection state. + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + if err := db.PingContext(ctx); err != nil { + db.Close() + return fmt.Errorf("pinging DB2: %w", sanitizeConnErr(err)) + } + + // Phase 1b: register d.db under the lock so other goroutines (e.g. ReadBatch, + // Close) see a consistent non-nil pointer. From here on, use the cleanup + // helper on every error path. + d.mu.Lock() + d.db = db + d.mu.Unlock() + + version, err := d.detectVersion(ctx) + if err != nil { + return cleanup(fmt.Errorf("detecting DB2 version: %w", err)) + } + d.mu.Lock() + d.version = version + d.mu.Unlock() + d.log.Infof("Connected to DB2 %s", version) + + actualMode, err := d.determineCheckpointMode() + if err != nil { + return cleanup(err) + } + d.mu.Lock() + d.checkpointMode = actualMode + d.mu.Unlock() + + // Initialize checkpoint persistence. + if d.cpCacheName == "" { + if err := d.initCheckpointTable(ctx); err != nil { + return cleanup(fmt.Errorf("initializing checkpoint table: %w", err)) + } + d.log.Infof("Using DB2 table %q for checkpoint persistence", d.cpCacheTableName) + } else { + d.log.Infof("Using external cache %q for checkpoint persistence", d.cpCacheName) + } + + // Open a dedicated auxiliary connection for out-of-band operations (signal + // polling, schema change detection, incremental snapshots). These run + // concurrently with the main CDC streaming loop, so they need their own + // connection pool to avoid starving d.db's single slot. + if d.signalTable != "" || d.emitSchemaChanges { + auxDB, err := sql.Open("db2-cli", d.dsn) + if err != nil { + return cleanup(fmt.Errorf("opening auxiliary DB2 connection: %w", sanitizeConnErr(err))) + } + auxDB.SetMaxOpenConns(1) + auxDB.SetMaxIdleConns(1) + if err := auxDB.PingContext(ctx); err != nil { + auxDB.Close() + return cleanup(fmt.Errorf("pinging auxiliary DB2 connection: %w", sanitizeConnErr(err))) + } + d.mu.Lock() + d.auxDB = auxDB + d.mu.Unlock() + } + + // Initialize signal table if configured. + if d.signalTable != "" { + if err := d.initSignalTable(ctx); err != nil { + return cleanup(fmt.Errorf("initializing signal table: %w", err)) + } + d.log.Infof("Using DB2 table %q for incremental snapshot signals", d.signalTable) + } + + // Phase 2: load checkpoint / incremental context — these call loadCheckpointKV + // internally, which acquires d.mu itself (safe because we released it above). + startingCSN, err := d.loadCheckpoint(ctx) + if err != nil { + return cleanup(fmt.Errorf("loading checkpoint: %w", err)) + } + + // Always assign startingCSN so runCDC sees a NullCSN when there is no + // checkpoint. The zero-value CSN{} has isNull=false, so leaving + // StartingCSN unset when no checkpoint exists would cause the snapshot + // condition (StartingCSN.IsNull()) to be false and the snapshot phase + // to be silently skipped. + // Hold d.mu for the write: a concurrent reconnect from ReadBatch could + // otherwise race against a runCDC goroutine reading streamConfig fields. + d.mu.Lock() + d.streamConfig.StartingCSN = startingCSN + d.mu.Unlock() + if !startingCSN.IsNull() { + d.log.Infof("Resuming from checkpoint CSN: %s", startingCSN) + // The stream poll query uses IBMSNAP_COMMITSEQ > X'', so + // setting StartingCSN = savedCSN resumes correctly from the next event. + // Calling .Next() here was a double-increment that silently dropped + // events at exactly checkpoint+1. + } else { + d.log.Info("No checkpoint found, starting from the beginning") + } + + loadedIncrCtx, err := d.loadIncrementalContext(ctx) + if err != nil { + d.log.Warnf("Failed to load incremental snapshot context, will restart snapshot on next signal: %v", err) + loadedIncrCtx = nil + } + d.mu.Lock() + d.pendingIncrementalCtx = loadedIncrCtx + d.mu.Unlock() + if loadedIncrCtx != nil { + ct := loadedIncrCtx.CurrentTable() + if ct != nil { + d.log.Infof("Resuming incremental snapshot: tables=%d currentTable=%s.%s", + len(loadedIncrCtx.Tables), safeLogID(ct.Schema), safeLogID(ct.Name)) + } else { + // All tables were already snapshotted before the crash; the context has + // an empty Tables queue. Discard it so we don't attempt a no-op resume. + d.log.Infof("Incremental snapshot context found but all tables already complete; clearing stale checkpoint") + d.mu.Lock() + d.pendingIncrementalCtx = nil + d.mu.Unlock() + if err := d.clearIncrementalContext(ctx); err != nil { + d.log.Warnf("Failed to clear stale incremental context: %v", err) + } + } + } + + // Seed the schema-changes watermark so we do not emit spurious schema_change + // events for tables that were registered before this connector started. + if d.emitSchemaChanges { + if err := d.seedSchemaChangesWatermark(ctx); err != nil { + // Non-fatal: lastSeenSynchCSN remains NullCSN (all-zeros). The first + // checkForSchemaChanges poll will match every registered table, producing + // a burst of spurious schema_change events at startup. Those events are + // noisy but not incorrect — downstream consumers must tolerate them. + d.log.Warnf("Failed to seed schema-changes watermark: %v", err) + } + } + + // Phase 3: derive the goroutine context and launch the CDC loop. + // Derive a long-lived context from the shutdown signaller, not from the Connect + // ctx which callers may cancel once Connect returns. + cdcCtx, cdcCancel := d.shutSig.SoftStopCtx(context.Background()) + d.mu.Lock() + d.cdcCancel = cdcCancel + d.mu.Unlock() + // Add before launching so Close()'s wg.Wait() cannot return between the + // unlock and the goroutine start. + d.wg.Add(1) + go d.runCDC(cdcCtx) + + return nil +} + +func (d *db2CDCInput) determineCheckpointMode() (string, error) { + switch d.checkpointMode { + case "csn": + if !d.version.SupportsCDC() { + return "", fmt.Errorf( + "checkpoint_mode='csn' requires DB2 10.1+, found version %s; upgrade or use checkpoint_mode='auto'", + d.version, + ) + } + d.log.Info("Using CSN checkpoint mode [EXPLICIT]") + return "csn", nil + + case "auto": + if d.version.AtLeast(10, 1) { + d.log.Infof("Auto-detected CSN support (DB2 %s) — using CSN checkpoint mode", d.version) + return "csn", nil + } + return "", fmt.Errorf("DB2 version %s does not support CSN (requires 10.1+); upgrade DB2", d.version) + + default: + return "", fmt.Errorf("invalid checkpoint_mode %q: must be 'csn' or 'auto'", d.checkpointMode) + } +} + +func (d *db2CDCInput) ReadBatch(ctx context.Context) (service.MessageBatch, service.AckFunc, error) { + // Snapshot shutSig under d.mu to avoid a data race with Connect(), which + // writes d.shutSig under d.mu. The channel returned by SoftStopChan() is + // stable for the lifetime of the signaller, so holding the channel reference + // past the lock is safe. + d.mu.Lock() + softStop := d.shutSig.SoftStopChan() + d.mu.Unlock() + + // Block until at least one event is available. + var firstEvent replication.ChangeEvent + select { + case firstEvent = <-d.eventChan: + case err := <-d.errChan: + return nil, nil, err + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-softStop: + return nil, nil, service.ErrEndOfInput + } + + // Snapshot PollBatchSize and capped under lock. Connect() replaces d.capped under + // d.mu on reconnect; reading it without the lock is a data race. + d.mu.Lock() + pollBatchSize := d.streamConfig.PollBatchSize + capped := d.capped + d.mu.Unlock() + + // Drain additional events non-blocking up to pollBatchSize. + rawEvents := make([]replication.ChangeEvent, 0, pollBatchSize) + rawEvents = append(rawEvents, firstEvent) +drainLoop: + for len(rawEvents) < pollBatchSize { + select { + case ev := <-d.eventChan: + rawEvents = append(rawEvents, ev) + default: + break drainLoop + } + } + + // Convert events to messages and track each CSN. + // resolveFns are always called (even on nack) to release Capped capacity slots. + // The on-disk checkpoint is only advanced when ackErr == nil, so the watermark + // never moves past unprocessed messages. AutoRetryNacks redelivers by calling + // the same ackFunc with nil on retry. + batch := make(service.MessageBatch, 0, len(rawEvents)) + resolveFns := make([]func() *replication.CSN, 0, len(rawEvents)) + + // Set lag gauge once per batch using the first (oldest) event, which has the + // maximum lag. Calling Set on every event would overwrite with decreasing lag + // values, masking the worst-case latency that matters for alerting. + for i, event := range rawEvents { + if i == 0 && !event.Timestamp.IsZero() { + d.lagGauge.Set(max(time.Since(event.Timestamp).Milliseconds(), 0)) + } + msg, err := d.eventToMessage(event) + if err != nil { + return nil, nil, err + } + var resolveFn func() *replication.CSN + if !event.CSN.IsNull() { + if resolveFn, err = capped.Track(ctx, event.CSN, 1); err != nil { + return nil, nil, fmt.Errorf("tracking CSN: %w", err) + } + } + batch = append(batch, msg) + resolveFns = append(resolveFns, resolveFn) + } + + ackFunc := func(ctx context.Context, ackErr error) error { + // Always call resolveFns to release Capped slots even on nack, so the + // checkpoint capacity does not leak when auto_replay_nacks is false. + var highest *replication.CSN + for _, fn := range resolveFns { + if fn == nil { + continue + } + if h := fn(); h != nil { + highest = h + } + } + if ackErr != nil { + // Slots released above; do not advance the checkpoint on batch failure. + // With auto_replay_nacks: true the batch will be retried and the same + // ackFunc called again with nil; with auto_replay_nacks: false the + // in-memory watermark advances but the on-disk checkpoint is NOT saved, + // so a restart will re-read from the last committed position. + return nil + } + if highest != nil { + if err := d.saveCheckpoint(ctx, *highest); err != nil { + d.log.Warnf("failed to save checkpoint: %v", err) + } + } + return nil + } + + return batch, ackFunc, nil +} + +func (d *db2CDCInput) Close(ctx context.Context) error { + d.mu.Lock() + if d.closed { + d.mu.Unlock() + return nil + } + d.closed = true + sig := d.shutSig // snapshot under lock before releasing + d.mu.Unlock() + + sig.TriggerSoftStop() + + done := make(chan struct{}) + go func() { + d.wg.Wait() + close(done) + }() + + select { + case <-done: + case <-ctx.Done(): + d.log.Warnf("context cancelled while waiting for CDC goroutines to stop: %v", ctx.Err()) + sig.TriggerHardStop() + d.mu.Lock() + cancel := d.cdcCancel + d.mu.Unlock() + if cancel != nil { + cancel() + } + // Wait for goroutines to drain before closing DB connections. The context + // is already cancelled so goroutines exit promptly; this prevents + // use-after-close on auxDB/db when goroutines are mid-query. + <-done + } + + d.mu.Lock() + defer d.mu.Unlock() + + if d.auxDB != nil { + if err := d.auxDB.Close(); err != nil { + d.log.Errorf("error closing auxiliary DB2 connection: %v", err) + } + d.auxDB = nil + } + + if d.db != nil { + if err := d.db.Close(); err != nil { + d.log.Errorf("error closing DB2 connection: %v", err) + } + d.db = nil + } + + d.streamer = nil + + return nil +} + +// runCDC executes the two-phase CDC loop: initial snapshot (optional) → streaming. +func (d *db2CDCInput) runCDC(ctx context.Context) { + defer d.wg.Done() + + d.mu.Lock() + shutSig := d.shutSig + doSnapshot := shouldTakeSnapshot(d.snapshotMode, d.streamConfig.StartingCSN) + d.mu.Unlock() + + if doSnapshot { + d.log.Info("Starting snapshot phase") + if err := d.runSnapshot(ctx); err != nil { + if ctx.Err() == nil && !shutSig.IsSoftStopSignalled() && !errors.Is(err, service.ErrEndOfInput) { + select { + case d.errChan <- fmt.Errorf("snapshot failed: %w", err): + default: + d.log.Warnf("Snapshot error dropped (channel full): %v", err) + } + } + return + } + d.log.Info("Snapshot complete") + if d.snapshotMode == snapshotModeInitialOnly { + d.log.Info("snapshot_mode=initial_only: stopping after snapshot") + // Signal ReadBatch to return ErrEndOfInput so the pipeline terminates + // cleanly. Without this, ReadBatch blocks forever on d.eventChan because + // runCDC exiting does not wake up the ReadBatch select. + shutSig.TriggerSoftStop() + return + } + } + + d.log.Info("Starting streaming phase") + if err := d.runStreaming(ctx); err != nil { + if ctx.Err() == nil && !shutSig.IsSoftStopSignalled() { + select { + case d.errChan <- fmt.Errorf("streaming failed: %w", err): + default: + d.log.Warnf("Streaming error dropped (channel full): %v", err) + } + } + } +} + +func (d *db2CDCInput) runSnapshot(ctx context.Context) error { + d.mu.Lock() + db := d.db + version := d.version + snapshotConfig := d.snapshotConfig + d.mu.Unlock() + if db == nil { + return nil + } + snapshotter := replication.NewSnapshotter(db, snapshotConfig, version) + + handler := func(event replication.ChangeEvent) error { + select { + case d.eventChan <- event: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + + startingCSN, err := snapshotter.Snapshot(ctx, handler) + if err != nil { + return err + } + + d.mu.Lock() + d.streamConfig.StartingCSN = startingCSN + d.mu.Unlock() + d.log.Infof("Snapshot captured at CSN: %s", startingCSN) + + return nil +} + +func (d *db2CDCInput) runStreaming(ctx context.Context) error { + d.mu.Lock() + db := d.db + // Snapshot streamConfig under the lock: the Go memory model does not guarantee + // visibility of concurrent struct field writes without synchronization, even on + // different fields. ReadBatch reads streamConfig.PollBatchSize under d.mu, so all + // reads of streamConfig fields from this goroutine must also hold d.mu. + streamConfig := d.streamConfig + version := d.version + d.mu.Unlock() + if db == nil { + return nil + } + streamer := replication.NewStreamer(db, streamConfig, version, replication.WithStreamerLogger(d.log)) + + if err := streamer.Initialize(ctx); err != nil { + return fmt.Errorf("initializing streamer: %w", err) + } + + // Make the Streamer accessible to pollSignals so the incremental snapshot engine + // can install deduplication windows. Must be set before any goroutine reads it. + d.mu.Lock() + d.streamer = streamer + d.mu.Unlock() + + // Read BackoffInterval from the already-locked local snapshot (captured at the + // top of runStreaming under d.mu). A second lock acquire would be a race-detector + // hit: both accesses land on the same struct with different goroutines writing + // adjacent fields, which the detector treats as a potential race even when the + // exact field has not been concurrently written. + backoffInterval := streamConfig.BackoffInterval + + // streamCtx is cancelled when streamer.Stream returns (success, error, or + // context cancellation). This guarantees sub-goroutines (heartbeat/signals/ + // schema) exit promptly after a stream failure rather than leaking until Close. + streamCtx, streamCancel := context.WithCancel(ctx) + defer streamCancel() + + // Pre-register all sub-goroutine counts before launching any. This prevents + // a race where Close()'s wg.Wait() completes between individual wg.Add+go pairs. + addCount := 0 + if d.heartbeatInterval > 0 { + addCount++ + } + if d.emitSchemaChanges { + addCount++ + } + if d.signalTable != "" { + addCount++ + } + if addCount > 0 { + d.wg.Add(addCount) + } + + if d.heartbeatInterval > 0 { + go d.runHeartbeat(streamCtx) + } + + if d.emitSchemaChanges { + go d.pollSchemaChanges(streamCtx, backoffInterval) + } + + if d.signalTable != "" { + go d.pollSignals(streamCtx, backoffInterval) + } + + handler := func(event replication.ChangeEvent) error { + select { + case d.eventChan <- event: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + + return streamer.Stream(ctx, handler) +} + +func (d *db2CDCInput) runHeartbeat(ctx context.Context) { + defer d.wg.Done() + ticker := time.NewTicker(d.heartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + hb := replication.ChangeEvent{ + Operation: replication.OpTypeHeartbeat, + Timestamp: time.Now().UTC(), + } + select { + case d.eventChan <- hb: + case <-ctx.Done(): + return + } + case <-ctx.Done(): + return + } + } +} + +func (d *db2CDCInput) pollSchemaChanges(ctx context.Context, interval time.Duration) { + defer d.wg.Done() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if err := d.checkForSchemaChanges(ctx); err != nil { + d.log.Warnf("schema change poll: %v", err) + } + case <-ctx.Done(): + return + } + } +} + +// seedSchemaChangesWatermark seeds lastSeenSynchCSN to the current maximum +// CD_NEW_SYNCHPOINT in IBMSNAP_REGISTER so that the first checkForSchemaChanges +// poll does not emit spurious schema_change events for tables that were already +// registered before this connector started. +func (d *db2CDCInput) seedSchemaChangesWatermark(ctx context.Context) error { + d.mu.Lock() + auxDB := d.auxDB + d.mu.Unlock() + if auxDB == nil { + return nil + } + var maxBytes []byte + err := auxDB.QueryRowContext(ctx, + fmt.Sprintf("SELECT MAX(CD_NEW_SYNCHPOINT) FROM %s.IBMSNAP_REGISTER WHERE SOURCE_OWNER = ?", d.asnCDCSchema), + d.schema, + ).Scan(&maxBytes) + if errors.Is(err, sql.ErrNoRows) || len(maxBytes) == 0 { + return nil + } + if err != nil { + return err + } + csn := replication.NewCSNFromDBValue(maxBytes) + d.mu.Lock() + d.lastSeenSynchCSN = csn + d.mu.Unlock() + return nil +} + +func (d *db2CDCInput) checkForSchemaChanges(ctx context.Context) error { + // Read auxDB, streamer, and lastSeenSynchCSN under a single lock acquisition. + d.mu.Lock() + auxDB := d.auxDB + streamer := d.streamer + lastSynch := d.lastSeenSynchCSN + d.mu.Unlock() + if auxDB == nil { + return nil + } + + // CommitSeqByteLen is 10 for DB2 ≤11.x and 16 for DB2 12.1+. The Streamer + // detects the actual length during Initialize() and stores it on its own + // config copy — read from the Streamer rather than d.streamConfig (which is + // never updated after the Streamer is constructed). + var byteLen int + if streamer != nil { + byteLen = streamer.CommitSeqByteLen() + } + if byteLen <= 0 { + byteLen = 10 + } + lastHex := lastSynch.SQLHex(byteLen) + + // schemaChangeQueryTemplate is pre-computed at construction; only lastHex varies. + // lastHex is from CSN.SQLHex() (hex digits only, enforced by the encoder) so it + // is safe to embed. schemaChangeFetchLimit caps memory use when many tables are + // added at once; the watermark advances per-row so subsequent polls catch overflow. + query := fmt.Sprintf(d.schemaChangeQueryTemplate, lastHex) + + rows, err := auxDB.QueryContext(ctx, query, d.schema) + if err != nil { + return fmt.Errorf("querying schema changes: %w", err) + } + defer rows.Close() + + for rows.Next() { + var srcOwner, srcTable string + var synchBytes []byte + if err := rows.Scan(&srcOwner, &srcTable, &synchBytes); err != nil { + d.log.Warnf("scanning schema change row: %v", err) + continue + } + cleanOwner := strings.TrimSpace(srcOwner) + cleanTable := strings.TrimSpace(srcTable) + if !isValidDB2Identifier(cleanOwner) || !isValidDB2Identifier(cleanTable) { + d.log.Warnf("schema change: skipping row with invalid identifier owner=%q table=%q", safeLogID(cleanOwner), safeLogID(cleanTable)) + continue + } + csn := replication.NewCSNFromDBValue(synchBytes) + event := replication.ChangeEvent{ + Schema: cleanOwner, + Table: cleanTable, + Operation: replication.OpTypeSchemaChange, + CSN: csn, + Timestamp: time.Now().UTC(), + } + // Invalidate the column metadata cache for this source table so that the + // next poll re-derives column indices from the updated change table schema + // (e.g., after ASNCDC.ADDCOLUMN). Safe when the table is not yet tracked + // (InvalidateColumnMetaForSource is a no-op for unknown source keys). + if streamer != nil { + streamer.InvalidateColumnMetaForSource(cleanOwner, cleanTable) + } + // Note: s.changeTables is not updated here. Tables added via ASNCDC.ADDTABLE + // are reported as schema_change events but not polled until the connector + // restarts. A future Streamer.AddTable() API would enable hot-add without restart. + // Advance the watermark only after the event is delivered so a crash/restart + // does not skip the event. + select { + case d.eventChan <- event: + d.mu.Lock() + d.lastSeenSynchCSN = csn + d.mu.Unlock() + case <-ctx.Done(): + return ctx.Err() + } + } + return rows.Err() +} + +// initSignalTable creates the signal table in DB2 if it does not exist. +func (d *db2CDCInput) initSignalTable(ctx context.Context) error { + d.mu.Lock() + auxDB := d.auxDB + d.mu.Unlock() + if auxDB == nil { + return errors.New("not connected") + } + if parts := strings.SplitN(d.signalTable, ".", 2); len(parts) == 2 { + // Double-quote the schema name for defense-in-depth; isValidDB2Identifier + // already guarantees [A-Z0-9_] so no embedded quotes can exist, but quoting + // makes the SQL safe by structure rather than by validation guarantee. + schemaName := replication.QuoteDB2Identifier(parts[0]) + if _, err := auxDB.ExecContext(ctx, "CREATE SCHEMA "+schemaName); err != nil && !isAlreadyExistsError(err) { + return fmt.Errorf("creating schema %q: %w", parts[0], err) + } + } + _, err := auxDB.ExecContext(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + ID VARCHAR(255) NOT NULL, + TYPE VARCHAR(64) NOT NULL, + DATA VARCHAR(2048), + PRIMARY KEY (ID) + )`, d.signalTable)) + if err != nil && !isAlreadyExistsError(err) { + return fmt.Errorf("create signal table %s: %w", d.signalTable, err) + } + return nil +} + +func (d *db2CDCInput) pollSignals(ctx context.Context, interval time.Duration) { + defer d.wg.Done() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if err := d.processSignals(ctx); err != nil { + d.log.Warnf("signal poll: %v", err) + } + case <-ctx.Done(): + return + } + } +} + +// processSignals polls the signal table for execute-snapshot requests and drives +// the incremental snapshot engine. +// +// Signal table ──► parseSnapshotSignalTables ──► dedup ──► filterPendingTables +// │ +// ▼ +// IncrementalSnapshotEngine.RunTable() +// │ +// ┌──────────────────┤ +// │ Per-chunk: │ +// │ checkpoint │ +// └──────────────────┘ +// │ +// clearIncrementalContext (all tables done) +// +// On Connect(), any saved IncrementalSnapshotContext is loaded into +// pendingIncrementalCtx and resumed here before new signals are processed. +func (d *db2CDCInput) processSignals(ctx context.Context) error { + d.mu.Lock() + auxDB := d.auxDB + d.mu.Unlock() + if auxDB == nil { + return nil + } + + // Handle stop-snapshot signals before anything else so they abort any pending resume. + if d.signalTable != "" { + stopRows, stopErr := auxDB.QueryContext(ctx, d.sigSelectStopSQL) + if stopErr != nil { + d.log.Warnf("querying stop-snapshot signals: %v", stopErr) + } else { + var stopIDs []string + for stopRows.Next() { + var id string + if e := stopRows.Scan(&id); e != nil { + d.log.Warnf("scanning stop-snapshot signal row: %v", e) + continue + } + stopIDs = append(stopIDs, id) + } + if e := stopRows.Err(); e != nil { + d.log.Warnf("iterating stop-snapshot signals: %v", e) + } + stopRows.Close() + if len(stopIDs) > 0 { + // Process only the first stop signal; close the channel once. + id := stopIDs[0] + d.log.Infof("stop-snapshot signal id=%q: aborting incremental snapshot", safeLogID(id)) + d.stopSnapshotMu.Lock() + close(d.stopSnapshotCh) + d.stopSnapshotCh = make(chan struct{}) + d.stopSnapshotMu.Unlock() + if err := d.clearIncrementalContext(ctx); err != nil { + d.log.Warnf("stop-snapshot: failed to clear incremental context checkpoint: %v", err) + } + d.mu.Lock() + d.pendingIncrementalCtx = nil + d.mu.Unlock() + for _, delID := range stopIDs { + if _, err := auxDB.ExecContext(ctx, d.sigDeleteSQL, delID); err != nil { + d.log.Warnf("stop-snapshot: deleting signal id=%q: %v", safeLogID(delID), err) + } + } + } + // Skip execute-snapshot processing this cycle — stop signals take precedence. + if len(stopIDs) > 0 { + return nil + } + } + } + + // Handle pause-snapshot signals. + if d.signalTable != "" { + pauseRows, pauseErr := auxDB.QueryContext(ctx, d.sigSelectPauseSQL) + if pauseErr != nil { + d.log.Warnf("querying pause-snapshot signals: %v", pauseErr) + } else { + var pauseIDs []string + for pauseRows.Next() { + var id string + if e := pauseRows.Scan(&id); e != nil { + d.log.Warnf("scanning pause-snapshot signal row: %v", e) + continue + } + pauseIDs = append(pauseIDs, id) + } + if e := pauseRows.Err(); e != nil { + d.log.Warnf("iterating pause-snapshot signals: %v", e) + } + pauseRows.Close() + if len(pauseIDs) > 0 { + // Process only the first pause signal; close the channel once. + id := pauseIDs[0] + d.log.Infof("pause-snapshot signal id=%q: pausing incremental snapshot", safeLogID(id)) + d.stopSnapshotMu.Lock() + close(d.stopSnapshotCh) + d.stopSnapshotCh = make(chan struct{}) + d.stopSnapshotMu.Unlock() + // Clear the in-memory resume pointer so the next processSignals call + // does not auto-resume the snapshot before a resume-snapshot signal + // arrives. The DB checkpoint is intentionally preserved so that + // resume-snapshot can reload it and continue from the same position. + d.mu.Lock() + pendingForPause := d.pendingIncrementalCtx + d.pendingIncrementalCtx = nil + d.mu.Unlock() + // Mark only the active snapshot's execute-snapshot signal rows as paused so + // unrelated pending snapshots (different signal IDs) are not accidentally paused. + if pendingForPause != nil && len(pendingForPause.SignalIDs) > 0 { + pausePlaceholders := make([]string, len(pendingForPause.SignalIDs)) + pauseArgs := make([]any, len(pendingForPause.SignalIDs)) + for i, sigID := range pendingForPause.SignalIDs { + pausePlaceholders[i] = "?" + pauseArgs[i] = sigID + } + scopedPauseSQL := fmt.Sprintf( + "UPDATE %s SET TYPE = 'paused-snapshot' WHERE ID IN (%s) AND TYPE = 'execute-snapshot'", + d.signalTable, strings.Join(pausePlaceholders, ","), + ) + if _, err := auxDB.ExecContext(ctx, scopedPauseSQL, pauseArgs...); err != nil { + d.log.Warnf("pause-snapshot: marking execute-snapshot signals as paused: %v", err) + } + } else { + // No active snapshot context — nothing to pause. A blanket UPDATE here + // would incorrectly pause unrelated execute-snapshot signals that have + // not yet been processed. + d.log.Infof("pause-snapshot signal id=%q: no active snapshot context; nothing to pause", safeLogID(id)) + } + for _, delID := range pauseIDs { + if _, err := auxDB.ExecContext(ctx, d.sigDeleteSQL, delID); err != nil { + d.log.Warnf("pause-snapshot: deleting pause signal id=%q: %v", safeLogID(delID), err) + } + } + // Skip resume/execute processing this tick — pause takes precedence. + // Without this return, a co-existing resume-snapshot row would flip the + // paused signals back to execute-snapshot in the same tick, defeating the pause. + return nil + } + } + } + + // Handle resume-snapshot signals — flip paused-snapshot back to execute-snapshot. + if d.signalTable != "" { + resumeRows, resumeErr := auxDB.QueryContext(ctx, d.sigSelectResumeSQL) + if resumeErr != nil { + d.log.Warnf("querying resume-snapshot signals: %v", resumeErr) + } else { + var resumeIDs []string + for resumeRows.Next() { + var id string + if e := resumeRows.Scan(&id); e != nil { + d.log.Warnf("scanning resume-snapshot signal row: %v", e) + continue + } + resumeIDs = append(resumeIDs, id) + } + if e := resumeRows.Err(); e != nil { + d.log.Warnf("iterating resume-snapshot signals: %v", e) + } + resumeRows.Close() + if len(resumeIDs) > 0 { + // Load the paused context first so we can scope the UPDATE to its + // SignalIDs, avoiding an inadvertent blanket-unpausing of any other + // concurrently-paused snapshots that belong to a different session. + inc, incErr := d.loadIncrementalContext(ctx) + if incErr != nil { + d.log.Warnf("resume-snapshot: loading incremental context: %v", incErr) + } + + if inc != nil && len(inc.SignalIDs) > 0 { + // Scoped UPDATE: only flip the signal rows that belong to this + // checkpoint's execute-snapshot batch. + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(inc.SignalIDs)), ",") + scopedSQL := fmt.Sprintf( + "UPDATE %s SET TYPE = 'execute-snapshot' WHERE ID IN (%s) AND TYPE = 'paused-snapshot'", + d.signalTable, placeholders, + ) + args := make([]any, len(inc.SignalIDs)) + for i, id := range inc.SignalIDs { + args[i] = id + } + if _, err := auxDB.ExecContext(ctx, scopedSQL, args...); err != nil { + d.log.Warnf("resume-snapshot: restoring execute-snapshot signals (scoped): %v", err) + } + } else { + // No SignalIDs in persisted checkpoint (either the checkpoint is + // absent or predates SignalID tracking). Executing an unscoped + // UPDATE would inadvertently un-pause unrelated concurrent snapshots + // in a multi-signal environment. Skip the UPDATE and log so the + // operator knows they may need to re-send the resume-snapshot signal. + d.log.Warnf("resume-snapshot: no SignalIDs in incremental checkpoint; skipping execute-snapshot restore — re-send resume-snapshot signal if snapshot does not resume") + } + + for _, id := range resumeIDs { + d.log.Infof("resume-snapshot signal id=%q: resuming incremental snapshot", safeLogID(id)) + if _, err := auxDB.ExecContext(ctx, d.sigDeleteSQL, id); err != nil { + d.log.Warnf("resume-snapshot: deleting resume signal id=%q: %v", safeLogID(id), err) + } + } + + if inc != nil { + d.mu.Lock() + d.pendingIncrementalCtx = inc + d.mu.Unlock() + d.log.Infof("resume-snapshot: reloaded incremental context (%d tables)", len(inc.Tables)) + } + } + } + } + + // Resume a snapshot that was interrupted by a previous restart before polling + // for new signals — this ensures we pick up where we left off. + d.mu.Lock() + pendingCtx := d.pendingIncrementalCtx + s := d.streamer // read under lock: d.streamer may be nil if Close() ran before we reach here + d.mu.Unlock() + if pendingCtx != nil && s != nil { + // Clear AFTER a successful run only. If runIncrementalSnapshotWithContext + // returns an error (e.g. context cancelled), keep pendingIncrementalCtx + // intact so the next poll can resume from the same position. + if err := d.runIncrementalSnapshotWithContext(ctx, s, pendingCtx); err != nil { + // context.Canceled with parent ctx still alive means a stop/pause signal + // aborted the snapshot intentionally — not a fatal error. + if errors.Is(err, context.Canceled) && ctx.Err() == nil { + d.log.Infof("db2 cdc: incremental snapshot resume aborted by stop/pause signal") + return nil + } + // errSignalCleanupRetry: snapshot done but signal DELETE failed. + // Leave pendingIncrementalCtx intact; next tick retries the delete + // via ictx.SignalIDs without re-running the snapshot. + if errors.Is(err, errSignalCleanupRetry) { + d.log.Infof("db2 cdc: signal cleanup will retry on next poll (transient DELETE failure)") + return nil + } + return fmt.Errorf("resuming incremental snapshot: %w", err) + } + d.mu.Lock() + d.pendingIncrementalCtx = nil + d.mu.Unlock() + } + + // Collect all pending signals into memory first, then close the result set + // before calling runIncrementalSnapshot. This avoids a single-connection + // deadlock: the SELECT would hold the auxDB connection open while the + // snapshot transaction needs that same connection. + type pendingSignal struct { + id, data string + } + var pending []pendingSignal + + rows, err := auxDB.QueryContext(ctx, d.sigSelectExecSQL) + if err != nil { + return fmt.Errorf("querying signals: %w", err) + } + for rows.Next() { + var id string + var data sql.NullString + if err := rows.Scan(&id, &data); err != nil { + d.log.Warnf("scanning signal row: %v", err) + continue + } + pending = append(pending, pendingSignal{id: id, data: data.String}) + } + scanErr := rows.Err() + rows.Close() // release auxDB connection before snapshot + if scanErr != nil { + return scanErr + } + + for _, sig := range pending { + tables := parseSnapshotSignalTables(sig.data, d.schema, func(sigSchema, table string) { + d.log.Warnf("execute-snapshot signal id=%q: schema prefix %q on table %q does not match connector schema %q; snapshotting %s.%s", + safeLogID(sig.id), safeLogID(sigSchema), safeLogID(table), d.schema, d.schema, safeLogID(table)) + }) + tables = d.filterPendingTables(tables) + if len(tables) == 0 { + d.log.Infof("execute-snapshot signal id=%q: all tables already in pending resume context; skipping", safeLogID(sig.id)) + if _, err := auxDB.ExecContext(ctx, d.sigDeleteExecSQL, sig.id); err != nil { + d.log.Warnf("execute-snapshot: deleting skipped signal id=%q: %v", safeLogID(sig.id), err) + } + continue + } + safeTables := make([]string, len(tables)) + for i, t := range tables { + safeTables[i] = safeLogID(t) + } + d.log.Infof("received execute-snapshot signal id=%q tables=%q", safeLogID(sig.id), safeTables) + if err := d.runIncrementalSnapshot(ctx, sig.id, tables); err != nil { + d.log.Warnf("incremental snapshot for signal %q: %v", safeLogID(sig.id), err) + if errors.Is(err, errSignalCleanupRetry) { + // pendingIncrementalCtx is still set from the retry path; break so + // filterPendingTables on subsequent iterations doesn't use stale state. + break + } + } + // Signal deletion is handled inside runIncrementalSnapshotWithContext after + // clearIncrementalContext to ensure atomicity (no duplicate delivery on crash). + } + return nil +} + +func (d *db2CDCInput) runIncrementalSnapshot(ctx context.Context, signalID string, tables []string) error { + // Validate and filter table names from the (untrusted) signal DATA field + // before they reach any SQL interpolation — applies to BOTH code paths below. + validated := make([]string, 0, len(tables)) + for _, t := range tables { + if !isValidDB2Identifier(t) { + d.log.Warnf("Incremental snapshot: ignoring table %q from signal — fails identifier validation (possible injection attempt)", safeLogID(t)) + continue + } + validated = append(validated, t) + } + filtered := make([]string, 0, len(validated)) + for _, t := range validated { + // Enforce the configured tables allowlist so signal-triggered snapshots + // cannot target tables the user did not explicitly configure. + if len(d.tables) > 0 && !slices.Contains(d.tables, t) { + d.log.Warnf("Incremental snapshot: ignoring table %q from signal — not in configured tables list", safeLogID(t)) + continue + } + if d.tableFilter == nil || d.tableFilter.Matches(t) { + filtered = append(filtered, t) + } + } + if len(filtered) == 0 { + d.log.Warnf("Incremental snapshot: all requested tables excluded by table filter or identifier validation; skipping") + return nil + } + + d.mu.Lock() + s := d.streamer + auxDB := d.auxDB + version := d.version + d.mu.Unlock() + + if s == nil { + // Streaming not yet initialised; fall back to non-coordinated snapshot. + // This only happens if a signal arrives before the Streamer is ready. + d.log.Warnf("Incremental snapshot requested before Streamer is ready; using full snapshot (no dedup window)") + if auxDB == nil { + return nil + } + cfg := replication.SnapshotConfig{ + Schemas: []string{d.schema}, + Tables: filtered, + AsnCDCSchema: d.asnCDCSchema, + BatchSize: d.snapshotConfig.BatchSize, + } + snapshotter := replication.NewSnapshotter(auxDB, cfg, version) + handler := func(event replication.ChangeEvent) error { + select { + case d.eventChan <- event: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + _, snapshotErr := snapshotter.Snapshot(ctx, handler) + if snapshotErr == nil && signalID != "" { + if _, delErr := auxDB.ExecContext(ctx, d.sigDeleteSQL, signalID); delErr != nil { + d.log.Warnf("fallback snapshot completed but could not delete signal id=%q: %v", safeLogID(signalID), delErr) + } + } + return snapshotErr + } + + iTables := make([]replication.IncrementalTable, len(filtered)) + for i, t := range filtered { + iTables[i] = replication.IncrementalTable{Schema: d.schema, Name: t, SignalID: signalID} + } + + ictx := &replication.IncrementalSnapshotContext{ + Tables: iTables, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + } + + return d.runIncrementalSnapshotWithContext(ctx, s, ictx) +} + +func (d *db2CDCInput) runIncrementalSnapshotWithContext(ctx context.Context, s *replication.Streamer, ictx *replication.IncrementalSnapshotContext) error { + d.mu.Lock() + auxDB := d.auxDB + d.mu.Unlock() + if auxDB == nil { + // auxDB is nil when the connector has been closed or Connect() has not yet + // been called. A nil here means the snapshot signal is silently dropped. + // Log so operators know rather than wonder why no snapshot rows arrived. + d.log.Warnf("db2 cdc: incremental snapshot requested but auxiliary DB connection is nil (connector closing?); snapshot dropped") + return nil + } + + // Derive a cancellable context so stop-snapshot signals can abort mid-snapshot. + d.stopSnapshotMu.Lock() + stopCh := d.stopSnapshotCh + d.stopSnapshotMu.Unlock() + + snapshotCtx, snapshotCancel := context.WithCancel(ctx) + stopWatchDone := make(chan struct{}) + pollerDone := make(chan struct{}) + // Cancel before waiting: snapshotCancel() must run first so the watcher and + // poller goroutines see snapshotCtx.Done() and exit. Defer is LIFO, so a + // single closure guarantees the correct order. + defer func() { + snapshotCancel() + <-stopWatchDone + <-pollerDone + }() + go func() { + defer close(stopWatchDone) + select { + case <-stopCh: + snapshotCancel() + case <-snapshotCtx.Done(): + } + }() + if d.signalTable != "" { + // Poll for stop/pause signals every 5 s so they can abort a running snapshot. + // processSignals is blocked while the snapshot runs and cannot do this itself. + // + // Note: this goroutine shares auxDB (MaxOpenConns(1)) with the ChunkReader. + // QueryRowContext here blocks until the current chunk SELECT finishes and + // rows.Close() returns the connection. Abort reaction is therefore bounded by + // the duration of one chunk read (~snapshot_max_batch_size rows). For default + // settings (1024 rows) this is typically sub-second; for very wide tables or + // slow storage it may take longer but the abort is guaranteed to fire after + // the chunk completes. + go func() { + defer close(pollerDone) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-snapshotCtx.Done(): + return + case <-ticker.C: + var signalType string + if err := auxDB.QueryRowContext(snapshotCtx, d.sigAbortPollSQL).Scan(&signalType); err != nil { + continue // no rows or transient error — keep polling + } + d.log.Infof("db2 cdc: %q signal received while snapshot is running; aborting snapshot", safeLogID(signalType)) + d.stopSnapshotMu.Lock() + close(d.stopSnapshotCh) + d.stopSnapshotCh = make(chan struct{}) + d.stopSnapshotMu.Unlock() + // Cancel immediately rather than waiting for the stopWatcher goroutine + // to observe the closed channel — eliminates one goroutine scheduling + // round-trip that could allow an extra chunk to start before abort. + snapshotCancel() + return + } + } + }() + } else { + close(pollerDone) // no signal table — poller not needed + } + + engine := replication.NewIncrementalSnapshotEngine( + auxDB, + d.asnCDCSchema, + d.snapshotConfig.BatchSize, + s, + func(ctx context.Context, inc *replication.IncrementalSnapshotContext) error { + return d.saveIncrementalContext(ctx, inc) + }, + d.log, + ) + + // Collect signal IDs before the loop: AdvanceTable removes entries from + // ictx.Tables, so by the time the loop exits ictx.Tables is empty and we + // cannot recover the signal IDs after the fact. + // + // On the cleanup-only path (Running()==false after restart), Tables is already + // empty so we recover IDs from ictx.SignalIDs instead. + signalIDs := make(map[string]struct{}) + for _, tbl := range ictx.Tables { + if tbl.SignalID != "" { + signalIDs[tbl.SignalID] = struct{}{} + } + } + // Populate from ictx.SignalIDs on the cleanup-only path (Tables already empty). + for _, sigID := range ictx.SignalIDs { + signalIDs[sigID] = struct{}{} + } + + // Persist signal IDs into the context checkpoint before the snapshot loop so + // that a cross-session retry can recover them even if the session crashes after + // all tables are snapshotted but before signal DELETE succeeds. + if len(signalIDs) > 0 && len(ictx.SignalIDs) == 0 { + for sigID := range signalIDs { + ictx.SignalIDs = append(ictx.SignalIDs, sigID) + } + if err := d.saveIncrementalContext(ctx, ictx); err != nil { + d.log.Warnf("failed to persist signal IDs in incremental context: %v", err) + } + } + + for ictx.Running() { + if err := snapshotCtx.Err(); err != nil { + return err + } + table := *ictx.CurrentTable() + streamCSN := s.CurrentCSN() + + d.log.Infof("Starting incremental snapshot for %s.%s (from CSN %s)", safeLogID(table.Schema), safeLogID(table.Name), streamCSN) + if err := engine.RunTable(snapshotCtx, ictx, table, streamCSN); err != nil { + return fmt.Errorf("incremental snapshot %s.%s: %w", table.Schema, table.Name, err) + } + d.log.Infof("Completed incremental snapshot for %s.%s", safeLogID(table.Schema), safeLogID(table.Name)) + ictx.AdvanceTable() + } + + // Delete signal rows BEFORE clearing the context checkpoint so that on a + // crash between these two operations the context is still present and the + // snapshot can resume from the last checkpoint rather than restarting from + // scratch and delivering duplicate rows. + // + // If any DELETE fails, return errSignalCleanupRetry so the caller preserves + // pendingIncrementalCtx. The next processSignals tick re-enters this function + // with Running()==false and populates signalIDs from ictx.SignalIDs to retry. + var anyDeleteFailed bool + for sigID := range signalIDs { + if _, err := auxDB.ExecContext(ctx, d.sigDeleteSQL, sigID); err != nil { + d.log.Warnf("failed to delete execute-snapshot signal %q after completion: %v", safeLogID(sigID), err) + anyDeleteFailed = true + } + } + if anyDeleteFailed { + // Preserve ictx so the next processSignals tick resumes from the last + // checkpoint rather than re-running the full snapshot (duplicate delivery). + d.mu.Lock() + d.pendingIncrementalCtx = ictx + d.mu.Unlock() + return errSignalCleanupRetry + } + + if err := d.clearIncrementalContext(ctx); err != nil { + d.log.Warnf("Failed to clear incremental context checkpoint: %v", err) + } + return nil +} + +// parseSnapshotSignalTables parses the DATA field of an execute-snapshot signal. +// Accepts JSON {"data-collections":["SCHEMA.TABLE",...]} or bare table names. +// Schema prefixes are stripped; duplicate table names are removed; the caller +// sets Schema separately in SnapshotConfig. +// warnFn is called (when non-nil) for each entry whose schema prefix does not +// match expectedSchema, so the caller can warn the operator. +func parseSnapshotSignalTables(data, expectedSchema string, warnFn func(signalSchema, table string)) []string { + // Use a pointer so we can distinguish an explicit empty array [] (key present, + // len 0) from an absent data-collections key (pointer nil). When the key is + // absent we fall through to the comma-split path for backward compatibility. + type signalData struct { + DataCollections *[]string `json:"data-collections"` + } + var sd signalData + var raw []string + jsonParsed := json.Unmarshal([]byte(data), &sd) == nil + if jsonParsed && sd.DataCollections != nil && len(*sd.DataCollections) == 0 { + // Explicit empty list means "no tables" — return nil rather than falling + // through to the comma-split path, which would treat the raw JSON string + // as a table name and emit a misleading isValidDB2Identifier warning. + return nil + } + stripSchema := func(t string) string { + t = strings.ToUpper(t) + if parts := strings.SplitN(t, ".", 2); len(parts) == 2 { + if warnFn != nil && parts[0] != expectedSchema { + warnFn(parts[0], parts[1]) + } + return parts[1] + } + return t + } + if jsonParsed && sd.DataCollections != nil && len(*sd.DataCollections) > 0 { + raw = make([]string, 0, len(*sd.DataCollections)) + for _, t := range *sd.DataCollections { + raw = append(raw, stripSchema(t)) + } + } else { + // Fallback: comma-separated table names. + if data == "" { + return nil + } + parts := strings.Split(data, ",") + raw = make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + raw = append(raw, stripSchema(p)) + } + } + // Deduplicate while preserving order. Allocate a fresh slice rather than + // raw[:0] to avoid sharing the backing array with the caller's input. + seen := make(map[string]struct{}, len(raw)) + out := make([]string, 0, len(raw)) + for _, t := range raw { + if _, ok := seen[t]; !ok { + seen[t] = struct{}{} + out = append(out, t) + } + } + return out +} + +// filterPendingTables returns tables with any entries already queued in +// pendingIncrementalCtx removed — those will be handled on resume. +// Allocates a fresh output slice to avoid aliasing the input backing array. +func (d *db2CDCInput) filterPendingTables(tables []string) []string { + d.mu.Lock() + pendingCtx := d.pendingIncrementalCtx + d.mu.Unlock() + if pendingCtx == nil { + return tables + } + pending := make(map[string]struct{}, len(pendingCtx.Tables)) + for _, t := range pendingCtx.Tables { + pending[t.Schema+"."+t.Name] = struct{}{} + } + out := make([]string, 0, len(tables)) + for _, t := range tables { + if _, skip := pending[d.schema+"."+t]; !skip { + out = append(out, t) + } + } + return out +} + +// debeziumOp maps a replication.OpType to the Debezium operation code used in the +// "op" field of the Debezium envelope and in the db2_op metadata key. +// +// Codes: c=insert, u=update, d=delete, r=snapshot read, hb=heartbeat. +// schema_change and any unrecognised type fall through to the string value of op. +func debeziumOp(op replication.OpType) string { + switch op { + case replication.OpTypeInsert: + return "c" + case replication.OpTypeUpdate: + return "u" + case replication.OpTypeDelete: + return "d" + case replication.OpTypeRead: + return "r" + case replication.OpTypeHeartbeat: + return "hb" + default: + return string(op) + } +} + +// idempotencyKey returns a deterministic string key for the given event. +// CDC events use schema.table.CSN.intentseq; snapshot events use PK column values only. +// Using only PK columns prevents non-PK columns (which may contain sensitive data) +// from appearing verbatim in message metadata. +// Consumers can use this key to deduplicate at-least-once delivery. +func idempotencyKey(event replication.ChangeEvent) string { + if !event.CSN.IsNull() { + var sb strings.Builder + // 36 = max CSN string length: "CSN:" + 32 hex digits (DB2 12.1 16-byte CSN). + // DB2 ≤11.x uses "CSN:" + 20 hex = 24; over-estimating by 12 avoids realloc. + sb.Grow(len(event.Schema) + 1 + len(event.Table) + 1 + 36 + 1 + 20) + sb.WriteString(event.Schema) + sb.WriteByte('.') + sb.WriteString(event.Table) + sb.WriteByte('.') + sb.WriteString(event.CSN.String()) + sb.WriteByte('.') + sb.WriteString(strconv.FormatInt(event.IntentSeq, 10)) + return sb.String() + } + // Snapshot event: build a stable key from PK column values only. + // PKColumns is always set by the snapshot and incremental snapshot code paths. + // PKColumns should always be set by snapshot/incremental code paths. An empty + // set indicates a programming error — log once per event so it is visible. + cols := event.PKColumns + if len(cols) == 0 { + allKeys := make([]string, 0, len(event.Data)) + for k := range event.Data { + allKeys = append(allKeys, k) + } + slices.Sort(allKeys) + h := fnv.New64a() + for _, k := range allKeys { + _, _ = fmt.Fprintf(h, "%s=%v\n", k, event.Data[k]) + } + return fmt.Sprintf("%s.%s.snapshot.nopk.%016x", event.Schema, event.Table, h.Sum64()) + } + // Use \x00 as key=value pair separator to avoid collisions when column + // values contain commas or equals signs (e.g. COL_A="x,y" vs COL_A="x"). + var sb strings.Builder + sb.Grow(len(event.Schema) + 1 + len(event.Table) + 9 + len(cols)*20) + sb.WriteString(event.Schema) + sb.WriteByte('.') + sb.WriteString(event.Table) + sb.WriteString(".snapshot.") + for i, k := range cols { + if i > 0 { + sb.WriteByte(0) + } + sb.WriteString(k) + sb.WriteByte('=') + switch v := event.Data[k].(type) { + case int64: + sb.WriteString(strconv.FormatInt(v, 10)) + case string: + sb.WriteString(v) + case json.Number: + sb.WriteString(v.String()) + case float64: + sb.WriteString(strconv.FormatFloat(v, 'f', -1, 64)) + case nil: + sb.WriteString("") + default: + fmt.Fprintf(&sb, "%v", v) + } + } + return sb.String() +} + +// debeziumSnapshotValue returns the Debezium snapshot field value for the +// given event. Debezium uses "true" during a snapshot, "last" for the final +// snapshot event, and "false" (or absent) during streaming. Because this +// connector does not track which snapshot row is the last, all snapshot rows +// emit "true"; streaming rows emit "false". +func debeziumSnapshotValue(op replication.OpType) string { + if op == replication.OpTypeRead { + return "true" + } + return "false" +} + +// eventToMessage converts a ChangeEvent to a Redpanda Connect message whose +// body matches the Debezium DB2 connector envelope format for drop-in +// compatibility with existing Debezium consumers. +// +// Envelope structure: +// +// { +// "before": null | { : , ... }, +// "after": null | { : , ... }, +// "source": { "connector": "redpanda.db2", "name": "redpanda.db2_cdc", "schema": "...", +// "table": "...", "commit_lsn": "CSN:...", "change_lsn": null, +// "snapshot": "true"|"false", "ts_ms": , ... }, +// "op": "c"|"u"|"d"|"r", +// "ts_ms": +// } +// +// The before/after fields follow Debezium semantics: +// - INSERT (op="c"): before=null, after=new row +// - UPDATE (op="u"): before=old row data, after=new row data +// - DELETE (op="d"): before=deleted row data, after=null +// - READ (op="r"): before=null, after=snapshot row +// +// Metadata keys set on the message: +// +// - db2_schema — source table schema (TABSCHEMA) +// - db2_table — source table name +// - db2_operation — human-readable op: "read", "insert", "delete", "update" +// - db2_op — Debezium op code: "r", "c", "d", "u" +// - db2_csn — commit sequence number string (backward-compat alias) +// - db2_commit_lsn — same as db2_csn (Debezium naming) +// - db2_connector — always "redpanda.db2" +// - db2_snapshot — "true" for snapshot rows, "false" for streaming rows +// - db2_idempotency_key — deterministic key for deduplication: "SCHEMA.TABLE.CSN.intentseq" for CDC events, "SCHEMA.TABLE.snapshot." for snapshot rows +// - db2_timestamp — IBMSNAP_LOGMARKER timestamp (RFC3339Nano; omitted if zero) +func (*db2CDCInput) eventToMessage(event replication.ChangeEvent) (*service.Message, error) { + if event.Operation == replication.OpTypeHeartbeat { + tsMs := event.Timestamp.UnixMilli() + envelope := map[string]any{ + "op": "hb", + "ts_ms": tsMs, + } + msg := service.NewMessage(nil) + msg.SetStructuredMut(envelope) + msg.MetaSetMut("db2_operation", "heartbeat") + msg.MetaSetMut("db2_op", "hb") + msg.MetaSetMut("db2_schema", "") + msg.MetaSetMut("db2_table", "") + msg.MetaSetMut("db2_csn", "") + msg.MetaSetMut("db2_commit_lsn", "") + msg.MetaSetMut("db2_connector", "redpanda.db2") + msg.MetaSetMut("db2_snapshot", "false") + msg.MetaSetMut("db2_idempotency_key", fmt.Sprintf("heartbeat.%d", event.Timestamp.UnixNano())) + if !event.Timestamp.IsZero() { + msg.MetaSetMut("db2_timestamp", event.Timestamp.Format(time.RFC3339Nano)) + } + return msg, nil + } + + if event.Operation == replication.OpTypeSchemaChange { + tsMs := event.Timestamp.UnixMilli() + envelope := map[string]any{ + "op": "schema_change", + "source": map[string]any{ + "schema": event.Schema, + "table": event.Table, + "csn": event.CSN.String(), + "ts_ms": tsMs, + }, + "ts_ms": tsMs, + } + msg := service.NewMessage(nil) + msg.SetStructuredMut(envelope) + msg.MetaSetMut("db2_operation", "schema_change") + msg.MetaSetMut("db2_schema", event.Schema) + msg.MetaSetMut("db2_table", event.Table) + msg.MetaSetMut("db2_op", "schema_change") + csnStr := event.CSN.String() + msg.MetaSetMut("db2_csn", csnStr) + msg.MetaSetMut("db2_commit_lsn", csnStr) + msg.MetaSetMut("db2_connector", "redpanda.db2") + msg.MetaSetMut("db2_snapshot", "false") + msg.MetaSetMut("db2_idempotency_key", fmt.Sprintf("%s.%s.schema_change.%s", event.Schema, event.Table, csnStr)) + if !event.Timestamp.IsZero() { + msg.MetaSetMut("db2_timestamp", event.Timestamp.Format(time.RFC3339Nano)) + } + return msg, nil + } + + // Determine source timestamp in epoch-milliseconds. + var tsMs int64 + if !event.Timestamp.IsZero() { + tsMs = event.Timestamp.UnixMilli() + } + + csnStr := event.CSN.String() // "" for snapshot rows (NullCSN); computed once and reused below + + // Build the source block to match Debezium's SourceInfo struct. + // change_lsn is not tracked separately by DB2 LUW SQL Replication + // (it only exposes IBMSNAP_COMMITSEQ = commit_lsn), so it is null. + source := map[string]any{ + "connector": "redpanda.db2", + "name": "redpanda.db2_cdc", + "schema": event.Schema, + "table": event.Table, + "commit_lsn": csnStr, + "change_lsn": nil, // not available from DB2 LUW SQL Replication + "snapshot": debeziumSnapshotValue(event.Operation), + "ts_ms": tsMs, + } + + // Populate before/after fields per Debezium semantics: + // INSERT (c): before=null, after=new row + // UPDATE (u): before=old row data, after=new row data + // DELETE (d): before=deleted row data, after=null + // READ (r): before=null, after=snapshot row + var before any + var after any + switch event.Operation { + case replication.OpTypeDelete: + before = event.Data + after = nil + case replication.OpTypeUpdate: + before = event.BeforeData + after = event.Data + default: + before = nil + after = event.Data + } + + opCode := debeziumOp(event.Operation) + envelope := map[string]any{ + "before": before, + "after": after, + "source": source, + "op": opCode, + "ts_ms": tsMs, + } + + msg := service.NewMessage(nil) + msg.SetStructuredMut(envelope) + + // Metadata — standard Debezium-named keys plus backward-compat aliases. + msg.MetaSetMut("db2_schema", event.Schema) + msg.MetaSetMut("db2_table", event.Table) + msg.MetaSetMut("db2_operation", string(event.Operation)) + msg.MetaSetMut("db2_op", opCode) + msg.MetaSetMut("db2_csn", csnStr) // backward-compat alias + msg.MetaSetMut("db2_commit_lsn", csnStr) // Debezium naming + msg.MetaSetMut("db2_connector", "redpanda.db2") + msg.MetaSetMut("db2_snapshot", debeziumSnapshotValue(event.Operation)) + msg.MetaSetMut("db2_idempotency_key", idempotencyKey(event)) + if !event.Timestamp.IsZero() { + msg.MetaSetMut("db2_timestamp", event.Timestamp.Format(time.RFC3339Nano)) + } + + return msg, nil +} + +// detectVersion queries DB2 for its version string. +func (d *db2CDCInput) detectVersion(ctx context.Context) (replication.Version, error) { + d.mu.Lock() + db := d.db + d.mu.Unlock() + if db == nil { + return replication.Version{}, errors.New("not connected") + } + var versionStr string + err := db.QueryRowContext(ctx, + "SELECT SERVICE_LEVEL FROM SYSIBMADM.ENV_INST_INFO FETCH FIRST 1 ROW ONLY", + ).Scan(&versionStr) + if err != nil { + // Fallback for older DB2 versions. + if err2 := db.QueryRowContext(ctx, "VALUES (PROD_RELEASE)").Scan(&versionStr); err2 != nil { + return replication.Version{}, fmt.Errorf("detecting DB2 version (ENV_INST_INFO: %w; PROD_RELEASE: %v)", err, err2) + } + } + + return replication.ParseVersion(versionStr) +} + +// initCheckpointTable creates the checkpoint table in DB2 if it does not exist. +func (d *db2CDCInput) initCheckpointTable(ctx context.Context) error { + d.mu.Lock() + db := d.db + d.mu.Unlock() + if db == nil { + return errors.New("not connected") + } + // Create the schema for the checkpoint table if it does not already exist. + // Only ignore the "already exists" error (SQLSTATE 42710); propagate + // permission errors, connection errors, etc. so the operator sees the real cause. + if parts := strings.SplitN(d.cpCacheTableName, ".", 2); len(parts) == 2 { + schemaName := replication.QuoteDB2Identifier(parts[0]) + if _, err := db.ExecContext(ctx, "CREATE SCHEMA "+schemaName); err != nil && !isAlreadyExistsError(err) { + return fmt.Errorf("create schema %s for checkpoint table: %w", parts[0], err) + } + } + + createSQL := fmt.Sprintf(` + CREATE TABLE %s ( + CACHE_KEY VARCHAR(255) NOT NULL, + CACHE_VAL VARCHAR(32672) NOT NULL, + PRIMARY KEY (CACHE_KEY) + )`, d.cpCacheTableName) + + _, err := db.ExecContext(ctx, createSQL) + if err != nil && !isAlreadyExistsError(err) { + return fmt.Errorf("create checkpoint table %s: %w", d.cpCacheTableName, err) + } + + return nil +} + +// saveCheckpointKV upserts an arbitrary key→value pair into the checkpoint store. +func (d *db2CDCInput) saveCheckpointKV(ctx context.Context, key, val string) error { + if d.cpCacheName != "" { + var cacheErr error + if err := d.res.AccessCache(ctx, d.cpCacheName, func(c service.Cache) { + cacheErr = c.Set(ctx, key, []byte(val), nil) + }); err != nil { + return fmt.Errorf("accessing checkpoint cache: %w", err) + } + return cacheErr + } + + d.mu.Lock() + db := d.db + d.mu.Unlock() + if db == nil { + // Connector has been closed: the checkpoint cannot be persisted. + // Return an error so the engine propagates the failure rather than + // silently treating the checkpoint as saved (crash would replay all + // already-processed rows on restart). + return errors.New("checkpoint not saved: DB connection is nil (connector closed)") + } + + mergeSQL := d.cpMergeSQL + if mergeSQL == "" { + mergeSQL = fmt.Sprintf(` + MERGE INTO %s AS T + USING (VALUES (?, ?)) AS S(CACHE_KEY, CACHE_VAL) + ON T.CACHE_KEY = S.CACHE_KEY + WHEN MATCHED THEN UPDATE SET T.CACHE_VAL = S.CACHE_VAL + WHEN NOT MATCHED THEN INSERT (CACHE_KEY, CACHE_VAL) VALUES (S.CACHE_KEY, S.CACHE_VAL) + `, d.cpCacheTableName) + } + _, err := db.ExecContext(ctx, mergeSQL, key, val) + if err != nil { + return fmt.Errorf("saving checkpoint KV to DB2 table: %w", err) + } + return nil +} + +// loadCheckpointKV returns the stored value for key, or "" if not found. +func (d *db2CDCInput) loadCheckpointKV(ctx context.Context, key string) (string, error) { + if d.cpCacheName != "" { + var val []byte + var cacheErr error + if err := d.res.AccessCache(ctx, d.cpCacheName, func(c service.Cache) { + val, cacheErr = c.Get(ctx, key) + }); err != nil { + return "", fmt.Errorf("accessing checkpoint cache: %w", err) + } + if errors.Is(cacheErr, service.ErrKeyNotFound) { + return "", nil + } + if cacheErr != nil { + return "", fmt.Errorf("reading checkpoint KV: %w", cacheErr) + } + return string(val), nil + } + + d.mu.Lock() + db := d.db + d.mu.Unlock() + if db == nil { + return "", nil + } + + selectSQL := d.cpSelectSQL + if selectSQL == "" { + selectSQL = fmt.Sprintf("SELECT CACHE_VAL FROM %s WHERE CACHE_KEY = ?", d.cpCacheTableName) + } + var val string + err := db.QueryRowContext(ctx, selectSQL, key).Scan(&val) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("loading checkpoint KV from DB2 table: %w", err) + } + return val, nil +} + +// deleteCheckpointKV removes a key from the checkpoint store. +func (d *db2CDCInput) deleteCheckpointKV(ctx context.Context, key string) error { + if d.cpCacheName != "" { + var cacheErr error + if err := d.res.AccessCache(ctx, d.cpCacheName, func(c service.Cache) { + cacheErr = c.Delete(ctx, key) + }); err != nil { + return fmt.Errorf("accessing checkpoint cache: %w", err) + } + if errors.Is(cacheErr, service.ErrKeyNotFound) { + return nil + } + return cacheErr + } + + d.mu.Lock() + db := d.db + d.mu.Unlock() + if db == nil { + return nil + } + deleteSQL := d.cpDeleteSQL + if deleteSQL == "" { + deleteSQL = fmt.Sprintf("DELETE FROM %s WHERE CACHE_KEY = ?", d.cpCacheTableName) + } + _, err := db.ExecContext(ctx, deleteSQL, key) + return err +} + +// loadCheckpoint reads the last saved CSN from the checkpoint store. +func (d *db2CDCInput) loadCheckpoint(ctx context.Context) (replication.CSN, error) { + val, err := d.loadCheckpointKV(ctx, d.checkpointCacheKey) + if err != nil || val == "" { + return replication.NullCSN(), err + } + return replication.ParseCSN(val) +} + +// saveCheckpoint persists the highest fully-processed CSN to the checkpoint store. +func (d *db2CDCInput) saveCheckpoint(ctx context.Context, csn replication.CSN) error { + return d.saveCheckpointKV(ctx, d.checkpointCacheKey, csn.String()) +} + +// incrementalContextKey returns the checkpoint store key for the incremental +// snapshot context, namespaced under d.checkpointCacheKey so that multiple +// connectors sharing the same external cache (checkpoint_cache) do not +// overwrite each other's in-progress snapshots. +func (d *db2CDCInput) incrementalContextKey() string { + return d.checkpointCacheKey + "_incremental_ctx" +} + +func (d *db2CDCInput) saveIncrementalContext(ctx context.Context, inc *replication.IncrementalSnapshotContext) error { + b, err := json.Marshal(inc) + if err != nil { + return fmt.Errorf("marshaling incremental context: %w", err) + } + return d.saveCheckpointKV(ctx, d.incrementalContextKey(), string(b)) +} + +func (d *db2CDCInput) loadIncrementalContext(ctx context.Context) (*replication.IncrementalSnapshotContext, error) { + val, err := d.loadCheckpointKV(ctx, d.incrementalContextKey()) + if err != nil || val == "" { + return nil, err + } + // Guard against a tampered checkpoint store sending a huge payload that + // would exhaust heap before the JSON decoder can reject it. + const maxIncrementalContextBytes = 1 << 20 // 1 MiB + if len(val) > maxIncrementalContextBytes { + return nil, fmt.Errorf("incremental context checkpoint too large (%d bytes, max %d) — discarding; snapshot will restart from beginning", len(val), maxIncrementalContextBytes) + } + // Use json.Decoder with UseNumber so BIGINT primary keys > 2^53 are preserved + // as json.Number rather than being truncated to float64 (which would corrupt + // the keyset-pagination cursor for large integer PKs on resume). + var inc replication.IncrementalSnapshotContext + dec := json.NewDecoder(strings.NewReader(val)) + dec.UseNumber() + if err := dec.Decode(&inc); err != nil { + return nil, fmt.Errorf("unmarshaling incremental context: %w", err) + } + if inc.LastEmittedPK == nil { + inc.LastEmittedPK = map[string][]any{} + } + if inc.MaxPK == nil { + inc.MaxPK = map[string][]any{} + } + // Validate deserialized table identifiers before they reach SQL interpolation. + // A tampered checkpoint store must not be a SQL injection vector. + for _, tbl := range inc.Tables { + if !isValidDB2Identifier(tbl.Schema) || !isValidDB2Identifier(tbl.Name) { + return nil, fmt.Errorf("checkpoint contains invalid table identifier %q.%q — discarding resume context", tbl.Schema, tbl.Name) + } + } + return &inc, nil +} + +func (d *db2CDCInput) clearIncrementalContext(ctx context.Context) error { + return d.deleteCheckpointKV(ctx, d.incrementalContextKey()) +} + +// isAlreadyExistsError returns true when the error indicates an object already +// exists. The IBM DB2 CLI driver may return two different SQLSTATE codes for +// this condition: +// - "42S01": ODBC standard code for "base table or view already exists" +// (returned by the CLI driver in practice for CREATE TABLE on an existing table) +// - "42710": IBM-specific DB2 SQLSTATE for "duplicate name" +// +// Uses the structured *db2cli.DB2Error type when available so the check is an +// exact string comparison rather than a substring scan, avoiding false positives. +func isAlreadyExistsError(err error) bool { + if err == nil { + return false + } + if db2Err, ok := errors.AsType[*db2cli.DB2Error](err); ok { + // 42S01 = ODBC "base table or view already exists" (CLI driver returns this) + // 42710 = IBM DB2 "duplicate name" + return db2Err.SQLState == "42710" || db2Err.SQLState == "42S01" + } + // Fallback for wrapped or non-DB2Error errors (e.g. wrapped fmt.Errorf). + msg := err.Error() + return strings.Contains(msg, "SQLSTATE=42710") || strings.Contains(msg, "SQLSTATE 42710") || + strings.Contains(msg, "SQLSTATE=42S01") || strings.Contains(msg, "[42S01]") +} + +// isValidDB2Identifier returns true if s is a valid unquoted DB2 identifier: +// starts with a letter or underscore, then contains only uppercase letters, +// digits, or underscores. This matches DB2's rules for ordinary identifiers and +// is sufficient to ensure the string is safe to embed directly in SQL strings +// without quoting or parameter binding. +func isValidDB2Identifier(s string) bool { + if s == "" { + return false + } + r0 := rune(s[0]) + if (r0 < 'A' || r0 > 'Z') && r0 != '_' { + return false + } + for _, r := range s[1:] { + if (r < 'A' || r > 'Z') && (r < '0' || r > '9') && r != '_' { + return false + } + } + return true +} + +// validateQualifiedIdentifier validates a possibly-qualified identifier of the form +// "TABLE" or "SCHEMA.TABLE". Each part must satisfy isValidDB2Identifier. +func validateQualifiedIdentifier(s string) error { + parts := strings.SplitN(s, ".", 2) + for _, part := range parts { + if !isValidDB2Identifier(part) { + return fmt.Errorf("identifier part %q contains invalid characters: only uppercase letters, digits, and underscores are allowed", part) + } + } + return nil +} + +// safeLogID truncates a signal-table ID value to at most maxLen printable ASCII +// characters before it appears in log output. Signal IDs are VARCHAR(255) values +// written by operators, but a maliciously crafted value could inject control +// characters into structured log output. Truncation is capped at 64 chars which +// is well above any legitimate signal ID length. +func safeLogID(id string) string { + const maxLen = 64 + // Strip non-printable ASCII to prevent log injection. + clean := strings.Map(func(r rune) rune { + if r >= 0x20 && r < 0x7F { + return r + } + return -1 + }, id) + if len(clean) > maxLen { + return clean[:maxLen] + "…" + } + return clean +} diff --git a/internal/impl/db2/input_db2_cdc_test.go b/internal/impl/db2/input_db2_cdc_test.go new file mode 100644 index 0000000000..fed076d5bd --- /dev/null +++ b/internal/impl/db2/input_db2_cdc_test.go @@ -0,0 +1,415 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2 + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + + "github.com/redpanda-data/connect/v4/internal/impl/db2/replication" + "github.com/redpanda-data/connect/v4/internal/license" +) + +func TestDB2CDCInputConfigParsing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configYAML string + errContains string + }{ + { + name: "valid minimal config", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES"] +`, + }, + { + name: "valid config with all fields", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES", "ORDERS"] +cdc_schema: "ASNCDC" +snapshot_mode: never +checkpoint_cache_table_name: "MYSCHEMA.MY_CHECKPOINT" +`, + }, + { + name: "schema with special chars rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2'; DROP TABLE FOO; --" +tables: ["EMPLOYEES"] +`, + errContains: "invalid characters", + }, + { + name: "cdc_schema with special chars rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES"] +cdc_schema: "ASN'; DROP TABLE FOO; --" +`, + errContains: "invalid characters", + }, + { + name: "checkpoint_cache_table_name with special chars rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES"] +checkpoint_cache_table_name: "RPCN.CDC'; DROP TABLE FOO" +`, + errContains: "invalid characters", + }, + { + name: "lowercase schema accepted (normalized to uppercase)", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "db2admin" +tables: ["EMPLOYEES"] +`, + }, + { + name: "empty tables rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: [] +`, + errContains: "either tables or table_include_regex must be specified", + }, + { + name: "snapshot_max_batch_size zero rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES"] +snapshot_max_batch_size: 0 +`, + errContains: "snapshot_max_batch_size", + }, + { + name: "poll_batch_size zero rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES"] +poll_batch_size: 0 +`, + errContains: "poll_batch_size", + }, + } + + spec := db2CDCConfigSpec() + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + env := service.NewEnvironment() + conf, err := spec.ParseYAML(tc.configYAML, env) + + var initErr error + if err == nil { + mgr := conf.Resources() + license.InjectTestService(mgr) + _, initErr = newDB2CDCInput(conf, mgr) + } + + if tc.errContains == "" { + require.NoError(t, err, "config parse error") + require.NoError(t, initErr, "init error") + } else { + if err != nil { + require.Contains(t, err.Error(), tc.errContains) + } else { + require.Error(t, initErr) + require.Contains(t, initErr.Error(), tc.errContains) + } + } + }) + } +} + +func TestDB2CDCConfigLinting(t *testing.T) { + t.Parallel() + + linter := service.NewEnvironment().NewComponentConfigLinter() + + tests := []struct { + name string + conf string + lintErr string + }{ + { + name: "valid config", + conf: ` +db2_cdc: + dsn: "DATABASE=SAMPLE;HOSTNAME=localhost;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" + schema: "DB2ADMIN" + tables: ["EMPLOYEES"] +`, + }, + { + name: "unknown field", + conf: ` +db2_cdc: + dsn: "DATABASE=SAMPLE;HOSTNAME=localhost;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" + schema: "DB2ADMIN" + tables: ["EMPLOYEES"] + unknown_field: "foo" +`, + lintErr: "unknown_field", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + lints, err := linter.LintInputYAML([]byte(tc.conf)) + require.NoError(t, err) + if tc.lintErr != "" { + found := false + for _, l := range lints { + if strings.Contains(l.Error(), tc.lintErr) { + found = true + break + } + } + assert.True(t, found, "expected lint containing %q, got: %v", tc.lintErr, lints) + } else { + assert.Empty(t, lints) + } + }) + } +} + +func TestIsValidDB2Identifier(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + valid bool + }{ + {"DB2ADMIN", true}, + {"EMPLOYEES", true}, + {"MY_TABLE_123", true}, + {"ASNCDC", true}, + {"", false}, + {"db2admin", false}, // lowercase not allowed (input must be pre-uppercased) + {"MY-TABLE", false}, // hyphen not allowed + {"MY TABLE", false}, // space not allowed + {"'; DROP TABLE T; --", false}, + {"SCHEMA.TABLE", false}, // dot not allowed (use validateQualifiedIdentifier for qualified names) + } + + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + assert.Equal(t, tc.valid, isValidDB2Identifier(tc.input)) + }) + } +} + +func TestValidateQualifiedIdentifier(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + wantErr bool + }{ + {"EMPLOYEES", false}, + {"RPCN.CDC_CHECKPOINT", false}, + {"MY_SCHEMA.MY_TABLE_123", false}, + {"", true}, + {"SCHEMA.'; DROP TABLE T", true}, + {"'; DROP TABLE T", true}, + {"SCHEMA.TABLE.EXTRA", true}, // second part "TABLE.EXTRA" contains a dot → invalid + } + + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + err := validateQualifiedIdentifier(tc.input) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestIsValidDB2IdentifierEdgeCases covers boundary values not exercised by the +// table-driven test above: single-character identifiers, numeric-only names, +// and underscore-only/leading/trailing underscore forms. +func TestIsValidDB2IdentifierEdgeCases(t *testing.T) { + t.Parallel() + + // Single uppercase letter is valid. + assert.True(t, isValidDB2Identifier("A")) + // Single digit is invalid — DB2 unquoted identifiers cannot start with a digit. + assert.False(t, isValidDB2Identifier("1"), "identifier starting with digit must be rejected") + // Underscore only is valid. + assert.True(t, isValidDB2Identifier("_")) + // Leading underscore is valid. + assert.True(t, isValidDB2Identifier("_TABLE")) + // Trailing underscore is valid. + assert.True(t, isValidDB2Identifier("TABLE_")) + // Mixed digits and letters is valid. + assert.True(t, isValidDB2Identifier("T1_A2")) + // Lowercase must be rejected — callers must upper-case before passing. + assert.False(t, isValidDB2Identifier("table")) + // Empty string is always rejected. + assert.False(t, isValidDB2Identifier("")) +} + +// TestValidateQualifiedIdentifierEdgeCases covers dotted-name forms and +// verifies the two-part limit: SCHEMA.TABLE is valid; SCHEMA.TABLE.EXTRA is +// not because the second segment "TABLE.EXTRA" contains a dot. +func TestValidateQualifiedIdentifierEdgeCases(t *testing.T) { + t.Parallel() + + // Unqualified (no dot) is valid. + assert.NoError(t, validateQualifiedIdentifier("MY_TABLE")) + // Fully qualified with schema is valid. + assert.NoError(t, validateQualifiedIdentifier("MYSCHEMA_CDC.MY_CHECKPOINT")) + // Three parts must be rejected — the second SplitN segment contains a dot. + assert.Error(t, validateQualifiedIdentifier("A.B.C")) + // Empty string must be rejected. + assert.Error(t, validateQualifiedIdentifier("")) + // Valid schema but invalid table part. + assert.Error(t, validateQualifiedIdentifier("SCHEMA.bad-table")) + // Valid schema, valid uppercase table. + assert.NoError(t, validateQualifiedIdentifier("SCHEMA.TABLE_123")) +} + +func TestTableRegexFilterConfig(t *testing.T) { + t.Parallel() + + spec := db2CDCConfigSpec() + + tests := []struct { + name string + configYAML string + errContains string + }{ + { + name: "include_regex accepts matching tables", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +table_include_regex: ["^EMP"] +`, + }, + { + name: "tables empty with include_regex is valid", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +table_include_regex: ["^ORDERS"] +`, + }, + { + name: "both tables and exclude_regex is valid", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES", "ORDERS_AUDIT"] +table_exclude_regex: ["_AUDIT$"] +`, + }, + { + name: "invalid include_regex rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +table_include_regex: ["[invalid"] +`, + errContains: "table_include_regex", + }, + { + name: "invalid exclude_regex rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +tables: ["EMPLOYEES"] +table_exclude_regex: ["[bad"] +`, + errContains: "table_exclude_regex", + }, + { + name: "neither tables nor include_regex rejected", + configYAML: ` +dsn: "DATABASE=SAMPLE;HOSTNAME=db2host;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=secret" +schema: "DB2ADMIN" +`, + errContains: "either tables or table_include_regex must be specified", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + env := service.NewEnvironment() + conf, err := spec.ParseYAML(tc.configYAML, env) + + var initErr error + if err == nil { + mgr := conf.Resources() + license.InjectTestService(mgr) + _, initErr = newDB2CDCInput(conf, mgr) + } + + if tc.errContains == "" { + require.NoError(t, err, "config parse error") + require.NoError(t, initErr, "init error") + } else { + if err != nil { + require.Contains(t, err.Error(), tc.errContains) + } else { + require.Error(t, initErr) + require.Contains(t, initErr.Error(), tc.errContains) + } + } + }) + } +} + +func TestParseSnapshotSignalTablesDeduplication(t *testing.T) { + t.Parallel() + tables := parseSnapshotSignalTables(`{"data-collections":["DB2INST1.ORDERS","DB2INST1.ORDERS","DB2INST1.PRODUCTS"]}`, "DB2INST1", nil) + require.Equal(t, []string{"ORDERS", "PRODUCTS"}, tables) +} + +func TestParseSnapshotSignalTablesFilterPending(t *testing.T) { + t.Parallel() + d := &db2CDCInput{ + pendingIncrementalCtx: &replication.IncrementalSnapshotContext{ + Tables: []replication.IncrementalTable{ + {Schema: "DB2INST1", Name: "ORDERS"}, + }, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + }, + schema: "DB2INST1", + } + tables := d.filterPendingTables([]string{"ORDERS", "PRODUCTS", "SHIPMENTS"}) + assert.Equal(t, []string{"PRODUCTS", "SHIPMENTS"}, tables, + "ORDERS should be filtered out because it is already in pendingIncrementalCtx") +} diff --git a/internal/impl/db2/integration_test.go b/internal/impl/db2/integration_test.go new file mode 100644 index 0000000000..2444ee2e80 --- /dev/null +++ b/internal/impl/db2/integration_test.go @@ -0,0 +1,1319 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2_test + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + _ "github.com/redpanda-data/benthos/v4/public/components/io" + _ "github.com/redpanda-data/benthos/v4/public/components/pure" + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + // Register the db2_cdc input. + _ "github.com/redpanda-data/connect/v4/internal/impl/db2" + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2test" + "github.com/redpanda-data/connect/v4/internal/license" +) + +// parseIntField extracts an integer from a JSON-decoded map value. +// DB2 CLI driver may return INTEGER columns as strings ("1") or as float64 (1) +// depending on the clidriver version and platform; this helper handles both. +func parseIntField(v any) int { + switch t := v.(type) { + case float64: + return int(t) + case string: + n, _ := strconv.Atoi(t) + return n + case int64: + return int(t) + case int: + return t + default: + return 0 + } +} + +// TestIntegrationDB2CDCDriver tests basic SQL driver connectivity and query execution. +func TestIntegrationDB2CDCDriver(t *testing.T) { + integration.CheckSkip(t) + t.Parallel() + + db := db2test.SetupTest(t) + ctx := t.Context() + + // Drop table if it exists from a previous run (idempotent setup). + _, _ = db.ExecContext(ctx, "DROP TABLE DB2INST1.INTEGRATION_TEST") + t.Cleanup(func() { _, _ = db.ExecContext(context.Background(), "DROP TABLE DB2INST1.INTEGRATION_TEST") }) + + // Create a test table and insert rows. + db.MustExecContext(ctx, ` + CREATE TABLE DB2INST1.INTEGRATION_TEST ( + ID INTEGER NOT NULL PRIMARY KEY, + NAME VARCHAR(100) + ) + `) + + for i := 1; i <= 10; i++ { + db.MustExecContext(ctx, + "INSERT INTO DB2INST1.INTEGRATION_TEST (ID, NAME) VALUES (?, ?)", + i, fmt.Sprintf("row-%d", i), + ) + } + + // Query rows back and verify the driver returns correct data. + rows, err := db.QueryContext(ctx, "SELECT ID, NAME FROM DB2INST1.INTEGRATION_TEST ORDER BY ID") + require.NoError(t, err) + defer rows.Close() + + var count int + for rows.Next() { + var id int + var name string + require.NoError(t, rows.Scan(&id, &name)) + t.Logf("row: id=%d name=%s", id, name) + assert.Equal(t, count+1, id) + assert.Equal(t, fmt.Sprintf("row-%d", count+1), name) + count++ + } + require.NoError(t, rows.Err()) + assert.Equal(t, 10, count) +} + +// TestIntegrationDB2DriverQueryContextCancel verifies that QueryContext returns +// within a short deadline when the context expires while a query is running. +// This exercises the QueryerContext + SQLCancel cancel path in driver.go. +// +// Uses a lock-based approach: a blocker connection holds an exclusive row lock +// inside an open transaction, and the cancellable query tries to acquire the +// same lock via SELECT...WITH RS USE AND KEEP EXCLUSIVE LOCKS. Because the +// blocker never commits, the query blocks indefinitely — until SQLCancel fires +// via the context deadline, proving the cancel path works. +// +// A computational cross-join is not reliable because DB2's query optimiser may +// complete it faster than the cancel deadline on a given instance. +func TestIntegrationDB2DriverQueryContextCancel(t *testing.T) { + integration.CheckSkip(t) + t.Parallel() + + db := db2test.SetupTest(t) + ctx := t.Context() + + // Set up a dedicated table for lock contention. + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.CANCEL_QUERY_LOCK_TEST`) + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.CANCEL_QUERY_LOCK_TEST (ID INTEGER NOT NULL PRIMARY KEY, VAL VARCHAR(100))`) + db.MustExecContext(ctx, `INSERT INTO DB2INST1.CANCEL_QUERY_LOCK_TEST (ID, VAL) VALUES (1, 'locked')`) + + // Open a second, independent *sql.DB so the blocker connection is completely + // separate from the pool used by the cancellable query. database/sql reuses + // connections from the pool, so using the same *sql.DB risks both the blocker + // and the query sharing one underlying db2Conn — which would serialize them + // rather than producing contention. + blocker, err := sql.Open("db2-cli", db.DSN) + require.NoError(t, err, "opening blocker connection") + blocker.SetMaxOpenConns(1) + t.Cleanup(func() { _ = blocker.Close() }) + + // Start a transaction on the blocker connection and acquire an exclusive lock + // on the test row by updating it (without committing). + blockerTx, err := blocker.BeginTx(ctx, nil) + require.NoError(t, err, "beginning blocker transaction") + _, err = blockerTx.ExecContext(ctx, + `UPDATE DB2INST1.CANCEL_QUERY_LOCK_TEST SET VAL = 'still-locked' WHERE ID = 1`) + require.NoError(t, err, "blocker UPDATE (acquiring row lock)") + + // The blocker holds the lock. Now run a SELECT that must acquire the same lock; + // it will block until the blocker commits/rolls back or until SQLCancel fires. + // WITH RS USE AND KEEP EXCLUSIVE LOCKS forces DB2 to request an X lock on the + // row rather than a share lock, causing contention with the blocker's U/X lock. + cancelCtx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + + start := time.Now() + rows, queryErr := db.QueryContext(cancelCtx, + `SELECT ID, VAL FROM DB2INST1.CANCEL_QUERY_LOCK_TEST WHERE ID = 1 WITH RS USE AND KEEP EXCLUSIVE LOCKS`) + if rows != nil { + _ = rows.Err() + _ = rows.Close() + } + elapsed := time.Since(start) + + // Release the blocker lock so the table can be dropped at test teardown. + _ = blockerTx.Rollback() + + // require.Error stops the test if err == nil, preventing nil dereference below. + require.Error(t, queryErr, "expected an error from cancelled context — SQLCancel should have fired") + assert.True(t, + errors.Is(queryErr, context.DeadlineExceeded) || + errors.Is(queryErr, context.Canceled) || + strings.Contains(queryErr.Error(), "context"), + "expected context error, got: %v", queryErr) + // Generous bound: must return well within 10 s (did not block forever). + assert.Less(t, elapsed, 10*time.Second, "QueryContext did not return promptly after cancel") +} + +// TestIntegrationDB2DriverExecContextCancel verifies that ExecContext returns +// within a short deadline when the context expires during a DML statement. +// This exercises the ExecerContext + SQLCancel cancel path in driver.go. +// +// Uses a lock-based approach: a blocker connection holds an exclusive row lock +// inside an open transaction, and the cancellable UPDATE tries to modify the +// same row. Because the blocker never commits, the UPDATE blocks indefinitely +// — until SQLCancel fires via the context deadline, proving the cancel path +// works for ExecerContext. +// +// A computational UPDATE over many rows is not reliable because DB2 can complete +// it faster than the cancel deadline on a given instance. +func TestIntegrationDB2DriverExecContextCancel(t *testing.T) { + integration.CheckSkip(t) + t.Parallel() + + db := db2test.SetupTest(t) + ctx := t.Context() + + // Set up a table for lock contention. + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.CANCEL_EXEC_LOCK_TEST`) + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.CANCEL_EXEC_LOCK_TEST (ID INTEGER NOT NULL PRIMARY KEY, VAL VARCHAR(100))`) + db.MustExecContext(ctx, `INSERT INTO DB2INST1.CANCEL_EXEC_LOCK_TEST (ID, VAL) VALUES (1, 'initial')`) + + // Open a second, independent *sql.DB for the blocker connection so it does + // not share the underlying db2Conn with the test's cancel query. + blocker, err := sql.Open("db2-cli", db.DSN) + require.NoError(t, err, "opening blocker connection") + blocker.SetMaxOpenConns(1) + t.Cleanup(func() { _ = blocker.Close() }) + + // Begin a transaction and acquire an exclusive row lock via UPDATE (no commit). + blockerTx, err := blocker.BeginTx(ctx, nil) + require.NoError(t, err, "beginning blocker transaction") + _, err = blockerTx.ExecContext(ctx, + `UPDATE DB2INST1.CANCEL_EXEC_LOCK_TEST SET VAL = 'held' WHERE ID = 1`) + require.NoError(t, err, "blocker UPDATE (acquiring row lock)") + + // Now try to UPDATE the same row with a short deadline — it must block + // waiting for the lock held by blockerTx, then be cancelled by SQLCancel. + cancelCtx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + + start := time.Now() + _, execErr := db.ExecContext(cancelCtx, + `UPDATE DB2INST1.CANCEL_EXEC_LOCK_TEST SET VAL = 'cancelled' WHERE ID = 1`) + elapsed := time.Since(start) + + // Release the blocker lock so the table can be dropped at test teardown. + _ = blockerTx.Rollback() + + require.Error(t, execErr, "expected an error from cancelled context — SQLCancel should have fired") + assert.True(t, + errors.Is(execErr, context.DeadlineExceeded) || + errors.Is(execErr, context.Canceled) || + strings.Contains(execErr.Error(), "context"), + "expected context error, got: %v", execErr) + assert.Less(t, elapsed, 10*time.Second, "ExecContext did not return promptly after cancel") +} + +// TestIntegrationDB2CDCSnapshotAndStreaming verifies the full snapshot → streaming flow. +func TestIntegrationDB2CDCSnapshotAndStreaming(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + // Deactivate the CDC registration before dropping the table. If asncap is + // running with this table registered (STATE='A'), it will read the DROP TABLE + // log entry and crash with ASN0061E. Setting STATE='I' first tells asncap + // to ignore log entries for this table, so the DROP is safe. + _, _ = db.ExecContext(ctx, "UPDATE ASNCDC.IBMSNAP_REGISTER SET STATE='I' WHERE SOURCE_OWNER='DB2INST1' AND SOURCE_TABLE='CDC_TEST_EMPLOYEES'") + _, _ = db.ExecContext(ctx, "DROP TABLE DB2INST1.CDC_TEST_EMPLOYEES") + // Cleanup runs in LIFO order: stream.Stop → this Cleanup. + // The stream is already stopped when we run, so deactivate then drop. + t.Cleanup(func() { + ctx2 := context.Background() + _, _ = db.ExecContext(ctx2, "UPDATE ASNCDC.IBMSNAP_REGISTER SET STATE='I' WHERE SOURCE_OWNER='DB2INST1' AND SOURCE_TABLE='CDC_TEST_EMPLOYEES'") + _, _ = db.ExecContext(ctx2, "DROP TABLE DB2INST1.CDC_TEST_EMPLOYEES") + }) + + // Clear any stale checkpoint so this run always starts with a fresh snapshot. + _, _ = db.ExecContext(ctx, "DELETE FROM DB2INST1.CDC_CHECKPOINT") + t.Cleanup(func() { _, _ = db.ExecContext(context.Background(), "DELETE FROM DB2INST1.CDC_CHECKPOINT") }) + + // Create the test table. + db.MustExecContext(ctx, ` + CREATE TABLE DB2INST1.CDC_TEST_EMPLOYEES ( + EMP_ID INTEGER NOT NULL PRIMARY KEY, + EMP_NAME VARCHAR(100) + ) + `) + + // Pre-populate for snapshot. + for i := 1; i <= 5; i++ { + db.MustExecContext(ctx, + "INSERT INTO DB2INST1.CDC_TEST_EMPLOYEES (EMP_ID, EMP_NAME) VALUES (?, ?)", + i, fmt.Sprintf("employee-%d", i), + ) + } + + // Enable ASNCDC capture on this table. + db.EnableASNCDC("DB2INST1", []string{"CDC_TEST_EMPLOYEES"}) + + // Build and run the connector. + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["CDC_TEST_EMPLOYEES"] + snapshot_mode: initial + snapshot_max_batch_size: 10 + poll_batch_size: 100 + stream_backoff_interval: 500ms + checkpoint_cache_table_name: "DB2INST1.CDC_CHECKPOINT" +`, db.DSN) + + var ( + received []string + receivedMu sync.Mutex + ) + + streamBuilder := service.NewStreamBuilder() + require.NoError(t, streamBuilder.AddInputYAML(connectorYAML)) + require.NoError(t, streamBuilder.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + receivedMu.Lock() + defer receivedMu.Unlock() + for _, msg := range batch { + b, err := msg.AsBytes() + if err != nil { + continue + } + op, _ := msg.MetaGet("db2_operation") + table, _ := msg.MetaGet("db2_table") + csn, _ := msg.MetaGet("db2_csn") + t.Logf("CDC event [%s] %s csn=%s: %s", op, table, csn, b) + received = append(received, string(b)) + } + return nil + })) + + stream, err := streamBuilder.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + streamDone := make(chan error, 1) + go func() { + streamDone <- stream.Run(ctx) + }() + t.Cleanup(func() { + if err := stream.StopWithin(10 * time.Second); err != nil { + t.Logf("stream stop: %v", err) + } + if err := <-streamDone; err != nil && !errors.Is(err, context.Canceled) { + t.Logf("stream error: %v", err) + } + }) + + // Wait for snapshot rows (5 pre-populated rows). + assert.Eventually(t, func() bool { + receivedMu.Lock() + defer receivedMu.Unlock() + return len(received) >= 5 + }, 2*time.Minute, 500*time.Millisecond, "snapshot: expected at least 5 events") + + // Log CT table and register state before streaming inserts. + { + var ctCount int + _ = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM ASNCDC."CDC_DB2INST1_CDC_TEST_EMPLOYEES"`).Scan(&ctCount) + var synchHex []byte + _ = db.QueryRowContext(ctx, + "SELECT SYNCHPOINT FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_OWNER='DB2INST1' AND SOURCE_TABLE='CDC_TEST_EMPLOYEES'", + ).Scan(&synchHex) + t.Logf("before streaming inserts: CT_count=%d synchpoint=%X asncap_pid=%s", + ctCount, synchHex, asncapPID()) + } + + // Insert streaming rows while the connector is running. + for i := 6; i <= 10; i++ { + db.MustExecContext(ctx, + "INSERT INTO DB2INST1.CDC_TEST_EMPLOYEES (EMP_ID, EMP_NAME) VALUES (?, ?)", + i, fmt.Sprintf("employee-%d", i), + ) + } + + // Log CT table state a few seconds after inserts — if asncap is working, + // the CT table should have rows by now. + time.Sleep(5 * time.Second) + { + var ctCount int + _ = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM ASNCDC."CDC_DB2INST1_CDC_TEST_EMPLOYEES"`).Scan(&ctCount) + var synchHex []byte + _ = db.QueryRowContext(ctx, + "SELECT SYNCHPOINT FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_OWNER='DB2INST1' AND SOURCE_TABLE='CDC_TEST_EMPLOYEES'", + ).Scan(&synchHex) + t.Logf("5s after streaming inserts: CT_count=%d synchpoint=%X asncap_pid=%s", + ctCount, synchHex, asncapPID()) + if ctCount == 0 { + t.Logf("WARNING: CT table is empty — asncap may not be capturing; printing asncap log") + if logOut, err := os.ReadFile("/tmp/asncap.log"); err == nil && len(logOut) > 0 { + tail := logOut + if len(tail) > 3000 { + tail = tail[len(tail)-3000:] + } + t.Logf("asncap log:\n%s", tail) + } + } + } + + // Wait for streaming rows (5 snapshot + 5 streaming = 10 total). + // asnccmd reinit can take >2 minutes on slow machines, so allow 4 minutes. + assert.Eventually(t, func() bool { + receivedMu.Lock() + defer receivedMu.Unlock() + return len(received) >= 10 + }, 4*time.Minute, 500*time.Millisecond, "streaming: expected at least 10 events total") +} + +// asncapPID returns the PID of the running asncap process for TESTDB, or "none". +func asncapPID() string { + out, _ := exec.Command("pgrep", "-f", "capture_server=TESTDB").Output() + pid := strings.TrimSpace(string(out)) + if pid == "" { + return "none" + } + return pid +} + +// TestIntegrationDB2CDCUpdateBeforeImage verifies that UPDATE operations +// produce a single event with op="u", before=old row, after=new row. +func TestIntegrationDB2CDCUpdateBeforeImage(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + // Cleanup from any previous run. + _, _ = db.ExecContext(ctx, `DROP TABLE ASNCDC."CDC_DB2INST1_CDC_UPDATE_TEST"`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_PRUNCNTL WHERE SOURCE_TABLE='CDC_UPDATE_TEST'`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_TABLE='CDC_UPDATE_TEST'`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.CDC_UPDATE_TEST`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.CDC_UPDATE_CHECKPOINT`) + + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.CDC_UPDATE_TEST (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR(100))`) + db.MustExecContext(ctx, `INSERT INTO DB2INST1.CDC_UPDATE_TEST (ID, NAME) VALUES (1, 'original')`) + db.EnableASNCDC("DB2INST1", []string{"CDC_UPDATE_TEST"}) + + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["CDC_UPDATE_TEST"] + snapshot_mode: never + poll_batch_size: 100 + stream_backoff_interval: 200ms + checkpoint_cache_table_name: "DB2INST1.CDC_UPDATE_CHECKPOINT" +`, db.DSN) + + type cdcEvent struct { + op string + before map[string]any + after map[string]any + } + var ( + received []cdcEvent + receivedMu sync.Mutex + ) + + streamBuilder := service.NewStreamBuilder() + require.NoError(t, streamBuilder.AddInputYAML(connectorYAML)) + require.NoError(t, streamBuilder.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + receivedMu.Lock() + defer receivedMu.Unlock() + for _, msg := range batch { + b, err := msg.AsBytes() + if err != nil { + continue + } + t.Logf("CDC event: %s", b) + var env map[string]any + if jsonErr := json.Unmarshal(b, &env); jsonErr != nil { + continue + } + op, _ := env["op"].(string) + var before, after map[string]any + if v, ok := env["before"]; ok && v != nil { + before, _ = v.(map[string]any) + } + if v, ok := env["after"]; ok && v != nil { + after, _ = v.(map[string]any) + } + received = append(received, cdcEvent{op: op, before: before, after: after}) + } + return nil + })) + + stream, err := streamBuilder.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + streamDone := make(chan error, 1) + go func() { + streamDone <- stream.Run(ctx) + }() + t.Cleanup(func() { + if err := stream.StopWithin(10 * time.Second); err != nil { + t.Logf("stream stop: %v", err) + } + if err := <-streamDone; err != nil && !errors.Is(err, context.Canceled) { + t.Logf("stream error: %v", err) + } + }) + + // Let streaming stabilize. + time.Sleep(5 * time.Second) + + // Perform an UPDATE — DB2 SQL Replication captures this as a D+I pair in the CD table. + db.MustExecContext(ctx, `UPDATE DB2INST1.CDC_UPDATE_TEST SET NAME = 'updated' WHERE ID = 1`) + + assert.Eventually(t, func() bool { + receivedMu.Lock() + defer receivedMu.Unlock() + for _, e := range received { + if e.op == "u" && e.before != nil && e.after != nil { + name, _ := e.before["NAME"].(string) + newName, _ := e.after["NAME"].(string) + return name == "original" && newName == "updated" + } + } + return false + }, 2*time.Minute, 500*time.Millisecond, + "expected UPDATE event with op='u', before.NAME='original', after.NAME='updated'") +} + +// TestIntegrationDB2CDCIncrementalSnapshotMultiChunk verifies that a table with more +// rows than snapshot_max_batch_size is fully snapshotted across multiple chunks with +// no missing or duplicate rows. +func TestIntegrationDB2CDCIncrementalSnapshotMultiChunk(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + const ( + srcTable = "INC_MC_TEST" + schema = "DB2INST1" + rowCount = 20 + chunkSize = 4 + ) + + _, _ = db.ExecContext(ctx, `DROP TABLE ASNCDC."CDC_DB2INST1_INC_MC_TEST"`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_PRUNCNTL WHERE SOURCE_TABLE='INC_MC_TEST'`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_TABLE='INC_MC_TEST'`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_MC_TEST`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_MC_CHECKPOINT`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_MC_SIGNALS`) + + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.INC_MC_TEST (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR(100))`) + for i := 1; i <= rowCount; i++ { + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_MC_TEST (ID, NAME) VALUES (?, ?)`, + i, fmt.Sprintf("row-%d", i), + ) + } + db.EnableASNCDC(schema, []string{srcTable}) + + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["INC_MC_TEST"] + snapshot_mode: never + snapshot_max_batch_size: %d + poll_batch_size: 100 + stream_backoff_interval: 300ms + checkpoint_cache_table_name: "DB2INST1.INC_MC_CHECKPOINT" + signal_table: "DB2INST1.INC_MC_SIGNALS" +`, db.DSN, chunkSize) + + type snapRow struct{ id int } + var ( + snapRows []snapRow + snapRowsMu sync.Mutex + ) + + streamBuilder := service.NewStreamBuilder() + require.NoError(t, streamBuilder.AddInputYAML(connectorYAML)) + require.NoError(t, streamBuilder.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + snapRowsMu.Lock() + defer snapRowsMu.Unlock() + for _, msg := range batch { + b, _ := msg.AsBytes() + var env map[string]any + if err := json.Unmarshal(b, &env); err != nil { + continue + } + op, _ := env["op"].(string) + if op != "r" { + continue + } + after, _ := env["after"].(map[string]any) + if after == nil { + continue + } + id := parseIntField(after["ID"]) + snapRows = append(snapRows, snapRow{id: id}) + t.Logf("snapshot op=r id=%d", id) + } + return nil + })) + + stream, err := streamBuilder.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + streamDone := make(chan error, 1) + go func() { streamDone <- stream.Run(ctx) }() + t.Cleanup(func() { + if err := stream.StopWithin(10 * time.Second); err != nil { + t.Logf("stream stop: %v", err) + } + if err := <-streamDone; err != nil && !errors.Is(err, context.Canceled) { + t.Logf("stream error: %v", err) + } + }) + + // Let streaming stabilize before triggering snapshot. + time.Sleep(5 * time.Second) + + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_MC_SIGNALS (ID, TYPE, DATA) VALUES ('mc-1', 'execute-snapshot', '{"data-collections":["DB2INST1.INC_MC_TEST"]}')`) + + // Wait for all rowCount snapshot events across multiple chunks (chunkSize=4 → 5 chunks). + assert.Eventually(t, func() bool { + snapRowsMu.Lock() + defer snapRowsMu.Unlock() + return len(snapRows) >= rowCount + }, 3*time.Minute, 500*time.Millisecond, + "multi-chunk snapshot: expected %d op=r events", rowCount) + + snapRowsMu.Lock() + defer snapRowsMu.Unlock() + + // Each row ID must appear exactly once. + idCounts := make(map[int]int, rowCount) + for _, r := range snapRows { + idCounts[r.id]++ + } + for i := 1; i <= rowCount; i++ { + assert.Equal(t, 1, idCounts[i], + "row ID=%d: expected exactly 1 snapshot event, got %d", i, idCounts[i]) + } +} + +// TestIntegrationDB2CDCIncrementalSnapshotWithConcurrentUpdate verifies the open-window +// deduplication invariant: when rows are updated concurrently with an incremental snapshot, +// no row ID appears more than once as a snapshot (op="r") event. This confirms the +// DeduplicationWindow correctly evicts snapshot rows whose CDC update arrived in-window. +func TestIntegrationDB2CDCIncrementalSnapshotWithConcurrentUpdate(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + const ( + srcTable = "INC_UPD_TEST" + schema = "DB2INST1" + rowCount = 10 + chunkSize = 3 + ) + + _, _ = db.ExecContext(ctx, `DROP TABLE ASNCDC."CDC_DB2INST1_INC_UPD_TEST"`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_PRUNCNTL WHERE SOURCE_TABLE='INC_UPD_TEST'`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_TABLE='INC_UPD_TEST'`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_UPD_TEST`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_UPD_CHECKPOINT`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_UPD_SIGNALS`) + + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.INC_UPD_TEST (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR(100))`) + for i := 1; i <= rowCount; i++ { + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_UPD_TEST (ID, NAME) VALUES (?, ?)`, + i, fmt.Sprintf("before-%d", i), + ) + } + db.EnableASNCDC(schema, []string{srcTable}) + + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["INC_UPD_TEST"] + snapshot_mode: never + snapshot_max_batch_size: %d + poll_batch_size: 100 + stream_backoff_interval: 300ms + checkpoint_cache_table_name: "DB2INST1.INC_UPD_CHECKPOINT" + signal_table: "DB2INST1.INC_UPD_SIGNALS" +`, db.DSN, chunkSize) + + // Track snapshot events (op="r") and CDC update events (op="u") per row ID. + type eventKind struct { + op string + value string // NAME value + } + var ( + byID = make(map[int][]eventKind) + byIDMu sync.Mutex + ) + + streamBuilder := service.NewStreamBuilder() + require.NoError(t, streamBuilder.AddInputYAML(connectorYAML)) + require.NoError(t, streamBuilder.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + byIDMu.Lock() + defer byIDMu.Unlock() + for _, msg := range batch { + b, _ := msg.AsBytes() + var env map[string]any + if err := json.Unmarshal(b, &env); err != nil { + continue + } + op, _ := env["op"].(string) + if op != "r" && op != "u" { + continue + } + // For op="r" and op="u", the row is in "after". + after, _ := env["after"].(map[string]any) + if after == nil { + continue + } + id := parseIntField(after["ID"]) + name, _ := after["NAME"].(string) + byID[id] = append(byID[id], eventKind{op: op, value: name}) + t.Logf("event op=%s id=%d name=%s", op, id, name) + } + return nil + })) + + stream, err := streamBuilder.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + streamDone := make(chan error, 1) + go func() { streamDone <- stream.Run(ctx) }() + t.Cleanup(func() { + if err := stream.StopWithin(10 * time.Second); err != nil { + t.Logf("stream stop: %v", err) + } + if err := <-streamDone; err != nil && !errors.Is(err, context.Canceled) { + t.Logf("stream error: %v", err) + } + }) + + time.Sleep(5 * time.Second) + + // Trigger snapshot, then immediately update all rows concurrently. + // Some updates will land inside the dedup window (evicting snapshot rows); + // others will land after — both orderings are correct by the algorithm. + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_UPD_SIGNALS (ID, TYPE, DATA) VALUES ('upd-1', 'execute-snapshot', '{"data-collections":["DB2INST1.INC_UPD_TEST"]}')`) + + go func() { + // Update all rows as fast as possible — overlapping with snapshot chunks. + for i := 1; i <= rowCount; i++ { + _, _ = db.ExecContext(ctx, + `UPDATE DB2INST1.INC_UPD_TEST SET NAME = ? WHERE ID = ?`, + fmt.Sprintf("after-%d", i), i, + ) + } + }() + + // Wait until we have at least rowCount total events (snapshot + CDC combined) + // and at least rowCount distinct IDs appear with some event. + assert.Eventually(t, func() bool { + byIDMu.Lock() + defer byIDMu.Unlock() + return len(byID) >= rowCount + }, 3*time.Minute, 500*time.Millisecond, + "concurrent-update: expected events for all %d row IDs", rowCount) + + // Wait for at least one CDC update event. The snapshot may complete before + // asncap flushes the in-flight update transactions to the CD table; give it + // extra time to deliver the CDC events so the dedup-window invariant can be + // verified. + assert.Eventually(t, func() bool { + byIDMu.Lock() + defer byIDMu.Unlock() + for _, events := range byID { + for _, e := range events { + if e.op == "u" { + return true + } + } + } + return false + }, 2*time.Minute, 500*time.Millisecond, + "concurrent-update: expected at least 1 CDC update event (proves dedup window was exercised)") + + // KEY INVARIANT: no row ID appears more than once as op="r" (snapshot). + // Regardless of whether the update was inside or outside the dedup window, + // the dedup window must never emit two snapshot events for the same row. + byIDMu.Lock() + defer byIDMu.Unlock() + + totalCDCUpdates := 0 + for id, events := range byID { + readCount := 0 + for _, e := range events { + if e.op == "r" { + readCount++ + } + if e.op == "u" { + totalCDCUpdates++ + } + } + assert.LessOrEqual(t, readCount, 1, + "row ID=%d: dedup window must emit at most 1 snapshot event, got %d", id, readCount) + } + assert.GreaterOrEqual(t, totalCDCUpdates, 1, + "expected at least 1 CDC update event — proves the dedup window was actually exercised") +} + +// TestIntegrationDB2CDCIncrementalSnapshotWithConcurrentDelete verifies that when rows +// are deleted during an incremental snapshot, each deleted row appears at most once as +// op="r": either the snapshot row is evicted by the in-window CDC delete (row appears +// only as op="d"), or the delete arrives after the window closes (row appears as op="r" +// then op="d"). No row may appear twice as op="r". +func TestIntegrationDB2CDCIncrementalSnapshotWithConcurrentDelete(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + const ( + srcTable = "INC_DEL_TEST" + schema = "DB2INST1" + rowCount = 10 + deleteCount = 5 // delete the first 5 rows + chunkSize = 3 + ) + + _, _ = db.ExecContext(ctx, `DROP TABLE ASNCDC."CDC_DB2INST1_INC_DEL_TEST"`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_PRUNCNTL WHERE SOURCE_TABLE='INC_DEL_TEST'`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_TABLE='INC_DEL_TEST'`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_DEL_TEST`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_DEL_CHECKPOINT`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_DEL_SIGNALS`) + + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.INC_DEL_TEST (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR(100))`) + for i := 1; i <= rowCount; i++ { + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_DEL_TEST (ID, NAME) VALUES (?, ?)`, + i, fmt.Sprintf("row-%d", i), + ) + } + db.EnableASNCDC(schema, []string{srcTable}) + + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["INC_DEL_TEST"] + snapshot_mode: never + snapshot_max_batch_size: %d + poll_batch_size: 100 + stream_backoff_interval: 300ms + checkpoint_cache_table_name: "DB2INST1.INC_DEL_CHECKPOINT" + signal_table: "DB2INST1.INC_DEL_SIGNALS" +`, db.DSN, chunkSize) + + // Track events per row ID. + var ( + snapCount = make(map[int]int) // op="r" count per ID + concurrentDeletes = make(map[int]int) // op="d" count per ID + eventsMu sync.Mutex + ) + + streamBuilder := service.NewStreamBuilder() + require.NoError(t, streamBuilder.AddInputYAML(connectorYAML)) + require.NoError(t, streamBuilder.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + eventsMu.Lock() + defer eventsMu.Unlock() + for _, msg := range batch { + b, _ := msg.AsBytes() + var env map[string]any + if err := json.Unmarshal(b, &env); err != nil { + continue + } + op, _ := env["op"].(string) + if op != "r" && op != "d" { + continue + } + // For op="d", row data is in "before"; for op="r" it's in "after". + rowData, _ := env["after"].(map[string]any) + if op == "d" { + rowData, _ = env["before"].(map[string]any) + } + if rowData == nil { + continue + } + id := parseIntField(rowData["ID"]) + if op == "r" { + snapCount[id]++ + t.Logf("op=r id=%d", id) + } else { + concurrentDeletes[id]++ + t.Logf("op=d id=%d", id) + } + } + return nil + })) + + stream, err := streamBuilder.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + streamDone := make(chan error, 1) + go func() { streamDone <- stream.Run(ctx) }() + t.Cleanup(func() { + if err := stream.StopWithin(10 * time.Second); err != nil { + t.Logf("stream stop: %v", err) + } + if err := <-streamDone; err != nil && !errors.Is(err, context.Canceled) { + t.Logf("stream error: %v", err) + } + }) + + time.Sleep(5 * time.Second) + + // Trigger snapshot and concurrently delete the first deleteCount rows. + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_DEL_SIGNALS (ID, TYPE, DATA) VALUES ('del-1', 'execute-snapshot', '{"data-collections":["DB2INST1.INC_DEL_TEST"]}')`) + + go func() { + for i := 1; i <= deleteCount; i++ { + _, _ = db.ExecContext(ctx, + `DELETE FROM DB2INST1.INC_DEL_TEST WHERE ID = ?`, i) + } + }() + + // Wait for snapshot events for the non-deleted rows (IDs deleteCount+1 through rowCount). + assert.Eventually(t, func() bool { + eventsMu.Lock() + defer eventsMu.Unlock() + for i := deleteCount + 1; i <= rowCount; i++ { + if snapCount[i] == 0 { + return false + } + } + return true + }, 3*time.Minute, 500*time.Millisecond, + "concurrent-delete: expected snapshot events for surviving rows %d-%d", deleteCount+1, rowCount) + + // KEY INVARIANT: no row ID appears more than once as op="r". + eventsMu.Lock() + defer eventsMu.Unlock() + + for i := 1; i <= rowCount; i++ { + assert.LessOrEqual(t, snapCount[i], 1, + "row ID=%d: dedup window must emit at most 1 snapshot event, got %d", i, snapCount[i]) + } +} + +// TestIntegrationDB2CDCIncrementalSnapshotWithConcurrentInsert verifies that new rows +// inserted after the snapshot window opens arrive via CDC (op="c") and are not emitted +// a second time as snapshot (op="r") rows — the dedup window never evicts CDC inserts. +func TestIntegrationDB2CDCIncrementalSnapshotWithConcurrentInsert(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + const ( + srcTable = "INC_INS_TEST" + schema = "DB2INST1" + baseRows = 10 + newRows = 5 + chunkSize = 4 + ) + + _, _ = db.ExecContext(ctx, `DROP TABLE ASNCDC."CDC_DB2INST1_INC_INS_TEST"`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_PRUNCNTL WHERE SOURCE_TABLE='INC_INS_TEST'`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_TABLE='INC_INS_TEST'`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_INS_TEST`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_INS_CHECKPOINT`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_INS_SIGNALS`) + + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.INC_INS_TEST (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR(100))`) + for i := 1; i <= baseRows; i++ { + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_INS_TEST (ID, NAME) VALUES (?, ?)`, + i, fmt.Sprintf("base-%d", i), + ) + } + db.EnableASNCDC(schema, []string{srcTable}) + + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["INC_INS_TEST"] + snapshot_mode: never + snapshot_max_batch_size: %d + poll_batch_size: 100 + stream_backoff_interval: 300ms + checkpoint_cache_table_name: "DB2INST1.INC_INS_CHECKPOINT" + signal_table: "DB2INST1.INC_INS_SIGNALS" +`, db.DSN, chunkSize) + + var ( + snapIDs = make(map[int]int) // op="r" count per ID + insertIDs = make(map[int]int) // op="c" count per ID + eventsMu sync.Mutex + ) + + streamBuilder := service.NewStreamBuilder() + require.NoError(t, streamBuilder.AddInputYAML(connectorYAML)) + require.NoError(t, streamBuilder.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + eventsMu.Lock() + defer eventsMu.Unlock() + for _, msg := range batch { + b, _ := msg.AsBytes() + var env map[string]any + if err := json.Unmarshal(b, &env); err != nil { + continue + } + op, _ := env["op"].(string) + if op != "r" && op != "c" { + continue + } + after, _ := env["after"].(map[string]any) + if after == nil { + continue + } + id := parseIntField(after["ID"]) + if op == "r" { + snapIDs[id]++ + t.Logf("op=r id=%d", id) + } else { + insertIDs[id]++ + t.Logf("op=c id=%d", id) + } + } + return nil + })) + + stream, err := streamBuilder.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + streamDone := make(chan error, 1) + go func() { streamDone <- stream.Run(ctx) }() + t.Cleanup(func() { + if err := stream.StopWithin(10 * time.Second); err != nil { + t.Logf("stream stop: %v", err) + } + if err := <-streamDone; err != nil && !errors.Is(err, context.Canceled) { + t.Logf("stream error: %v", err) + } + }) + + time.Sleep(5 * time.Second) + + // Trigger snapshot and concurrently insert new rows beyond the original key range. + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_INS_SIGNALS (ID, TYPE, DATA) VALUES ('ins-1', 'execute-snapshot', '{"data-collections":["DB2INST1.INC_INS_TEST"]}')`) + + go func() { + for i := baseRows + 1; i <= baseRows+newRows; i++ { + _, _ = db.ExecContext(ctx, + `INSERT INTO DB2INST1.INC_INS_TEST (ID, NAME) VALUES (?, ?)`, + i, fmt.Sprintf("new-%d", i), + ) + } + }() + + // Wait for the base snapshot rows. + assert.Eventually(t, func() bool { + eventsMu.Lock() + defer eventsMu.Unlock() + return len(snapIDs) >= baseRows + }, 3*time.Minute, 500*time.Millisecond, + "concurrent-insert: expected %d snapshot events for base rows", baseRows) + + // Wait for all new rows to appear as CDC inserts. + assert.Eventually(t, func() bool { + eventsMu.Lock() + defer eventsMu.Unlock() + for i := baseRows + 1; i <= baseRows+newRows; i++ { + // New rows may appear as op="c" (CDC) or op="r" (if the snapshot chunk + // was read after the insert and before the chunk's highCSN). Either is + // correct; what matters is no double-emission. + if snapIDs[i]+insertIDs[i] == 0 { + return false + } + } + return true + }, 3*time.Minute, 500*time.Millisecond, + "concurrent-insert: expected all new row IDs to appear via CDC or snapshot") + + eventsMu.Lock() + defer eventsMu.Unlock() + + // Base rows should appear exactly once as op="r" (they were in the snapshot). + for i := 1; i <= baseRows; i++ { + assert.Equal(t, 1, snapIDs[i], + "base row ID=%d: expected exactly 1 snapshot event", i) + } + + // New rows must appear exactly once — LessOrEqual(1) would allow silent drops (count=0). + for i := baseRows + 1; i <= baseRows+newRows; i++ { + total := snapIDs[i] + insertIDs[i] + assert.Equal(t, 1, total, + "new row ID=%d: must appear exactly once across op=r and op=c, got %d", i, total) + } +} + +// TestIntegrationDB2CDCIncrementalSnapshot verifies that inserting an execute-snapshot +// signal row triggers an ad-hoc re-snapshot of the specified table. +func TestIntegrationDB2CDCIncrementalSnapshot(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + _, _ = db.ExecContext(ctx, `DROP TABLE ASNCDC."CDC_DB2INST1_INC_SNAP_TEST"`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_PRUNCNTL WHERE SOURCE_TABLE='INC_SNAP_TEST'`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_TABLE='INC_SNAP_TEST'`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_SNAP_TEST`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_SNAP_CHECKPOINT`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.CDC_SIGNALS`) + + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.INC_SNAP_TEST (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR(100))`) + db.MustExecContext(ctx, `INSERT INTO DB2INST1.INC_SNAP_TEST (ID, NAME) VALUES (1, 'row-1')`) + db.MustExecContext(ctx, `INSERT INTO DB2INST1.INC_SNAP_TEST (ID, NAME) VALUES (2, 'row-2')`) + db.EnableASNCDC("DB2INST1", []string{"INC_SNAP_TEST"}) + + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["INC_SNAP_TEST"] + snapshot_mode: never + poll_batch_size: 100 + stream_backoff_interval: 500ms + checkpoint_cache_table_name: "DB2INST1.INC_SNAP_CHECKPOINT" + signal_table: "DB2INST1.CDC_SIGNALS" +`, db.DSN) + + var ( + received []string + receivedMu sync.Mutex + ) + + streamBuilder := service.NewStreamBuilder() + require.NoError(t, streamBuilder.AddInputYAML(connectorYAML)) + require.NoError(t, streamBuilder.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + receivedMu.Lock() + defer receivedMu.Unlock() + for _, msg := range batch { + b, err := msg.AsBytes() + if err != nil { + continue + } + op, _ := msg.MetaGet("db2_operation") + t.Logf("CDC event [%s]: %s", op, b) + received = append(received, string(b)) + } + return nil + })) + + stream, err := streamBuilder.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + streamDone := make(chan error, 1) + go func() { + streamDone <- stream.Run(ctx) + }() + t.Cleanup(func() { + if err := stream.StopWithin(10 * time.Second); err != nil { + t.Logf("stream stop: %v", err) + } + if err := <-streamDone; err != nil && !errors.Is(err, context.Canceled) { + t.Logf("stream error: %v", err) + } + }) + + // Let streaming stabilize (snapshot_mode=never so no initial events). + time.Sleep(5 * time.Second) + + receivedMu.Lock() + initialCount := len(received) + receivedMu.Unlock() + t.Logf("events before signal: %d", initialCount) + + // Insert execute-snapshot signal. + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.CDC_SIGNALS (ID, TYPE, DATA) VALUES ('sig-1', 'execute-snapshot', '{"data-collections":["DB2INST1.INC_SNAP_TEST"]}')`) + + // Wait for the re-snapshot to deliver the 2 pre-existing rows. + assert.Eventually(t, func() bool { + receivedMu.Lock() + defer receivedMu.Unlock() + return len(received)-initialCount >= 2 + }, 2*time.Minute, 500*time.Millisecond, + "incremental snapshot: expected at least 2 events after execute-snapshot signal") +} + +// TestIntegrationDB2CDCIncrementalSnapshotResumeAfterRestart verifies that the +// incremental snapshot engine resumes from a per-chunk checkpoint after a +// connector restart rather than re-delivering all rows from the beginning. +// +// Algorithm under test: +// 1. Insert 20 rows into INC_RESUME_TEST (chunkSize=4 → 5 chunks) +// 2. Start connector; trigger snapshot; wait until ≥2 chunks processed (≥8 op="r") +// 3. Stop the connector (simulating a restart) +// 4. Start a NEW connector instance with the same checkpoint table +// 5. Verify all 20 rows appear as op="r" and no row ID appears more than once +func TestIntegrationDB2CDCIncrementalSnapshotResumeAfterRestart(t *testing.T) { + integration.CheckSkip(t) + + db := db2test.SetupTest(t) + ctx := t.Context() + + const ( + srcTable = "INC_RESUME_TEST" + schema = "DB2INST1" + rowCount = 20 + chunkSize = 4 + ) + + _, _ = db.ExecContext(ctx, `DROP TABLE ASNCDC."CDC_DB2INST1_INC_RESUME_TEST"`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_PRUNCNTL WHERE SOURCE_TABLE='INC_RESUME_TEST'`) + _, _ = db.ExecContext(ctx, `DELETE FROM ASNCDC.IBMSNAP_REGISTER WHERE SOURCE_TABLE='INC_RESUME_TEST'`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_RESUME_TEST`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_RESUME_CHECKPOINT`) + _, _ = db.ExecContext(ctx, `DROP TABLE DB2INST1.INC_RESUME_SIGNALS`) + + db.MustExecContext(ctx, `CREATE TABLE DB2INST1.INC_RESUME_TEST (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR(100))`) + for i := 1; i <= rowCount; i++ { + db.MustExecContext(ctx, `INSERT INTO DB2INST1.INC_RESUME_TEST (ID, NAME) VALUES (?, ?)`, i, fmt.Sprintf("row-%d", i)) + } + db.EnableASNCDC(schema, []string{srcTable}) + + connectorYAML := fmt.Sprintf(` +db2_cdc: + dsn: %q + schema: "DB2INST1" + tables: ["INC_RESUME_TEST"] + snapshot_mode: never + snapshot_max_batch_size: %d + poll_batch_size: 100 + stream_backoff_interval: 300ms + checkpoint_cache_table_name: "DB2INST1.INC_RESUME_CHECKPOINT" + signal_table: "DB2INST1.INC_RESUME_SIGNALS" +`, db.DSN, chunkSize) + + // snapshotByID tracks how many times each row ID was delivered as op="r". + snapshotByID := make(map[int]int) + var mu sync.Mutex + + buildStream := func(t *testing.T) *service.Stream { + t.Helper() + sb := service.NewStreamBuilder() + require.NoError(t, sb.AddInputYAML(connectorYAML)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + b, _ := msg.AsBytes() + var env map[string]any + if err := json.Unmarshal(b, &env); err != nil { + continue + } + if env["op"] != "r" { + continue + } + after, _ := env["after"].(map[string]any) + if after == nil { + continue + } + id := parseIntField(after["ID"]) + snapshotByID[id]++ + } + return nil + })) + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + return stream + } + + // Phase 1: start connector, trigger snapshot, wait for ≥2 chunks worth of rows. + stream1 := buildStream(t) + done1 := make(chan error, 1) + go func() { done1 <- stream1.Run(ctx) }() + + time.Sleep(5 * time.Second) // let streaming stabilise + + db.MustExecContext(ctx, + `INSERT INTO DB2INST1.INC_RESUME_SIGNALS (ID, TYPE, DATA) VALUES ('resume-1', 'execute-snapshot', '{"data-collections":["DB2INST1.INC_RESUME_TEST"]}')`) + + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(snapshotByID) >= 2*chunkSize + }, 2*time.Minute, 500*time.Millisecond, + "expected at least %d op=r events after 2 chunks", 2*chunkSize) + + // Phase 2: stop connector (simulate crash/restart). + require.NoError(t, stream1.StopWithin(10*time.Second)) + <-done1 + + mu.Lock() + rowsAfterPhase1 := len(snapshotByID) + mu.Unlock() + t.Logf("Phase 1: %d rows snapshotted before restart", rowsAfterPhase1) + + // Phase 3: start a new connector with the same checkpoint table. + stream2 := buildStream(t) + done2 := make(chan error, 1) + go func() { done2 <- stream2.Run(ctx) }() + t.Cleanup(func() { + _ = stream2.StopWithin(10 * time.Second) + <-done2 + }) + + // Wait for all 20 rows to appear. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(snapshotByID) == rowCount + }, 3*time.Minute, 500*time.Millisecond, + "expected all %d rows after resume", rowCount) + + // Verify no duplicates — each row must appear exactly once. + mu.Lock() + defer mu.Unlock() + for id, count := range snapshotByID { + assert.Equal(t, 1, count, + "row id=%d appeared %d times (expected exactly 1 — no duplicates across restart)", id, count) + } +} diff --git a/internal/impl/db2/replication/batch_truncation_test.go b/internal/impl/db2/replication/batch_truncation_test.go new file mode 100644 index 0000000000..bc987bfefe --- /dev/null +++ b/internal/impl/db2/replication/batch_truncation_test.go @@ -0,0 +1,207 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "context" + "database/sql/driver" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPollChangesBatchTruncationSafety verifies the watermark safety rule: +// when any change table returns exactly PollBatchSize rows the batch may have +// been cut in the middle of a transaction. The connector must NOT advance the +// checkpoint past the last CSN seen because the remaining rows in that +// transaction would be permanently skipped by the "COMMITSEQ > afterCSN" +// predicate on the next poll. +// +// Scenario: PollBatchSize=2, change table returns exactly 2 rows at CSN=10 and +// CSN=20. The last CSN (20) is trimmed; only the row at CSN=10 is delivered +// and maxCSN = 10. The next poll re-fetches from afterCSN=10 so CSN=20 is seen. +func TestPollChangesBatchTruncationSafety(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + cols := []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", "ID", + } + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 50}}, + }, nil + } + // Exactly PollBatchSize (2) rows — triggers truncation logic. + return cols, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 10}, int64(1), "I", ts, int64(1)}, + {[]byte{0, 0, 0, 0, 0, 0, 0, 20}, int64(1), "I", ts, int64(2)}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 2, // batch exactly fills → truncation kicks in + }, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + events, maxCSN, _, err := s.pollChanges(context.Background(), NewCSN(0), nil) + + require.NoError(t, err) + // Only the first event (CSN=10) is delivered; CSN=20 is trimmed for re-fetch. + require.Len(t, events, 1, "batch-full: events at the last CSN must be trimmed") + assert.Equal(t, uint64(10), events[0].CSN.Uint64()) + assert.Equal(t, uint64(10), maxCSN.Uint64(), + "maxCSN must be the penultimate distinct CSN, not the trimmed last CSN") +} + +// TestPollChangesBatchTruncationRetrimMultipleRows verifies that when multiple +// rows share the highest CSN in a full batch, ALL of them are trimmed together. +// Only rows with strictly lower CSN values are delivered. +// +// Scenario: PollBatchSize=3, rows at CSN=10, CSN=20, CSN=20. Both CSN=20 rows +// form the D+I pair of an UPDATE and must be trimmed together — splitting them +// would deliver a dangling delete with no matching insert. Only CSN=10 is +// delivered; maxCSN = 10. +func TestPollChangesBatchTruncationRetrimMultipleRows(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + cols := []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", "ID", + } + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 50}}, + }, nil + } + // Exactly PollBatchSize (3) rows; last two share CSN=20 (a D+I UPDATE pair). + return cols, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 10}, int64(1), "I", ts, int64(1)}, + {[]byte{0, 0, 0, 0, 0, 0, 0, 20}, int64(1), "D", ts, int64(2)}, // before-image + {[]byte{0, 0, 0, 0, 0, 0, 0, 20}, int64(2), "I", ts, int64(2)}, // after-image + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 3, + }, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + events, maxCSN, _, err := s.pollChanges(context.Background(), NewCSN(0), nil) + + require.NoError(t, err) + // Both CSN=20 rows (the D+I pair) are trimmed together so the pair is never split. + require.Len(t, events, 1, "both rows at the last CSN must be trimmed when batch is full") + assert.Equal(t, uint64(10), events[0].CSN.Uint64()) + assert.Equal(t, uint64(10), maxCSN.Uint64()) +} + +// TestPollChangesBatchTruncationAllSameCSN verifies the edge-case fallback: +// when every event in a full batch shares the same CSN (a single very large +// transaction exceeding PollBatchSize), no penultimate CSN exists. The connector +// must return all events and advance past that CSN, accepting at-least-once +// delivery semantics. +// +// Without this fallback the connector would stall permanently when a single +// transaction produces more change rows than PollBatchSize. +func TestPollChangesBatchTruncationAllSameCSN(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + sharedCSN := []byte{0, 0, 0, 0, 0, 0, 0, 42} + cols := []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", "ID", + } + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 50}}, + }, nil + } + // All 3 rows share the same CSN — single oversized transaction. + return cols, [][]driver.Value{ + {sharedCSN, int64(1), "I", ts, int64(1)}, + {sharedCSN, int64(2), "I", ts, int64(2)}, + {sharedCSN, int64(3), "I", ts, int64(3)}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 3, // exactly full; all rows share one CSN + }, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + events, maxCSN, _, err := s.pollChanges(context.Background(), NewCSN(0), nil) + + require.NoError(t, err) + // All 3 events must be returned so the connector can make progress. + require.Len(t, events, 3, + "oversized single-transaction batch must deliver all events to avoid stall") + assert.Equal(t, uint64(42), maxCSN.Uint64(), + "maxCSN must equal the shared CSN so the next poll starts after it") +} + +// TestPollChangesNoBatchTruncationWhenUnderLimit verifies that batch truncation +// is NOT applied when the change table returns fewer rows than PollBatchSize. +// All events must be delivered and maxCSN must equal the highest CSN seen. +func TestPollChangesNoBatchTruncationWhenUnderLimit(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + cols := []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", "ID", + } + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 50}}, + }, nil + } + // 2 rows when PollBatchSize=5 — batch is NOT full. + return cols, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 10}, int64(1), "I", ts, int64(1)}, + {[]byte{0, 0, 0, 0, 0, 0, 0, 20}, int64(1), "I", ts, int64(2)}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 5, // batch NOT full → no truncation applied + }, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + events, maxCSN, _, err := s.pollChanges(context.Background(), NewCSN(0), nil) + + require.NoError(t, err) + require.Len(t, events, 2, "under-limit batch must not be truncated") + assert.Equal(t, uint64(10), events[0].CSN.Uint64()) + assert.Equal(t, uint64(20), events[1].CSN.Uint64()) + assert.Equal(t, uint64(20), maxCSN.Uint64(), + "maxCSN must be the highest CSN when batch is under the limit") +} diff --git a/internal/impl/db2/replication/concurrency_stress_test.go b/internal/impl/db2/replication/concurrency_stress_test.go new file mode 100644 index 0000000000..e5cd851e06 --- /dev/null +++ b/internal/impl/db2/replication/concurrency_stress_test.go @@ -0,0 +1,501 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "context" + "database/sql/driver" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// TestConcurrencyDeduplicationWindow — stress test for DeduplicationWindow +// --------------------------------------------------------------------------- + +// TestConcurrencyDeduplicationWindowAddEvictFlushWait exercises the production +// concurrency pattern for DeduplicationWindow: +// +// - AddRow: called sequentially before the window is installed (signals goroutine, +// before SetWindow). +// - Evict + Flush: called sequentially from the Streamer goroutine (always the same +// goroutine — no concurrent Evict/Flush in production). +// - Wait: called from multiple "RunTable-like" goroutines concurrently. +// +// Invariants verified: +// 1. No panics or data-race detector hits. +// 2. Flush is idempotent: a second call returns an empty slice. +// 3. All Wait goroutines unblock after Flush is called. +// 4. Rows evicted before Flush are absent from the flushed result. +// 5. The window terminates within the 10-second deadline. +func TestConcurrencyDeduplicationWindowAddEvictFlushWait(t *testing.T) { + t.Parallel() + + const ( + numRows = 500 + numEvictions = 200 // evict the first 200 rows before flush + numWaiters = 8 + ) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Phase 1: sequential AddRow — production: this happens in ReadChunk before SetWindow. + win := NewDeduplicationWindow("S", "T", NewCSN(100), NewCSN(200), []string{"ID"}) + for i := range numRows { + win.AddRow(map[string]any{"ID": int64(i), "VAL": "snap"}) + } + require.Equal(t, numRows, win.Len()) + + // Phase 2: launch Wait goroutines — they block until Flush is called. + // Production: RunTable calls win.Wait() on the signals goroutine. + var waitWg sync.WaitGroup + waitErrs := make([]error, numWaiters) + for i := range numWaiters { + waitWg.Add(1) + go func(i int) { + defer waitWg.Done() + waitErrs[i] = win.Wait(ctx) + }(i) + } + + // Phase 3: sequential Evict + Flush from a single "Streamer" goroutine. + // Production: Evict and Flush are always called from the same pollChanges goroutine. + streamerDone := make(chan struct{}) + var evictedCount int + var flushed1 []ChangeEvent + go func() { + defer close(streamerDone) + for i := range numEvictions { + if win.Evict("S", "T", map[string]any{"ID": int64(i)}, NewCSN(150)) { + evictedCount++ + } + runtime.Gosched() + } + flushed1 = win.Flush() + // Second Flush must be idempotent. + _ = win.Flush() // return value checked after streamerDone + }() + + <-streamerDone + + // Second Flush from the test goroutine (any goroutine) must return empty. + flushed2 := win.Flush() + assert.Empty(t, flushed2, "second Flush must return empty slice") + + // All Wait goroutines must unblock after Flush. + waitWg.Wait() + for i, err := range waitErrs { + assert.NoError(t, err, "Wait goroutine %d must not return an error", i) + } + + // Flushed events must all be OpTypeRead. + for _, ev := range flushed1 { + assert.Equal(t, OpTypeRead, ev.Operation) + assert.Equal(t, "S", ev.Schema) + assert.Equal(t, "T", ev.Table) + } + + // Invariant: flushed + evicted == numRows. + assert.Equal(t, numRows, len(flushed1)+evictedCount, + "flushed rows + evicted rows must equal total rows added") +} + +// TestConcurrencyDeduplicationWindowContextCancel verifies that Wait returns +// ctx.Err() when the context is cancelled before Flush is called. +func TestConcurrencyDeduplicationWindowContextCancel(t *testing.T) { + t.Parallel() + + win := NewDeduplicationWindow("S", "T", NewCSN(0), NewCSN(100), []string{"ID"}) + win.AddRow(map[string]any{"ID": int64(1)}) + + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { + done <- win.Wait(ctx) + }() + + cancel() + + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled, "Wait must return context.Canceled") + case <-time.After(5 * time.Second): + t.Fatal("Wait did not return after context cancellation") + } +} + +// --------------------------------------------------------------------------- +// TestConcurrencyStreamerSetWindowPollChanges — stress test for Streamer +// --------------------------------------------------------------------------- + +// TestConcurrencyStreamerSetWindowPollChanges exercises concurrent SetWindow +// (from a "RunTable-like" goroutine) and pollChanges (from the Streamer's +// Stream loop). The test verifies that no data races occur and that every +// installed window is eventually flushed. +// +// Approach: +// - A fake DB returns a rising SYNCHPOINT on each call and one CDC INSERT +// per poll (CSN advances monotonically). +// - A parallel goroutine repeatedly installs DeduplicationWindows via +// SetWindow with a HighCSN that the Streamer will pass in a few polls. +// - We drive the Streamer by calling pollChanges in a tight loop instead of +// calling Stream, so we can terminate cleanly within the test timeout. +func TestConcurrencyStreamerSetWindowPollChanges(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const totalPolls = 200 + + // csn advances 1 per poll call. + var pollNum atomic.Int64 + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + n := pollNum.Load() + csnBytes := makeCSNBytes(uint64(n + 1)) + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, + [][]driver.Value{{driver.Value(csnBytes)}}, nil + } + // Change table: one INSERT per poll at the current CSN. + return []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", + }, [][]driver.Value{{ + driver.Value(csnBytes), + driver.Value(int64(1)), + driver.Value("I"), + driver.Value(time.Now()), + driver.Value(n), + }}, nil + }, + }) + + streamer := NewStreamer(db, StreamConfig{ + Schemas: []string{"S"}, + PollBatchSize: 100, + }, Version{}) + streamer.changeTables["S.T"] = "ASNCDC.T_CT" + + // windowInstaller simulates what RunTable does: install a window with + // HighCSN just ahead of the current streamer position, then wait for it. + var installerWg sync.WaitGroup + var windowsInstalled atomic.Int64 + var windowsFlushed atomic.Int64 + + installerWg.Go(func() { + for { + select { + case <-ctx.Done(): + return + default: + } + current := pollNum.Load() + if current >= totalPolls { + return + } + // Install a window with HighCSN = current+5 (the Streamer will pass + // it within a few polls). + highCSN := NewCSN(uint64(current + 5)) + win := NewDeduplicationWindow("S", "T", NewCSN(uint64(current)), highCSN, []string{"ID"}) + win.AddRow(map[string]any{"ID": current}) + streamer.SetWindow(win) + windowsInstalled.Add(1) + + // Wait for flush with a per-window timeout. + winCtx, winCancel := context.WithTimeout(ctx, 3*time.Second) + err := win.Wait(winCtx) + winCancel() + if err == nil { + windowsFlushed.Add(1) + } + // Yield so pollChanges can advance. + runtime.Gosched() + } + }) + + // Driver loop: run pollChanges until totalPolls or ctx expires. + currentCSN := NullCSN() + intentSeqs := map[string]int64{} + for pollNum.Load() < totalPolls { + select { + case <-ctx.Done(): + t.Errorf("context expired before completing %d polls (done %d)", + totalPolls, pollNum.Load()) + cancel() + goto done + default: + } + + events, newCSN, newSeqs, err := streamer.pollChanges(ctx, currentCSN, intentSeqs) + if err != nil { + if ctx.Err() != nil { + break + } + t.Errorf("pollChanges error: %v", err) + break + } + if len(events) > 0 { + currentCSN = newCSN + intentSeqs = newSeqs + } + pollNum.Add(1) + runtime.Gosched() + } + +done: + cancel() + installerWg.Wait() + + // We don't assert exact counts because the window installer and poll loop + // race legitimately; the important thing is no race-detector hits and no + // deadlocks. + t.Logf("windows installed=%d flushed=%d polls=%d", + windowsInstalled.Load(), windowsFlushed.Load(), pollNum.Load()) +} + +// TestConcurrencyStreamerSetWindowOnce exercises the scenario where SetWindow +// is called exactly once while pollChanges is running concurrently. +// The window must be flushed exactly once with consistent state. +func TestConcurrencyStreamerSetWindowOnce(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const ( + numRows = 100 + windowLo = uint64(50) + windowHi = uint64(150) + ) + + // SYNCHPOINT starts below windowHi and rises to windowHi+1 after a few calls. + var synchCallCount atomic.Int64 + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + n := synchCallCount.Add(1) + // Return windowHi+1 after the 3rd call so the streamer passes the window. + var csn uint64 + if n >= 3 { + csn = windowHi + 1 + } else { + csn = windowLo - 1 + } + return []string{"MAX(SYNCHPOINT)"}, + [][]driver.Value{{driver.Value(makeCSNBytes(csn))}}, nil + } + // CDC event at CSN = windowHi (within the window bracket). + return []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", + }, [][]driver.Value{{ + driver.Value(makeCSNBytes(windowHi)), + driver.Value(int64(1)), + driver.Value("I"), + driver.Value(time.Now()), + driver.Value(int64(42)), + }}, nil + }, + }) + + streamer := NewStreamer(db, StreamConfig{ + Schemas: []string{"S"}, + PollBatchSize: 1000, + }, Version{}) + streamer.changeTables["S.T"] = "ASNCDC.T_CT" + + // Build the deduplication window: numRows rows; ID=42 will be evicted by CDC. + win := NewDeduplicationWindow("S", "T", NewCSN(windowLo), NewCSN(windowHi), []string{"ID"}) + for i := range numRows { + win.AddRow(map[string]any{"ID": int64(i)}) + } + streamer.SetWindow(win) + + // Run pollChanges in a goroutine; the main goroutine waits for the window. + var pollWg sync.WaitGroup + pollWg.Go(func() { + currentCSN := NullCSN() + for { + select { + case <-ctx.Done(): + return + default: + } + _, newCSN, _, err := streamer.pollChanges(ctx, currentCSN, nil) + if err != nil || ctx.Err() != nil { + return + } + currentCSN = newCSN + // Stop once the window has been flushed. + streamer.windowMu.Lock() + active := streamer.activeWindow + streamer.windowMu.Unlock() + if active == nil { + return + } + } + }) + + // Wait for the window to be flushed. + err := win.Wait(ctx) + require.NoError(t, err, "window must be flushed within timeout") + + cancel() + pollWg.Wait() + + // Second Flush must be idempotent. + flushed2 := win.Flush() + assert.Empty(t, flushed2, "second Flush must return empty slice") + + // ID=42 was evicted by the CDC event at windowHi; all others survive. + // We can't call Flush again to get results, but we can check LastPKValues. + lastPK := win.LastPKValues() + require.NotNil(t, lastPK, "LastPKValues must return the last-row PK even after Flush") +} + +// TestConcurrencyIncrementalEngineRunTable stress-tests RunTable by running the +// engine loop concurrently with a Streamer that flushes windows as fast as it can. +// Uses the fake DB pattern from incremental_test.go. +func TestConcurrencyIncrementalEngineRunTable(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // 3 chunks of 10 rows each; engine should call checkpoint 3 times. + var chunkCall atomic.Int32 + var checkpointCalled atomic.Int32 + + streamer := NewStreamer(nil, StreamConfig{}, Version{}) + + db := openFakeDB(t, &replFakeHandlers{ + query: func(query string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(query, "KEYCOLUSE") { + return []string{"COLNAME"}, [][]driver.Value{{driver.Value("ID")}}, nil + } + if strings.Contains(query, "SYSCAT.COLUMNS") { + return []string{"COLNAME", "NULLS"}, [][]driver.Value{{driver.Value("ID"), driver.Value("N")}}, nil + } + if strings.Contains(query, "FETCH FIRST 1 ROW ONLY") { + // selectMaxPK: returns the lexicographic max row. + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(30))}}, nil + } + if strings.Contains(query, "SYNCHPOINT") { + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(500))}}, nil + } + n := chunkCall.Add(1) + switch n { + case 1: + rows := make([][]driver.Value, 10) + for i := range rows { + rows[i] = []driver.Value{driver.Value(int64(i + 1))} + } + return []string{"ID"}, rows, nil + case 2: + rows := make([][]driver.Value, 10) + for i := range rows { + rows[i] = []driver.Value{driver.Value(int64(i + 11))} + } + return []string{"ID"}, rows, nil + case 3: + rows := make([][]driver.Value, 10) + for i := range rows { + rows[i] = []driver.Value{driver.Value(int64(i + 21))} + } + return []string{"ID"}, rows, nil + default: + return []string{"ID"}, nil, nil // empty → engine exits + } + }, + }) + + engine := NewIncrementalSnapshotEngine(db, "ASNCDC", 10, streamer, + func(_ context.Context, _ *IncrementalSnapshotContext) error { + checkpointCalled.Add(1) + return nil + }, nil, + ) + + ictx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T"}}, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + } + + // Engine runs on a dedicated goroutine; the test goroutine acts as the Streamer + // by flushing windows as soon as they appear. + var allEvents []ChangeEvent + var engineErr error + errCh := make(chan error, 1) + + go func() { + errCh <- engine.RunTable(ctx, ictx, ictx.Tables[0], NewCSN(0)) + }() + + // Flush loop: flush windows until engine finishes. + for { + select { + case err := <-errCh: + engineErr = err + goto engineDone + case <-ctx.Done(): + t.Error("context expired before engine completed") + goto engineDone + default: + } + + streamer.windowMu.Lock() + w := streamer.activeWindow + streamer.windowMu.Unlock() + + if w != nil { + flushed := w.Flush() + allEvents = append(allEvents, flushed...) + streamer.windowMu.Lock() + if streamer.activeWindow == w { + streamer.activeWindow = nil + } + streamer.windowMu.Unlock() + } else { + runtime.Gosched() + } + } + +engineDone: + require.NoError(t, engineErr) + assert.Equal(t, int32(3), checkpointCalled.Load(), "checkpoint must be called once per chunk") + assert.Len(t, allEvents, 30, "all 30 snapshot rows must be emitted") + for _, ev := range allEvents { + assert.Equal(t, OpTypeRead, ev.Operation) + } +} + +// makeCSNBytes converts a uint64 into the big-endian 8-byte representation +// compatible with the fake DB driver used in this package's tests. +func makeCSNBytes(val uint64) []byte { + b := make([]byte, 8) + for i := 7; i >= 0; i-- { + b[i] = byte(val) + val >>= 8 + } + return b +} diff --git a/internal/impl/db2/replication/fakedb_test.go b/internal/impl/db2/replication/fakedb_test.go new file mode 100644 index 0000000000..70af435c50 --- /dev/null +++ b/internal/impl/db2/replication/fakedb_test.go @@ -0,0 +1,165 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "sync" + "sync/atomic" + "testing" +) + +// Fake SQL driver for unit tests in the replication package. +// Registered once as "fake-db2-repl". Each test creates an independent fake +// DB via openFakeDB, which stores its query/exec handlers keyed by a unique DSN. + +var ( + replFakeDrvOnce sync.Once + replFakeConns sync.Map + replFakeConnSeq atomic.Int64 +) + +// replFakeHandlers configures the behaviour of a fake DB connection. +type replFakeHandlers struct { + // query is called for every SELECT. Return (nil, nil, nil) for no rows. + query func(query string, args []driver.Value) (columns []string, rows [][]driver.Value, err error) + // colTypes optionally maps column name → DB type name for a given query. + // When nil, ColumnTypeDatabaseTypeName returns "" for all columns. + colTypes func(query string) map[string]string + // exec is called for every INSERT/UPDATE/DELETE/CREATE/MERGE. May be nil. + exec func(query string, args []driver.Value) error +} + +// openFakeDB creates a *sql.DB backed by the given handlers. +// The DB is closed and its DSN entry removed when t completes. +func openFakeDB(t *testing.T, h *replFakeHandlers) *sql.DB { + t.Helper() + replFakeDrvOnce.Do(func() { sql.Register("fake-db2-repl", &replFakeDriver{}) }) + dsn := fmt.Sprintf("repl-fake-%d", replFakeConnSeq.Add(1)) + replFakeConns.Store(dsn, h) + db, err := sql.Open("fake-db2-repl", dsn) + if err != nil { + t.Fatalf("openFakeDB: %v", err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + t.Cleanup(func() { + _ = db.Close() + replFakeConns.Delete(dsn) + }) + return db +} + +// ---- driver.Driver ---- + +type replFakeDriver struct{} + +func (*replFakeDriver) Open(dsn string) (driver.Conn, error) { + v, ok := replFakeConns.Load(dsn) + if !ok { + return nil, fmt.Errorf("fake-db2-repl: no handlers for DSN %q", dsn) + } + return &replFakeConn{h: v.(*replFakeHandlers)}, nil +} + +// ---- driver.Conn + driver.ConnBeginTx ---- + +type replFakeConn struct{ h *replFakeHandlers } + +func (c *replFakeConn) Prepare(query string) (driver.Stmt, error) { + return &replFakeStmt{c: c, q: query}, nil +} +func (*replFakeConn) Close() error { return nil } +func (*replFakeConn) Begin() (driver.Tx, error) { return &replFakeTx{}, nil } +func (*replFakeConn) BeginTx(_ context.Context, _ driver.TxOptions) (driver.Tx, error) { + return &replFakeTx{}, nil +} + +// ---- driver.Tx ---- + +type replFakeTx struct{} + +func (*replFakeTx) Commit() error { return nil } +func (*replFakeTx) Rollback() error { return nil } + +// ---- driver.Stmt ---- + +type replFakeStmt struct { + c *replFakeConn + q string +} + +func (*replFakeStmt) Close() error { return nil } +func (*replFakeStmt) NumInput() int { return -1 } + +func (s *replFakeStmt) Exec(args []driver.Value) (driver.Result, error) { + if s.c.h.exec != nil { + return &replFakeResult{}, s.c.h.exec(s.q, args) + } + return &replFakeResult{}, nil +} + +func (s *replFakeStmt) Query(args []driver.Value) (driver.Rows, error) { + if s.c.h.query == nil { + return &replFakeRows{}, nil + } + cols, data, err := s.c.h.query(s.q, args) + if err != nil { + return nil, err + } + var ct map[string]string + if s.c.h.colTypes != nil { + ct = s.c.h.colTypes(s.q) + } + return &replFakeRows{cols: cols, data: data, colTypes: ct}, nil +} + +// ---- driver.Rows + RowsColumnTypeDatabaseTypeName ---- + +type replFakeRows struct { + cols []string + colTypes map[string]string // column name → DB type name (e.g. "VARCHAR", "INTEGER") + data [][]driver.Value + pos int +} + +func (r *replFakeRows) Columns() []string { return r.cols } +func (*replFakeRows) Close() error { return nil } + +func (r *replFakeRows) Next(dest []driver.Value) error { + if r.pos >= len(r.data) { + return io.EOF + } + for i, v := range r.data[r.pos] { + if i < len(dest) { + dest[i] = v + } + } + r.pos++ + return nil +} + +// ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName. +func (r *replFakeRows) ColumnTypeDatabaseTypeName(index int) string { + if r.colTypes == nil || index >= len(r.cols) { + return "" + } + return r.colTypes[r.cols[index]] +} + +// ---- driver.Result ---- + +type replFakeResult struct{} + +func (*replFakeResult) LastInsertId() (int64, error) { return 0, nil } +func (*replFakeResult) RowsAffected() (int64, error) { return 1, nil } diff --git a/internal/impl/db2/replication/incremental.go b/internal/impl/db2/replication/incremental.go new file mode 100644 index 0000000000..eb33e13533 --- /dev/null +++ b/internal/impl/db2/replication/incremental.go @@ -0,0 +1,856 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +// Package replication implements DB2 SQL Replication CDC streaming and the +// open-window deduplication algorithm for incremental snapshotting. +// +// # Open-Window Deduplication Algorithm +// +// Adapted from Netflix DBLog / Debezium DB2 connector. Guarantees that each +// source row is emitted exactly once (op="r") or superseded by the CDC event +// that modified it during the snapshot window. +// +// ┌──────────────────────────────────────────────────────────────────────┐ +// │ INCREMENTAL SNAPSHOT ENGINE │ +// │ │ +// │ CDC stream ──────────────────────────────────────────────────────► │ +// │ (always running) │ +// │ │ +// │ For each chunk: │ +// │ │ +// │ 1. lowCSN = MAX(SYNCHPOINT) ←── watermark before SELECT │ +// │ 2. SELECT * FROM table WHERE pk > lastKey ORDER BY pk │ +// │ FETCH FIRST chunkSize ROWS ONLY │ +// │ 3. highCSN = MAX(SYNCHPOINT) ←── watermark after SELECT │ +// │ │ +// │ 4. Open DeduplicationWindow(lowCSN, highCSN) │ +// │ ┌────────────────────────────────────────────────┐ │ +// │ │ CDC event arrives with CSN in (low, high]? │ │ +// │ │ YES → Evict(pk) from window │ │ +// │ │ NO → Pass through to output │ │ +// │ └────────────────────────────────────────────────┘ │ +// │ │ +// │ 5. Streamer advances past highCSN → calls window.Flush() │ +// │ 6. Surviving rows emitted as op="r" (snapshot read) │ +// │ 7. Checkpoint: save LastEmittedPK so crash can resume here │ +// │ 8. Repeat from step 1 with lastKey = LastPKValues() │ +// │ until chunk is empty (table fully snapshotted) │ +// └──────────────────────────────────────────────────────────────────────┘ +// +// Correctness invariant: any row modified during (lowCSN, highCSN] appears in +// the CDC stream and is evicted. Snapshot emits only rows whose state was +// stable across the entire chunk window. +package replication + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "sync" +) + +// ---- IncrementalTable ------------------------------------------------------- + +// IncrementalTable describes one entry in the pending incremental snapshot queue. +// Tables are processed in queue order; each table is fully snapshotted before +// the next one begins. The queue is persisted in the checkpoint store so that a +// restart resumes the in-progress snapshot rather than discarding completed work. +type IncrementalTable struct { + Schema string `json:"schema"` // DB2 source schema (TABSCHEMA), upper-cased + Name string `json:"name"` // DB2 table name (TABNAME), upper-cased + SignalID string `json:"signal_id"` // ID of the execute-snapshot signal that triggered this entry +} + +// ---- IncrementalSnapshotContext --------------------------------------------- + +// IncrementalSnapshotContext holds all persisted state for an in-progress +// incremental snapshot. Serialized into the checkpoint store so restarts +// resume at the exact last-emitted row. +type IncrementalSnapshotContext struct { + // Tables is the ordered queue of tables still to be snapshotted. + Tables []IncrementalTable `json:"tables"` + + // LastEmittedPK is the last primary-key value successfully emitted to the + // event channel for each table (keyed "schema.table"). The next chunk + // query starts WHERE pk > LastEmittedPK. + LastEmittedPK map[string][]any `json:"last_emitted_pk"` + + // MaxPK is the upper-bound primary key captured at chunk-0 time (SELECT MAX(pk)). + // Stored for observability; the chunk loop ends when the chunk query returns empty. + MaxPK map[string][]any `json:"max_pk"` + + // SignalIDs holds the IDs of the execute-snapshot signal rows that triggered + // this snapshot. Persisted so that if signal DELETE fails after all tables + // are snapshotted, the next session can retry deletion using this set rather + // than re-running the full snapshot. + SignalIDs []string `json:"signal_ids,omitempty"` +} + +// Running reports whether the incremental snapshot has tables remaining in the +// queue. Returns false when all tables have been fully snapshotted. +func (c *IncrementalSnapshotContext) Running() bool { + return len(c.Tables) > 0 +} + +// CurrentTable returns a pointer to the head of the snapshot queue, or nil when +// the queue is empty (Running() == false). The caller must not modify the +// returned value directly; use AdvanceTable to dequeue the current table. +func (c *IncrementalSnapshotContext) CurrentTable() *IncrementalTable { + if !c.Running() { + return nil + } + return &c.Tables[0] +} + +// AdvanceTable dequeues the current table from the snapshot queue. +// After AdvanceTable, CurrentTable returns the next table, or nil if +// the queue is now empty. Callers should persist the context after advancing. +func (c *IncrementalSnapshotContext) AdvanceTable() { + if len(c.Tables) > 0 { + c.Tables = c.Tables[1:] + } +} + +// ---- DeduplicationWindow ---------------------------------------------------- + +// DeduplicationWindow holds in-memory snapshot rows for one chunk. +// Filled by ChunkReader; CDC events arriving within (LowCSN, HighCSN] evict +// rows whose PK was modified. Surviving rows are flushed as OpTypeRead events. +// +// Thread-safety: mu protects all mutable map/slice fields. AddRow is called +// only from the engine goroutine before SetWindow; Evict/Flush are called from +// the Streamer goroutine after SetWindow. The mu guards against future callers +// that might violate this implicit sequencing, and against data-race detection. +type DeduplicationWindow struct { + Schema string + Table string + LowCSN CSN // exclusive lower bound (stream was here when chunk started) + HighCSN CSN // inclusive upper bound (MAX(SYNCHPOINT) after chunk SELECT) + + mu sync.Mutex + pks []string + rows map[string]map[string]any + pkOrder []string // preserves insertion order for deterministic Flush output + lastUniquePK []any // PK values of the last row added via a new (unseen) key + + flushedOnce sync.Once + flushed chan struct{} // closed when Flush() is called +} + +// NewDeduplicationWindow creates a DeduplicationWindow for the given table and +// CSN bracket. pks is the ordered list of primary-key column names used to build +// per-row identity keys. The LowCSN and HighCSN bounds define the half-open +// interval (LowCSN, HighCSN] within which CDC events will evict snapshot rows. +func NewDeduplicationWindow(schema, table string, lowCSN, highCSN CSN, pks []string) *DeduplicationWindow { + return &DeduplicationWindow{ + Schema: schema, + Table: table, + LowCSN: lowCSN, + HighCSN: highCSN, + pks: pks, + rows: make(map[string]map[string]any), + flushed: make(chan struct{}), + } +} + +// normalizePKVal converts []byte values to string so that snapshot rows +// (where the DB2 ODBC driver returns CHAR/VARCHAR columns as []byte) and +// CDC rows (where convertDB2Value has already converted CHAR/VARCHAR to string) +// produce identical pkKey lookup keys. Without this normalisation, Evict always +// misses for varchar/char PK tables, causing every modified row to be emitted +// twice (once as op="r" from the snapshot and once from the CDC stream). +func normalizePKVal(v any) any { + if b, ok := v.([]byte); ok { + return string(b) + } + return v +} + +// pkKey builds a deterministic lookup key from the PK column values in rowData. +// Uses strings.Builder to avoid the JSON encoder overhead (alloc + reflection) +// that the previous json.Marshal approach paid on every AddRow and Evict call. +// The format is "\x00"-delimited scalar representations — sufficient because: +// - PK column values are scalar types (int64, string, json.Number, nil) +// - The key is only used for map lookups, never serialised or parsed +// - "\x00" cannot appear in a valid DB2 identifier or scalar column value +// - normalizePKVal ensures []byte and string produce the same key +func (w *DeduplicationWindow) pkKey(rowData map[string]any) string { + var sb strings.Builder + sb.Grow(len(w.pks) * 16) + for i, pk := range w.pks { + if i > 0 { + sb.WriteByte(0) // \x00 separator + } + // Type-switch on the expected scalar PK types to avoid fmt.Fprintf's + // reflection overhead (called per-row in the hot snapshot path). + switch v := normalizePKVal(rowData[pk]).(type) { + case string: + sb.WriteString(v) + case int64: + var buf [20]byte + sb.Write(strconv.AppendInt(buf[:0], v, 10)) + case int: + var buf [20]byte + sb.Write(strconv.AppendInt(buf[:0], int64(v), 10)) + case json.Number: + sb.WriteString(v.String()) + case float64: + var buf [32]byte + sb.Write(strconv.AppendFloat(buf[:0], v, 'f', -1, 64)) + case nil: + sb.WriteString("\x00") // \x00 prefix distinguishes SQL NULL from the literal string "" + default: + fmt.Fprintf(&sb, "%v", v) + } + } + return sb.String() +} + +// AddRow adds a snapshot row to the window. If a row with the same PK was +// already added, it is replaced with the new value (last-write wins for the +// chunk SELECT). Duplicate PKs within a chunk SELECT should not occur in +// practice (primary keys are unique by definition), but the replacement +// behaviour is safe. +func (w *DeduplicationWindow) AddRow(row map[string]any) { + w.mu.Lock() + defer w.mu.Unlock() + key := w.pkKey(row) + if _, exists := w.rows[key]; !exists { + w.pkOrder = append(w.pkOrder, key) + // Capture PK values for the keyset cursor. Done only on first-seen keys + // so lastUniquePK always reflects pkOrder[len-1], not a later duplicate. + pkVals := make([]any, len(w.pks)) + for i, pk := range w.pks { + pkVals[i] = normalizePKVal(row[pk]) + } + w.lastUniquePK = pkVals + } + w.rows[key] = row +} + +// Evict removes the row matching pk if the CDC event's CSN falls within +// (LowCSN, HighCSN] and the schema/table match. Returns true if evicted. +// The CDC copy is authoritative; the snapshot copy is discarded. +func (w *DeduplicationWindow) Evict(schema, table string, rowData map[string]any, csn CSN) bool { + if schema != w.Schema || table != w.Table { + return false + } + // Half-open interval: csn must be strictly after LowCSN and at-or-before HighCSN. + // When HighCSN is null (newly-registered table with no committed txns yet), + // there is no upper bound — evict any event after LowCSN so that snapshot + // rows are not duplicated by the CDC stream. + if !csn.Greater(w.LowCSN) || (!w.HighCSN.IsNull() && csn.Greater(w.HighCSN)) { + return false + } + w.mu.Lock() + defer w.mu.Unlock() + key := w.pkKey(rowData) + if _, ok := w.rows[key]; ok { + delete(w.rows, key) + return true + } + return false +} + +// Len returns the number of rows surviving in the window (added minus evicted). +// After Flush() the rows map is cleared and Len returns 0, but LastPKValues() +// still returns the last PK for keyset pagination. +func (w *DeduplicationWindow) Len() int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.rows) +} + +// Empty reports whether no rows were added to the window, meaning the chunk +// SELECT returned zero rows and the table has been fully snapshotted. +func (w *DeduplicationWindow) Empty() bool { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.pkOrder) == 0 +} + +// LastPKValues returns the primary-key values of the last (highest-PK) row +// added to the window, in PK column order. Returns nil if the window is empty. +// Used as the keyset cursor for the next chunk's WHERE pk > lastKey predicate. +// The values are valid even after Flush() because lastUniquePK is not cleared. +func (w *DeduplicationWindow) LastPKValues() []any { + w.mu.Lock() + defer w.mu.Unlock() + return w.lastUniquePK +} + +// Flush returns surviving rows as OpTypeRead ChangeEvents and signals any Wait callers. +// Calling Flush more than once returns an empty slice after the first call. +func (w *DeduplicationWindow) Flush() []ChangeEvent { + w.mu.Lock() + events := make([]ChangeEvent, 0, len(w.rows)) + for _, key := range w.pkOrder { + row, ok := w.rows[key] + if !ok { + continue + } + events = append(events, ChangeEvent{ + Schema: w.Schema, + Table: w.Table, + Operation: OpTypeRead, + CSN: NullCSN(), + Data: row, + PKColumns: w.pks, + }) + } + clear(w.rows) + // Preserve pkOrder so LastPKValues() can still return the last-row PK after + // Flush() — RunTable uses this as a keyset-pagination cursor for the next chunk. + // Signal under the lock so no Evict() can interleave between "state committed" + // and "flushed channel closed", which would corrupt pkOrder seen by Wait() callers. + w.flushedOnce.Do(func() { close(w.flushed) }) + w.mu.Unlock() + return events +} + +// Wait blocks until Flush has been called or ctx is cancelled. +// flushed is given priority over ctx: if both are ready simultaneously, a +// closed flushed channel means the rows were already emitted and the caller +// should advance the checkpoint rather than treating the flush as a loss. +func (w *DeduplicationWindow) Wait(ctx context.Context) error { + select { + case <-w.flushed: + return nil + case <-ctx.Done(): + // Re-check flushed: if both fired simultaneously prefer the flush result + // so the engine advances LastEmittedPK and avoids duplicate rows on resume. + select { + case <-w.flushed: + return nil + default: + } + return ctx.Err() + } +} + +// ---- ChunkReader ------------------------------------------------------------ + +// ChunkReader reads one chunk of rows from a source table, bracketed by CSN watermarks. +type ChunkReader struct { + db *sql.DB + cdcSchema string + schema string + table string + pks []string + batchSize int + // Pre-computed SQL strings avoid per-chunk fmt.Sprintf / strings.Builder overhead. + synchpointQuery string + queryNoBounds string // SELECT … ORDER BY … FETCH FIRST N + queryLowerOnly string // SELECT … WHERE pk > (?) ORDER BY … FETCH FIRST N + queryUpperOnly string // SELECT … WHERE pk <= (?) ORDER BY … FETCH FIRST N + queryBothBounds string // SELECT … WHERE pk > (?) AND pk <= (?) ORDER BY … FETCH FIRST N + // cachedCols caches the column names after the first chunk to avoid calling + // rows.Columns() (which invokes SQLDescribeCol via CGo) on every chunk. + cachedCols []string +} + +// NewChunkReader creates a ChunkReader for one table. +// All identifier arguments (cdcSchema, schema, table, pks) are embedded in SQL +// format strings; callers are responsible for validating them with +// isValidDB2Identifier before constructing a ChunkReader. +func NewChunkReader(db *sql.DB, cdcSchema, schema, table string, pks []string, batchSize int) *ChunkReader { + quotedSchema := QuoteDB2Identifier(schema) + quotedTable := QuoteDB2Identifier(table) + quotedPKs := make([]string, len(pks)) + for i, pk := range pks { + quotedPKs[i] = QuoteDB2Identifier(pk) + } + + base := fmt.Sprintf("SELECT * FROM %s.%s", quotedSchema, quotedTable) + orderSuffix := fmt.Sprintf(" ORDER BY %s FETCH FIRST %d ROWS ONLY", + strings.Join(quotedPKs, ", "), batchSize) + + pkBound := func(op string) string { + if len(quotedPKs) == 1 { + return quotedPKs[0] + " " + op + " ?" + } + pkList := strings.Join(quotedPKs, ", ") + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(quotedPKs)), ", ") + return fmt.Sprintf("(%s) %s (%s)", pkList, op, placeholders) + } + + lowerClause := pkBound(">") + upperClause := pkBound("<=") + + return &ChunkReader{ + db: db, + cdcSchema: cdcSchema, + schema: schema, + table: table, + pks: pks, + batchSize: batchSize, + // Prefer SYNCHPOINT (capture-daemon high-water mark) and fall back to + // CD_NEW_SYNCHPOINT (registration watermark) only when SYNCHPOINT is NULL + // for newly-added tables. Using MAX(CD_NEW_SYNCHPOINT, SYNCHPOINT) can produce + // a watermark ahead of where the streamer has processed, causing Wait() to + // block indefinitely on tables with no recent traffic. + // Filter by both SOURCE_OWNER and SOURCE_TABLE so that a high-watermark from + // a different table in the same schema does not inflate lowCSN for this table, + // which would cause the deduplication window to evict snapshot rows that the + // CDC stream has not yet delivered. + // Use ? parameters for SOURCE_OWNER/SOURCE_TABLE (string-column predicates); + // cdcSchema is a validated identifier and is safe to embed directly. + synchpointQuery: fmt.Sprintf(` + SELECT COALESCE(MAX(SYNCHPOINT), MAX(CD_NEW_SYNCHPOINT)) + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER = ? + AND SOURCE_TABLE = ?`, + cdcSchema, + ), + queryNoBounds: base + orderSuffix, + queryLowerOnly: base + " WHERE " + lowerClause + orderSuffix, + queryUpperOnly: base + " WHERE " + upperClause + orderSuffix, + queryBothBounds: base + " WHERE " + lowerClause + " AND " + upperClause + orderSuffix, + } +} + +// ReadChunk reads one chunk starting after lastKey (nil = first chunk) and +// bounded above by maxKey (nil = no upper bound). maxKey must be the value +// returned by selectMaxPK at snapshot start so the chunk loop terminates even +// when rows are inserted into the table during snapshotting. +// Returns a DeduplicationWindow ready for Streamer-driven eviction. +func (r *ChunkReader) ReadChunk(ctx context.Context, streamCSN CSN, lastKey, maxKey []any) (*DeduplicationWindow, error) { + // Capture lowCSN = MAX(SYNCHPOINT) just before the chunk SELECT. + lowCSN, err := r.captureMaxSynchpoint(ctx) + if err != nil { + return nil, fmt.Errorf("capturing low watermark: %w", err) + } + // Use the higher of the stream's current CSN and the DB watermark as the + // effective lower bound so any CDC events since the last stream poll are + // included in the window. + if streamCSN.Greater(lowCSN) { + lowCSN = streamCSN + } + + rowData, err := r.selectChunk(ctx, lastKey, maxKey) + if err != nil { + return nil, fmt.Errorf("reading chunk: %w", err) + } + + // Capture highCSN = MAX(SYNCHPOINT) just after the chunk SELECT. + highCSN, err := r.captureMaxSynchpoint(ctx) + if err != nil { + return nil, fmt.Errorf("capturing high watermark: %w", err) + } + + // Guard against an inverted window bracket: if the CDC stream has already + // advanced past the post-SELECT DB watermark (e.g. asncap wrote the commit + // record to IBMSNAP_REGISTER AFTER our highCSN read), clamp highCSN to + // lowCSN. The resulting bracket (lowCSN, lowCSN] is empty, so no CDC + // events are evicted — but the streamer has already passed highCSN and will + // flush the window immediately, emitting all snapshot rows correctly. + if highCSN.Less(lowCSN) { + highCSN = lowCSN + } + + win := NewDeduplicationWindow(r.schema, r.table, lowCSN, highCSN, r.pks) + win.pkOrder = make([]string, 0, len(rowData)) // pre-size to exact chunk length + for _, rowMap := range rowData { + win.AddRow(rowMap) + } + return win, nil +} + +// captureMaxSynchpoint queries the highest committed CDC watermark from +// IBMSNAP_REGISTER for the source schema. It unions both SYNCHPOINT +// (updated after each captured transaction) and CD_NEW_SYNCHPOINT (set at +// ADDTABLE time) so that newly-registered tables with a null SYNCHPOINT still +// return a non-null watermark, matching the logic in Streamer.getUpperBound. +func (r *ChunkReader) captureMaxSynchpoint(ctx context.Context) (CSN, error) { + var synchBytes []byte + if err := r.db.QueryRowContext(ctx, r.synchpointQuery, r.schema, r.table).Scan(&synchBytes); err != nil { + if errors.Is(err, sql.ErrNoRows) { + // Table not yet registered in IBMSNAP_REGISTER — return NullCSN so + // the caller treats this as "no watermark" rather than a fatal error. + return NullCSN(), nil + } + return CSN{}, fmt.Errorf("querying SYNCHPOINT watermark: %w", err) + } + if len(synchBytes) == 0 { + return NullCSN(), nil + } + return NewCSNFromDBValue(synchBytes), nil +} + +// selectChunk issues a keyset-paginated SELECT for the chunk. +// lastKey is the exclusive lower bound (nil = start from beginning). +// maxKey is the inclusive upper bound captured at snapshot start (nil = unbounded). +// Returns rows as pre-built column→value maps, eliminating the []any→map +// conversion that ReadChunk would otherwise do on every row. +func (r *ChunkReader) selectChunk(ctx context.Context, lastKey, maxKey []any) ([]map[string]any, error) { + // Select the pre-built query string based on which bounds are present, + // avoiding per-chunk string allocation. + var query string + var args []any + switch { + case len(lastKey) > 0 && len(maxKey) > 0: + query = r.queryBothBounds + args = make([]any, 0, len(lastKey)+len(maxKey)) + args = append(args, lastKey...) + args = append(args, maxKey...) + case len(lastKey) > 0: + query = r.queryLowerOnly + args = lastKey + case len(maxKey) > 0: + // Upper bound from snapshot-start maxPK prevents chasing rows inserted + // during snapshotting (would cause an infinite chunk loop on active tables). + query = r.queryUpperOnly + args = maxKey + default: + query = r.queryNoBounds + } + + rows, err := r.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + // Cache column names after the first chunk to avoid per-chunk SQLDescribeCol + // round-trips. Invalidate if the table schema changed (e.g. ADDCOLUMN). + freshCols, err := rows.Columns() + if err != nil { + return nil, err + } + if r.cachedCols == nil || len(r.cachedCols) != len(freshCols) { + r.cachedCols = freshCols + } + cols := r.cachedCols + + // scanDest is a shared buffer reused across rows: Scan fills it, then we + // copy values into a new map before the next rows.Next() call overwrites it. + // This halves per-row allocations vs allocating a fresh []any per row. + scanDest := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range cols { + ptrs[i] = &scanDest[i] + } + + result := make([]map[string]any, 0, r.batchSize) + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + return nil, err + } + rowMap := make(map[string]any, len(cols)) + for i, col := range cols { + rowMap[col] = scanDest[i] + } + result = append(result, rowMap) + } + return result, rows.Err() +} + +// ---- Logger ----------------------------------------------------------------- + +// Logger is the minimal logging interface used by IncrementalSnapshotEngine to +// emit progress messages and warnings. It is satisfied by *service.Logger +// (Redpanda Connect's logging type) and by the noopLogger stub used in tests. +type Logger interface { + Infof(format string, args ...any) + Warnf(format string, args ...any) +} + +type noopLogger struct{} + +func (noopLogger) Infof(string, ...any) {} +func (noopLogger) Warnf(string, ...any) {} + +// ---- IncrementalSnapshotEngine ---------------------------------------------- + +// IncrementalSnapshotEngine drives the open-window chunk loop for one table at +// a time. It coordinates with a running Streamer via DeduplicationWindow: for +// each chunk it captures a low/high CSN watermark, hands the window to the +// Streamer for in-flight CDC event eviction, and waits for the Streamer to pass +// the highCSN before flushing surviving snapshot rows as OpTypeRead events. +// Concurrency: RunTable blocks the calling goroutine until one table is complete. +// The Streamer runs concurrently on a separate goroutine. +type IncrementalSnapshotEngine struct { + db *sql.DB + cdcSchema string + chunkSize int + streamer *Streamer + checkpointFn func(ctx context.Context, ictx *IncrementalSnapshotContext) error + logger Logger +} + +// NewIncrementalSnapshotEngine creates an IncrementalSnapshotEngine that uses db +// to read source table rows, cdcSchema to query IBMSNAP_REGISTER for watermarks, +// chunkSize as the FETCH FIRST N ROWS ONLY limit per chunk, streamer to install +// DeduplicationWindows, checkpointFn to persist progress after each chunk, and +// logger to emit per-chunk progress (pass nil for no-op logging). +func NewIncrementalSnapshotEngine( + db *sql.DB, + cdcSchema string, + chunkSize int, + streamer *Streamer, + checkpointFn func(ctx context.Context, ictx *IncrementalSnapshotContext) error, + logger Logger, +) *IncrementalSnapshotEngine { + if logger == nil { + logger = noopLogger{} + } + return &IncrementalSnapshotEngine{ + db: db, + cdcSchema: cdcSchema, + chunkSize: chunkSize, + streamer: streamer, + checkpointFn: checkpointFn, + logger: logger, + } +} + +// RunTable snapshots one table using the open-window algorithm. +// It calls checkpointFn after each successful chunk so a crash resumes from +// the last fully-emitted primary key. +// Returns nil when the table is fully snapshotted. +func (e *IncrementalSnapshotEngine) RunTable( + ctx context.Context, + ictx *IncrementalSnapshotContext, + table IncrementalTable, + streamCSN CSN, +) error { + // Defense-in-depth: validate identifiers at the engine boundary so no + // SQL injection is possible even if the caller skips upstream validation. + if !isValidDB2IdentifierInternal(table.Schema) || !isValidDB2IdentifierInternal(table.Name) { + return fmt.Errorf("unsafe table identifier %q.%q: must be non-empty uppercase alphanumeric+underscore", table.Schema, table.Name) + } + + tableKey := table.Schema + "." + table.Name + + pks, err := e.discoverPrimaryKeys(ctx, table.Schema, table.Name) + if err != nil { + return fmt.Errorf("discovering primary keys for %s.%s: %w", table.Schema, table.Name, err) + } + for _, pk := range pks { + if !isValidDB2IdentifierInternal(pk) { + return fmt.Errorf("catalog returned unsafe PK column name %q for %s.%s", pk, table.Schema, table.Name) + } + } + + if err := e.checkPKNotNull(ctx, table.Schema, table.Name, pks); err != nil { + return fmt.Errorf("primary key nullability check for %s.%s: %w", table.Schema, table.Name, err) + } + + // Discover and persist max PK only on the first chunk (not on resume). + if _, ok := ictx.MaxPK[tableKey]; !ok { + maxPK, err := e.selectMaxPK(ctx, table.Schema, table.Name, pks) + if err != nil { + return fmt.Errorf("selecting max PK for %s.%s: %w", table.Schema, table.Name, err) + } + if maxPK == nil { + // Table is empty at snapshot start — nothing to snapshot. + // Store an empty (non-nil) slice as a sentinel so a checkpoint resume + // does not re-run selectMaxPK and re-enter the loop. + ictx.MaxPK[tableKey] = []any{} + e.logger.Infof("incremental snapshot: table=%q.%q is empty, skipping", table.Schema, table.Name) + return nil + } + ictx.MaxPK[tableKey] = maxPK + } + + // An empty-sentinel MaxPK means the table was empty at snapshot start. + if len(ictx.MaxPK[tableKey]) == 0 { + return nil + } + + cr := NewChunkReader(e.db, e.cdcSchema, table.Schema, table.Name, pks, e.chunkSize) + + for { + if err := ctx.Err(); err != nil { + return err + } + + lastKey := ictx.LastEmittedPK[tableKey] + maxKey := ictx.MaxPK[tableKey] + win, err := cr.ReadChunk(ctx, streamCSN, lastKey, maxKey) + if err != nil { + return fmt.Errorf("reading chunk for %s.%s: %w", table.Schema, table.Name, err) + } + + e.logger.Infof("incremental snapshot chunk: table=%q.%q lastKeyLen=%d csnWindow=[%s,%s]", + table.Schema, table.Name, len(ictx.LastEmittedPK[tableKey]), win.LowCSN, win.HighCSN) + + if win.Empty() { + // No more rows — table fully snapshotted. + return nil + } + + rowsInChunk := win.Len() + + // Hand the window to the Streamer; CDC events arriving in-window will + // evict rows before they reach the handler. The Streamer flushes the + // window once it has processed all events up to HighCSN. + e.streamer.SetWindow(win) + + // Block until the Streamer has flushed the window (passed HighCSN). + if err := win.Wait(ctx); err != nil { + // Remove the stale window reference so CDC events on the next + // snapshot round do not incorrectly evict rows from a new window. + e.streamer.ClearWindow(win) + return err + } + + e.logger.Infof("incremental snapshot chunk: table=%q.%q chunkRows=%d csnLow=%s csnHigh=%s", + table.Schema, table.Name, rowsInChunk, win.LowCSN, win.HighCSN) + + // Advance the resume position to the last row emitted by this chunk. + if lastPK := win.LastPKValues(); lastPK != nil { + ictx.LastEmittedPK[tableKey] = lastPK + } + // Only advance streamCSN if the window produced a real upper bound; + // a null HighCSN (newly-registered table with no committed transactions) + // must not overwrite an already-established floor — doing so would cause + // the next chunk's LowCSN to regress, widening the eviction window and + // risking duplicate/out-of-order delivery. + if !win.HighCSN.IsNull() && win.HighCSN.Greater(streamCSN) { + streamCSN = win.HighCSN + } + + if err := e.checkpointFn(ctx, ictx); err != nil { + return fmt.Errorf("checkpointing after chunk: %w", err) + } + } +} + +// discoverPrimaryKeys queries SYSCAT.KEYCOLUSE for the PK column list. +func (e *IncrementalSnapshotEngine) discoverPrimaryKeys(ctx context.Context, schema, table string) ([]string, error) { + query := ` + SELECT k.COLNAME + FROM SYSCAT.KEYCOLUSE k + JOIN SYSCAT.TABCONST c + ON c.CONSTNAME = k.CONSTNAME + AND c.TABSCHEMA = k.TABSCHEMA + AND c.TABNAME = k.TABNAME + WHERE c.TYPE = 'P' + AND k.TABSCHEMA = ? + AND k.TABNAME = ? + ORDER BY k.COLSEQ` + + rows, err := e.db.QueryContext(ctx, query, schema, table) + if err != nil { + return nil, fmt.Errorf("querying primary keys: %w", err) + } + defer rows.Close() + + var pks []string + for rows.Next() { + var col string + if err := rows.Scan(&col); err != nil { + return nil, err + } + pks = append(pks, strings.TrimSpace(col)) + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(pks) == 0 { + return nil, fmt.Errorf("table %q.%q has no primary key", schema, table) + } + return pks, nil +} + +// checkPKNotNull verifies that all pk columns are NOT NULL in SYSCAT.COLUMNS. +// A nullable PK makes WHERE pk > lastKey undefined (NULL comparisons are UNKNOWN in DB2), +// which would silently skip rows and break exactly-once delivery guarantees. +func (e *IncrementalSnapshotEngine) checkPKNotNull(ctx context.Context, schema, table string, pks []string) error { + if len(pks) == 0 { + return nil + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(pks)), ",") + query := fmt.Sprintf(` + SELECT COLNAME, NULLS + FROM SYSCAT.COLUMNS + WHERE TABSCHEMA = ? + AND TABNAME = ? + AND COLNAME IN (%s)`, placeholders) + + args := make([]any, 0, 2+len(pks)) + args = append(args, schema, table) + for _, pk := range pks { + args = append(args, pk) + } + + rows, err := e.db.QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("querying column nullability: %w", err) + } + defer rows.Close() + + var nullable []string + for rows.Next() { + var colName, nulls string + if err := rows.Scan(&colName, &nulls); err != nil { + return err + } + if strings.TrimSpace(nulls) == "Y" { + nullable = append(nullable, strings.TrimSpace(colName)) + } + } + if err := rows.Err(); err != nil { + return err + } + if len(nullable) > 0 { + return fmt.Errorf("table %q.%q has nullable primary key column(s) %v: keyset pagination requires NOT NULL PKs", + schema, table, nullable) + } + return nil +} + +// selectMaxPK queries the maximum primary key tuple for the snapshot upper bound. +// +// For compound PKs, per-column MAX() produces a synthetic tuple (MAX(a), MAX(b)) +// that may not correspond to any real row and can be lexicographically greater than +// any real row. For example, rows (1,100),(2,50) yield MAX(a)=2, MAX(b)=100 → (2,100), +// which exceeds the real max (2,50) and allows rows inserted AFTER highCSN to slip +// through the upper-bound filter without being evicted. To avoid this, we select the +// lexicographically largest row directly using an ORDER BY … DESC FETCH FIRST 1. +func (e *IncrementalSnapshotEngine) selectMaxPK(ctx context.Context, schema, table string, pks []string) ([]any, error) { + quotedSchema := QuoteDB2Identifier(schema) + quotedTable := QuoteDB2Identifier(table) + quotedPKs := make([]string, len(pks)) + for i, pk := range pks { + quotedPKs[i] = QuoteDB2Identifier(pk) + } + // Select only the PK columns in descending order to get the lexicographic maximum. + // Each quotedPK already carries its own DESC direction; the format string must + // not add a trailing DESC (that would double-up on single-PK tables). + query := fmt.Sprintf( + "SELECT %s FROM %s.%s ORDER BY %s FETCH FIRST 1 ROW ONLY", + strings.Join(quotedPKs, ", "), + quotedSchema, quotedTable, + strings.Join(quotedPKs, " DESC, ")+" DESC", + ) + row := e.db.QueryRowContext(ctx, query) + vals := make([]any, len(pks)) + ptrs := make([]any, len(pks)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := row.Scan(ptrs...); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("selecting max PK for %s.%s: %w", schema, table, err) + } + // Normalize []byte → string so that VARCHAR/CHAR max-PK values survive + // JSON round-trip in the checkpoint store (json.Marshal encodes []byte as + // base64, which would not compare correctly as a bound parameter). + for i, v := range vals { + vals[i] = normalizePKVal(v) + } + return vals, nil +} diff --git a/internal/impl/db2/replication/incremental_test.go b/internal/impl/db2/replication/incremental_test.go new file mode 100644 index 0000000000..2725d7f9ed --- /dev/null +++ b/internal/impl/db2/replication/incremental_test.go @@ -0,0 +1,729 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "context" + "database/sql/driver" + "encoding/binary" + "encoding/json" + "fmt" + "runtime" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// incTestCSNBytes converts a uint64 into the 8-byte big-endian representation +// that NewCSNFromDBValue will parse back to the same uint64 value. +func incTestCSNBytes(val uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, val) + return b +} + +// --------------------------------------------------------------------------- +// Task 1: IncrementalSnapshotContext +// --------------------------------------------------------------------------- + +// jsonRoundTrip mirrors the actual save/load path used by saveIncrementalContext +// and loadIncrementalContext: json.Marshal to save, json.Decoder.UseNumber to +// load (preserving BIGINT PKs as json.Number instead of float64). +func jsonRoundTrip(t *testing.T, src *IncrementalSnapshotContext) *IncrementalSnapshotContext { + t.Helper() + b, err := json.Marshal(src) + require.NoError(t, err) + var dst IncrementalSnapshotContext + dec := json.NewDecoder(strings.NewReader(string(b))) + dec.UseNumber() + require.NoError(t, dec.Decode(&dst)) + if dst.LastEmittedPK == nil { + dst.LastEmittedPK = map[string][]any{} + } + if dst.MaxPK == nil { + dst.MaxPK = map[string][]any{} + } + return &dst +} + +func TestIncrementalSnapshotContextRoundTrip(t *testing.T) { + t.Parallel() + + ctx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T", SignalID: "sig1"}}, + LastEmittedPK: map[string][]any{"S.T": {json.Number("42")}}, + MaxPK: map[string][]any{"S.T": {json.Number("1000")}}, + } + got := jsonRoundTrip(t, ctx) + assert.Equal(t, ctx.Tables, got.Tables) + assert.Equal(t, ctx.LastEmittedPK, got.LastEmittedPK) + assert.Equal(t, ctx.MaxPK, got.MaxPK) +} + +func TestIncrementalSnapshotContextEmpty(t *testing.T) { + t.Parallel() + + got := jsonRoundTrip(t, &IncrementalSnapshotContext{}) + assert.False(t, got.Running()) + assert.Nil(t, got.CurrentTable()) +} + +func TestIncrementalSnapshotContextRunning(t *testing.T) { + t.Parallel() + + ctx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{ + {Schema: "S", Name: "T1", SignalID: "s1"}, + {Schema: "S", Name: "T2", SignalID: "s2"}, + }, + } + assert.True(t, ctx.Running()) + assert.Equal(t, &ctx.Tables[0], ctx.CurrentTable()) + + ctx.AdvanceTable() + assert.True(t, ctx.Running()) + assert.Equal(t, "T2", ctx.CurrentTable().Name) + + ctx.AdvanceTable() + assert.False(t, ctx.Running()) + assert.Nil(t, ctx.CurrentTable()) +} + +// --------------------------------------------------------------------------- +// Task 2: DeduplicationWindow +// --------------------------------------------------------------------------- + +func TestDeduplicationWindowNoOverlap(t *testing.T) { + t.Parallel() + + w := NewDeduplicationWindow("S", "T", NewCSN(100), NewCSN(200), []string{"ID"}) + w.AddRow(map[string]any{"ID": int64(1), "VAL": "a"}) + w.AddRow(map[string]any{"ID": int64(2), "VAL": "b"}) + + events := w.Flush() + assert.Len(t, events, 2) + for _, e := range events { + assert.Equal(t, OpTypeRead, e.Operation) + assert.Equal(t, "S", e.Schema) + assert.Equal(t, "T", e.Table) + } +} + +func TestDeduplicationWindowCDCEvicts(t *testing.T) { + t.Parallel() + + w := NewDeduplicationWindow("S", "T", NewCSN(100), NewCSN(200), []string{"ID"}) + w.AddRow(map[string]any{"ID": int64(1), "VAL": "old"}) + w.AddRow(map[string]any{"ID": int64(2), "VAL": "b"}) + + // CDC event for ID=1 within the window evicts the snapshot copy. + evicted := w.Evict("S", "T", map[string]any{"ID": int64(1), "VAL": "new"}, NewCSN(150)) + assert.True(t, evicted) + + events := w.Flush() + require.Len(t, events, 1) + assert.Equal(t, int64(2), events[0].Data["ID"]) +} + +func TestDeduplicationWindowOutOfRangeCSNNotEvicted(t *testing.T) { + t.Parallel() + + w := NewDeduplicationWindow("S", "T", NewCSN(100), NewCSN(200), []string{"ID"}) + w.AddRow(map[string]any{"ID": int64(1), "VAL": "snap"}) + + // CDC event BEFORE lowCSN — already committed before snapshot opened. + notEvicted := w.Evict("S", "T", map[string]any{"ID": int64(1)}, NewCSN(50)) + assert.False(t, notEvicted) + + // CDC event AFTER highCSN — happened after snapshot closed. + notEvicted2 := w.Evict("S", "T", map[string]any{"ID": int64(1)}, NewCSN(300)) + assert.False(t, notEvicted2) + + // Exactly at lowCSN — boundary is exclusive on low side. + notEvicted3 := w.Evict("S", "T", map[string]any{"ID": int64(1)}, NewCSN(100)) + assert.False(t, notEvicted3) + + // Exactly at highCSN — boundary is inclusive on high side. + w2 := NewDeduplicationWindow("S", "T", NewCSN(100), NewCSN(200), []string{"ID"}) + w2.AddRow(map[string]any{"ID": int64(2)}) + evictedAtHigh := w2.Evict("S", "T", map[string]any{"ID": int64(2)}, NewCSN(200)) + assert.True(t, evictedAtHigh, "highCSN boundary must be inclusive") + events2 := w2.Flush() + assert.Empty(t, events2, "evicted row must not appear in flush") + + events := w.Flush() + assert.Len(t, events, 1) +} + +func TestDeduplicationWindowWrongTable(t *testing.T) { + t.Parallel() + + w := NewDeduplicationWindow("S", "T", NewCSN(100), NewCSN(200), []string{"ID"}) + w.AddRow(map[string]any{"ID": int64(1)}) + + notEvicted := w.Evict("S", "OTHER", map[string]any{"ID": int64(1)}, NewCSN(150)) + assert.False(t, notEvicted) + + assert.Len(t, w.Flush(), 1) +} + +func TestDeduplicationWindowEmpty(t *testing.T) { + t.Parallel() + + w := NewDeduplicationWindow("S", "T", NewCSN(0), NewCSN(100), []string{"ID"}) + assert.True(t, w.Empty()) + + w.AddRow(map[string]any{"ID": int64(1)}) + assert.False(t, w.Empty()) +} + +func TestDeduplicationWindowLastPKValues(t *testing.T) { + t.Parallel() + + w := NewDeduplicationWindow("S", "T", NewCSN(0), NewCSN(100), []string{"ID"}) + assert.Nil(t, w.LastPKValues()) + + w.AddRow(map[string]any{"ID": int64(10)}) + w.AddRow(map[string]any{"ID": int64(20)}) + pkvs := w.LastPKValues() + require.Len(t, pkvs, 1) + // LastPKValues returns the direct row values (no JSON round-trip), so the + // type is exactly what was stored via AddRow. + assert.Equal(t, int64(20), pkvs[0]) +} + +// TestDeduplicationWindowByteVsStringPK verifies that Evict correctly matches a +// snapshot row whose PK was stored as []byte (DB2 ODBC returns CHAR/VARCHAR as +// []byte) against a CDC event with the same PK stored as string (after +// convertDB2Value). Without normalizePKVal this cross-type match always fails, +// causing duplicate delivery for every VARCHAR-PK table. +func TestDeduplicationWindowByteVsStringPK(t *testing.T) { + t.Parallel() + + w := NewDeduplicationWindow("S", "T", NewCSN(100), NewCSN(200), []string{"ID"}) + // Snapshot row: DB2 ODBC driver returns VARCHAR as []byte. + w.AddRow(map[string]any{"ID": []byte("FOO")}) + require.Equal(t, 1, w.Len()) + + // CDC eviction event: convertDB2Value has converted the same column to string. + evicted := w.Evict("S", "T", map[string]any{"ID": "FOO"}, NewCSN(150)) + assert.True(t, evicted, "Evict must match []byte snapshot PK against string CDC PK") + assert.Equal(t, 0, w.Len(), "window must have 0 surviving rows after eviction") + + // LastPKValues must return string (not []byte) so JSON checkpoint round-trips correctly. + pkvs := w.LastPKValues() + require.Len(t, pkvs, 1) + _, isString := pkvs[0].(string) + assert.True(t, isString, "LastPKValues must return string, not []byte, for varchar PKs") + assert.Equal(t, "FOO", pkvs[0]) +} + +// --------------------------------------------------------------------------- +// Task 3: ChunkReader +// --------------------------------------------------------------------------- + +func TestChunkReaderFirstChunk(t *testing.T) { + t.Parallel() + + queryCount := atomic.Int32{} + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + n := queryCount.Add(1) + switch n { + case 1: // captureMaxSynchpoint (lowCSN = 100) + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(100))}}, nil + case 2: // chunk SELECT + return []string{"ID", "VAL"}, [][]driver.Value{ + {driver.Value(int64(1)), driver.Value("a")}, + {driver.Value(int64(2)), driver.Value("b")}, + }, nil + case 3: // captureMaxSynchpoint (highCSN = 102) + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(102))}}, nil + default: + return nil, nil, nil + } + }, + }) + + cr := NewChunkReader(db, "ASNCDC", "S", "T", []string{"ID"}, 1000) + win, err := cr.ReadChunk(context.Background(), NewCSN(90), nil, nil) + require.NoError(t, err) + assert.Equal(t, uint64(100), win.LowCSN.Uint64()) + assert.Equal(t, uint64(102), win.HighCSN.Uint64()) + events := win.Flush() + assert.Len(t, events, 2) +} + +func TestChunkReaderResumesFromLastKey(t *testing.T) { + t.Parallel() + + var capturedQuery string + queryCount := atomic.Int32{} + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + n := queryCount.Add(1) + switch n { + case 1: + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(200))}}, nil + case 2: + capturedQuery = q + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(43))}}, nil + case 3: + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(201))}}, nil + default: + return nil, nil, nil + } + }, + }) + + cr := NewChunkReader(db, "ASNCDC", "S", "T", []string{"ID"}, 1000) + _, err := cr.ReadChunk(context.Background(), NewCSN(190), []any{int64(42)}, nil) + require.NoError(t, err) + // Verify that the chunk query contains a WHERE clause for keyset pagination. + assert.True(t, strings.Contains(capturedQuery, "WHERE"), "chunk query should contain WHERE clause, got: %s", capturedQuery) +} + +func TestChunkReaderEmptyResult(t *testing.T) { + t.Parallel() + + queryCount := atomic.Int32{} + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + n := queryCount.Add(1) + switch n { + case 1: + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(300))}}, nil + case 2: + return []string{"ID"}, nil, nil // empty chunk + case 3: + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(300))}}, nil + default: + return nil, nil, nil + } + }, + }) + + cr := NewChunkReader(db, "ASNCDC", "S", "T", []string{"ID"}, 1000) + win, err := cr.ReadChunk(context.Background(), NewCSN(290), []any{int64(99)}, nil) + require.NoError(t, err) + assert.True(t, win.Empty()) +} + +// --------------------------------------------------------------------------- +// Task 5: IncrementalSnapshotEngine +// --------------------------------------------------------------------------- + +func TestEngineRunsSingleTableAllChunks(t *testing.T) { + t.Parallel() + + // 3 rows, chunkSize=2 → chunk1=[1,2], chunk2=[3], chunk3=empty → done. + // Engine must emit all 3 snapshot events and call checkpoint twice. + + chunkCall := atomic.Int32{} + streamer := NewStreamer(nil, StreamConfig{}, Version{}) + checkpointCalled := atomic.Int32{} + var allEvents []ChangeEvent + + db := openFakeDB(t, &replFakeHandlers{ + query: func(query string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(query, "KEYCOLUSE") { + return []string{"COLNAME"}, [][]driver.Value{{driver.Value("ID")}}, nil + } + if strings.Contains(query, "SYSCAT.COLUMNS") { + return []string{"COLNAME", "NULLS"}, [][]driver.Value{{driver.Value("ID"), driver.Value("N")}}, nil + } + if strings.Contains(query, "FETCH FIRST 1 ROW ONLY") { + // selectMaxPK: return the lexicographic max row for the table. + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(3))}}, nil + } + if strings.Contains(query, "SYNCHPOINT") { + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(500))}}, nil + } + n := chunkCall.Add(1) + switch n { + case 1: + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(1))}, {driver.Value(int64(2))}}, nil + case 2: + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(3))}}, nil + default: + return []string{"ID"}, nil, nil + } + }, + }) + + engine := NewIncrementalSnapshotEngine(db, "ASNCDC", 2, streamer, func(_ context.Context, _ *IncrementalSnapshotContext) error { + checkpointCalled.Add(1) + return nil + }, nil) + + ictx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "ORDERS", SignalID: "sig1"}}, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + } + table := *ictx.CurrentTable() + + errCh := make(chan error, 1) + go func() { + errCh <- engine.RunTable(context.Background(), ictx, table, NewCSN(0)) + }() + + // Flush windows until the engine finishes. A window flush unblocks the engine + // goroutine; when RunTable returns, errCh fires. + flushUntilDone := func() error { + for { + select { + case err := <-errCh: + return err + default: + } + streamer.windowMu.Lock() + win := streamer.activeWindow + streamer.windowMu.Unlock() + if win != nil { + allEvents = append(allEvents, win.Flush()...) + streamer.windowMu.Lock() + streamer.activeWindow = nil + streamer.windowMu.Unlock() + } else { + runtime.Gosched() + } + } + } + + require.NoError(t, flushUntilDone()) + assert.Len(t, allEvents, 3) + assert.Equal(t, int32(2), checkpointCalled.Load()) +} + +func TestEngineResumesFromCheckpoint(t *testing.T) { + t.Parallel() + + // LastEmittedPK = [1] → engine starts with chunk after ID=1, gets rows [2,3], then empty. + chunkCall := atomic.Int32{} + + streamer := NewStreamer(nil, StreamConfig{}, Version{}) + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "KEYCOLUSE") { + return []string{"COLNAME"}, [][]driver.Value{{driver.Value("ID")}}, nil + } + if strings.Contains(q, "SYSCAT.COLUMNS") { + return []string{"COLNAME", "NULLS"}, [][]driver.Value{{driver.Value("ID"), driver.Value("N")}}, nil + } + if strings.Contains(q, "FETCH FIRST 1 ROW ONLY") { + // selectMaxPK: return the lexicographic max row for the table. + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(3))}}, nil + } + if strings.Contains(q, "SYNCHPOINT") { + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(600))}}, nil + } + n := chunkCall.Add(1) + switch n { + case 1: + return []string{"ID"}, [][]driver.Value{ + {driver.Value(int64(2))}, + {driver.Value(int64(3))}, + }, nil + default: + return []string{"ID"}, nil, nil + } + }, + }) + + engine := NewIncrementalSnapshotEngine(db, "ASNCDC", 100, streamer, func(_ context.Context, _ *IncrementalSnapshotContext) error { + return nil + }, nil) + + ictx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T", SignalID: "sig1"}}, + LastEmittedPK: map[string][]any{"S.T": {json.Number("1")}}, + MaxPK: map[string][]any{"S.T": {json.Number("3")}}, + } + + table := *ictx.CurrentTable() + + var allEvents []ChangeEvent + errCh := make(chan error, 1) + go func() { + errCh <- engine.RunTable(context.Background(), ictx, table, NewCSN(0)) + }() + + for { + select { + case err := <-errCh: + require.NoError(t, err) + assert.Len(t, allEvents, 2) + return + default: + } + streamer.windowMu.Lock() + win := streamer.activeWindow + streamer.windowMu.Unlock() + if win != nil { + allEvents = append(allEvents, win.Flush()...) + streamer.windowMu.Lock() + streamer.activeWindow = nil + streamer.windowMu.Unlock() + } else { + runtime.Gosched() + } + } +} + +func TestEngineContextCancellation(t *testing.T) { + t.Parallel() + + streamer := NewStreamer(nil, StreamConfig{}, Version{}) + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "KEYCOLUSE") { + return []string{"COLNAME"}, [][]driver.Value{{driver.Value("ID")}}, nil + } + if strings.Contains(q, "SYSCAT.COLUMNS") { + return []string{"COLNAME", "NULLS"}, [][]driver.Value{{driver.Value("ID"), driver.Value("N")}}, nil + } + if strings.Contains(q, "FETCH FIRST 1 ROW ONLY") { + // selectMaxPK: return the lexicographic max row for the table. + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(100))}}, nil + } + if strings.Contains(q, "SYNCHPOINT") { + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(700))}}, nil + } + // Return a non-empty chunk so the engine sets a window and blocks on Wait. + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(1))}}, nil + }, + }) + + engine := NewIncrementalSnapshotEngine(db, "ASNCDC", 100, streamer, func(_ context.Context, _ *IncrementalSnapshotContext) error { + return nil + }, nil) + + ictx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T", SignalID: "sig1"}}, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + } + + ctx, cancel := context.WithCancel(context.Background()) + + errCh := make(chan error, 1) + go func() { + errCh <- engine.RunTable(ctx, ictx, *ictx.CurrentTable(), NewCSN(0)) + }() + + // Wait for engine to set window (it is now blocked on win.Wait). + for { + streamer.windowMu.Lock() + win := streamer.activeWindow + streamer.windowMu.Unlock() + if win != nil { + break + } + runtime.Gosched() + } + + // Cancel the context — engine should unblock and return ctx.Err(). + cancel() + + err := <-errCh + assert.ErrorIs(t, err, context.Canceled) +} + +func TestDeduplicationWindowLen(t *testing.T) { + t.Parallel() + win := NewDeduplicationWindow("S", "T", NullCSN(), NullCSN(), []string{"ID"}) + require.Equal(t, 0, win.Len()) + win.AddRow(map[string]any{"ID": 1}) + win.AddRow(map[string]any{"ID": 2}) + require.Equal(t, 2, win.Len()) + win.AddRow(map[string]any{"ID": 1}) // duplicate key — overwrites, no new entry + require.Equal(t, 2, win.Len()) +} + +type testLogger struct{ lines []string } + +func (l *testLogger) Infof(f string, a ...any) { l.lines = append(l.lines, fmt.Sprintf(f, a...)) } +func (l *testLogger) Warnf(f string, a ...any) { l.lines = append(l.lines, fmt.Sprintf(f, a...)) } + +func TestEngineSkipsEmptyTable(t *testing.T) { + t.Parallel() + + log := &testLogger{} + + // empty-table scenario: selectMaxPK returns no rows → engine logs "empty, skipping" + // and returns nil without issuing any SYNCHPOINT or chunk SELECT queries. + chunkQueryCount := atomic.Int32{} + db := openFakeDB(t, &replFakeHandlers{ + query: func(query string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(query, "KEYCOLUSE") { + return []string{"COLNAME"}, [][]driver.Value{{driver.Value("ID")}}, nil + } + if strings.Contains(query, "SYSCAT.COLUMNS") { + return []string{"COLNAME", "NULLS"}, [][]driver.Value{{driver.Value("ID"), driver.Value("N")}}, nil + } + if strings.Contains(query, "FETCH FIRST 1 ROW ONLY") { + // selectMaxPK: empty table returns no rows. + return []string{"ID"}, nil, nil + } + // Any other query (SYNCHPOINT or chunk SELECT) is unexpected. + chunkQueryCount.Add(1) + return []string{"ID"}, nil, nil + }, + }) + + streamer := NewStreamer(nil, StreamConfig{}, Version{}) + ictx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T"}}, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + } + + engine := NewIncrementalSnapshotEngine(db, "ASNCDC", 10, streamer, + func(context.Context, *IncrementalSnapshotContext) error { return nil }, + log, + ) + + err := engine.RunTable(context.Background(), ictx, ictx.Tables[0], NullCSN()) + require.NoError(t, err) + assert.Equal(t, int32(0), chunkQueryCount.Load(), "no SYNCHPOINT or chunk queries should be issued for an empty table") + + require.NotEmpty(t, log.lines, "expected a log line about the empty table") + combined := strings.Join(log.lines, "\n") + require.Contains(t, combined, "empty", "expected 'empty' in log output for an empty table snapshot") +} + +func TestEngineLogsChunkProgress(t *testing.T) { + t.Parallel() + + log := &testLogger{} + var chunkCall atomic.Int32 + + streamer := NewStreamer(nil, StreamConfig{}, Version{}) + db := openFakeDB(t, &replFakeHandlers{ + query: func(query string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(query, "KEYCOLUSE") { + return []string{"COLNAME"}, [][]driver.Value{{driver.Value("ID")}}, nil + } + if strings.Contains(query, "SYSCAT.COLUMNS") { + return []string{"COLNAME", "NULLS"}, [][]driver.Value{{driver.Value("ID"), driver.Value("N")}}, nil + } + if strings.Contains(query, "FETCH FIRST 1 ROW ONLY") { + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(2))}}, nil + } + if strings.Contains(query, "SYNCHPOINT") { + return []string{"1"}, [][]driver.Value{{driver.Value(incTestCSNBytes(100))}}, nil + } + n := chunkCall.Add(1) + if n == 1 { + return []string{"ID"}, [][]driver.Value{{driver.Value(int64(1))}, {driver.Value(int64(2))}}, nil + } + return []string{"ID"}, nil, nil + }, + }) + + ictx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T"}}, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + } + + engine := NewIncrementalSnapshotEngine(db, "ASNCDC", 2, streamer, + func(context.Context, *IncrementalSnapshotContext) error { return nil }, + log, + ) + + errCh := make(chan error, 1) + go func() { errCh <- engine.RunTable(context.Background(), ictx, ictx.Tables[0], NullCSN()) }() + + // Flush windows until the engine finishes. + for { + streamer.windowMu.Lock() + w := streamer.activeWindow + streamer.windowMu.Unlock() + if w != nil { + w.Flush() + streamer.windowMu.Lock() + if streamer.activeWindow == w { + streamer.activeWindow = nil + } + streamer.windowMu.Unlock() + } + select { + case err := <-errCh: + require.NoError(t, err) + goto done + default: + } + } +done: + require.NotEmpty(t, log.lines, "expected at least one log line, got none") + combined := strings.Join(log.lines, "\n") + require.Contains(t, combined, "chunk", "expected chunk-level progress log") +} + +func TestRunTableFailsOnNullablePK(t *testing.T) { + t.Parallel() + + // Engine must fail fast when SYSCAT.COLUMNS reports NULLS='Y' for the PK column. + queryCount := atomic.Int32{} + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + n := queryCount.Add(1) + switch n { + case 1: // discoverPrimaryKeys + return []string{"COLNAME"}, [][]driver.Value{{driver.Value("ID")}}, nil + case 2: // checkPKNotNull — nullable + return []string{"COLNAME", "NULLS"}, [][]driver.Value{ + {driver.Value("ID"), driver.Value("Y")}, + }, nil + default: + return nil, nil, nil + } + }, + }) + + streamer := NewStreamer(nil, StreamConfig{}, Version{}) + engine := NewIncrementalSnapshotEngine(db, "ASNCDC", 10, streamer, + func(context.Context, *IncrementalSnapshotContext) error { return nil }, nil) + ictx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T"}}, + LastEmittedPK: map[string][]any{}, + MaxPK: map[string][]any{}, + } + err := engine.RunTable(context.Background(), ictx, ictx.Tables[0], NullCSN()) + require.Error(t, err) + assert.Contains(t, err.Error(), "nullable") +} + +func TestIncrementalContextBIGINTRoundtrip(t *testing.T) { + t.Parallel() + + // 2^53 + 1 = 9007199254740993 — cannot be represented exactly as float64. + // Verify that the json.Marshal + json.Decoder.UseNumber checkpoint path + // preserves the full integer value (matching loadIncrementalContext). + const bigintStr = "9007199254740993" + ctx := &IncrementalSnapshotContext{ + Tables: []IncrementalTable{{Schema: "S", Name: "T"}}, + LastEmittedPK: map[string][]any{"S.T": {json.Number(bigintStr)}}, + MaxPK: map[string][]any{"S.T": {json.Number(bigintStr)}}, + } + got := jsonRoundTrip(t, ctx) + + pk := got.LastEmittedPK["S.T"] + require.Len(t, pk, 1) + num, ok := pk[0].(json.Number) + require.True(t, ok, "expected json.Number, got %T(%v)", pk[0], pk[0]) + assert.Equal(t, bigintStr, num.String()) +} diff --git a/internal/impl/db2/replication/pairing_test.go b/internal/impl/db2/replication/pairing_test.go new file mode 100644 index 0000000000..2b86817650 --- /dev/null +++ b/internal/impl/db2/replication/pairing_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPairOpcodeEvents(t *testing.T) { + csn100 := NewCSN(100) + csn101 := NewCSN(101) + + tests := []struct { + name string + input []ChangeEvent + expect []ChangeEvent + }{ + { + name: "update_pair_merged", + input: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: opTypeUpdateBefore, Data: map[string]any{"ID": 1, "NAME": "old"}}, + {CSN: csn100, IntentSeq: 2, Operation: opTypeUpdateAfter, Data: map[string]any{"ID": 1, "NAME": "new"}}, + }, + expect: []ChangeEvent{ + { + CSN: csn100, IntentSeq: 2, Operation: OpTypeUpdate, + Data: map[string]any{"ID": 1, "NAME": "new"}, + BeforeData: map[string]any{"ID": 1, "NAME": "old"}, + }, + }, + }, + { + name: "standalone_delete", + input: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: OpTypeDelete, Data: map[string]any{"ID": 2}}, + }, + expect: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: OpTypeDelete, Data: map[string]any{"ID": 2}}, + }, + }, + { + name: "standalone_insert", + input: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: OpTypeInsert, Data: map[string]any{"ID": 3}}, + }, + expect: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: OpTypeInsert, Data: map[string]any{"ID": 3}}, + }, + }, + { + name: "delete_then_update_pair", + input: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: OpTypeDelete, Data: map[string]any{"ID": 1}}, + {CSN: csn100, IntentSeq: 2, Operation: opTypeUpdateBefore, Data: map[string]any{"ID": 2, "NAME": "old"}}, + {CSN: csn100, IntentSeq: 3, Operation: opTypeUpdateAfter, Data: map[string]any{"ID": 2, "NAME": "new"}}, + }, + expect: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: OpTypeDelete, Data: map[string]any{"ID": 1}}, + { + CSN: csn100, IntentSeq: 3, Operation: OpTypeUpdate, + Data: map[string]any{"ID": 2, "NAME": "new"}, + BeforeData: map[string]any{"ID": 2, "NAME": "old"}, + }, + }, + }, + { + name: "two_update_pairs_same_csn", + input: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: opTypeUpdateBefore, Data: map[string]any{"ID": 1, "V": "a"}}, + {CSN: csn100, IntentSeq: 2, Operation: opTypeUpdateAfter, Data: map[string]any{"ID": 1, "V": "b"}}, + {CSN: csn100, IntentSeq: 3, Operation: opTypeUpdateBefore, Data: map[string]any{"ID": 2, "V": "x"}}, + {CSN: csn100, IntentSeq: 4, Operation: opTypeUpdateAfter, Data: map[string]any{"ID": 2, "V": "y"}}, + }, + expect: []ChangeEvent{ + { + CSN: csn100, IntentSeq: 2, Operation: OpTypeUpdate, + Data: map[string]any{"ID": 1, "V": "b"}, BeforeData: map[string]any{"ID": 1, "V": "a"}, + }, + { + CSN: csn100, IntentSeq: 4, Operation: OpTypeUpdate, + Data: map[string]any{"ID": 2, "V": "y"}, BeforeData: map[string]any{"ID": 2, "V": "x"}, + }, + }, + }, + { + // Cross-batch D+I pairs are handled upstream: pollChangeTable strips the trailing + // opTypeUpdateBefore into pendingBeforeByTable. pairOpcodeEvents only sees complete + // pairs. If an unmatched before-image arrives here (different CSNs, a DB2 anomaly + // that shouldn't happen in production), the before-image is discarded and the + // orphaned after-image is re-classified as an INSERT so no internal opcode leaks + // downstream. Consumers see the current row value rather than an unknown opcode. + name: "mismatched_csn_orphaned_after_becomes_insert", + input: []ChangeEvent{ + {CSN: csn100, IntentSeq: 1, Operation: opTypeUpdateBefore, Data: map[string]any{"ID": 1}}, + {CSN: csn101, IntentSeq: 2, Operation: opTypeUpdateAfter, Data: map[string]any{"ID": 1}}, + }, + expect: []ChangeEvent{ + // Before-image discarded; after-image re-classified as INSERT. + {CSN: csn101, IntentSeq: 2, Operation: OpTypeInsert, Data: map[string]any{"ID": 1}}, + }, + }, + { + name: "empty", + input: []ChangeEvent{}, + expect: []ChangeEvent{}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := pairOpcodeEvents(tc.input) + assert.Equal(t, tc.expect, got) + }) + } +} diff --git a/internal/impl/db2/replication/snapshot.go b/internal/impl/db2/replication/snapshot.go new file mode 100644 index 0000000000..1eeb29b362 --- /dev/null +++ b/internal/impl/db2/replication/snapshot.go @@ -0,0 +1,509 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2cli" +) + +// SnapshotConfig holds configuration for the initial full-table snapshot phase. +// The snapshot reads all rows from each configured table inside a single +// read-only serializable (DB2 RR) transaction, ensuring a consistent view. +type SnapshotConfig struct { + // Schemas is the list of DB2 source schemas (TABSCHEMA) whose tables are snapshotted. + // Must have at least one element; each entry must be a valid DB2 identifier + // (uppercase alphanumeric+underscore). For single-schema operation pass a one-element slice. + Schemas []string + + // Tables is the list of table names (without schema prefix) to snapshot. + // If empty, the snapshot is a no-op. + Tables []string + + // AsnCDCSchema is the CDC control schema that owns IBMSNAP_REGISTER. + // Defaults to "ASNCDC" if empty. Used to capture the starting CSN watermark + // from ASNCDC.IBMSNAP_REGISTER before reading the first row. + // + // IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=tables-ibmsnap-register + AsnCDCSchema string + + // BatchSize is the number of rows fetched per keyset-paginated query. + // Reducing BatchSize lowers memory usage; increasing it improves throughput. + BatchSize int + + // TableFilter narrows which tables in Tables are snapshotted. When nil all + // tables in Tables are included. This allows table_include_regex / + // table_exclude_regex from the connector config to apply to snapshot tables. + TableFilter func(string) bool + + // Parallelism is the number of tables to snapshot concurrently. + // 0 or 1 means sequential (default). Maximum is 16. + // Note: all goroutines share the same read-only transaction, so this is safe + // for correctness but each goroutine holds its own statement handle. + Parallelism int + + // IsolationLevel is the SQL isolation level for the snapshot transaction. + // Defaults to sql.LevelSerializable (DB2 "RR" — Repeatable Read, no phantoms). + // sql.LevelReadCommitted (DB2 "CS") lowers lock overhead but does not guarantee + // snapshot consistency across multiple tables. + IsolationLevel sql.IsolationLevel +} + +// effectiveIsolationLevel returns the configured isolation level, defaulting to +// sql.LevelSerializable (DB2 RR) when the field is zero. +func (c *SnapshotConfig) effectiveIsolationLevel() sql.IsolationLevel { + if c.IsolationLevel == 0 { + return sql.LevelSerializable + } + return c.IsolationLevel +} + +// schemasINClause returns a SQL IN clause fragment like "'S1', 'S2'". +// Each schema name is validated before embedding so that callers that bypass +// Snapshot() cannot inject SQL via an unvalidated Schemas value. +func (c *SnapshotConfig) schemasINClause() string { + quoted := make([]string, 0, len(c.Schemas)) + for _, s := range c.Schemas { + if !isValidDB2IdentifierInternal(s) { + continue + } + quoted = append(quoted, "'"+s+"'") + } + return strings.Join(quoted, ", ") +} + +// asncdcSchema returns the CDC control schema, defaulting to "ASNCDC". +func (c *SnapshotConfig) asncdcSchema() string { + if c.AsnCDCSchema != "" { + return c.AsnCDCSchema + } + return "ASNCDC" +} + +// Snapshotter performs the initial full-table read of all monitored tables +// before the CDC streaming phase begins. +// +// All tables are read inside a single read-only REPEATABLE READ transaction so +// that rows from different tables are consistent with each other. The CDC log +// position (CSN) is captured from ASNCDC.IBMSNAP_REGISTER *before* the first +// row is read; streaming resumes from this CSN, ensuring no changes are missed. +// +// Large tables are paginated using keyset pagination (ORDER BY pk / WHERE pk > +// lastKey FETCH FIRST N ROWS ONLY), which is efficient regardless of table size +// and avoids OFFSET-based scans. +type Snapshotter struct { + db *sql.DB + config SnapshotConfig + version Version +} + +// NewSnapshotter creates a Snapshotter that will use db to read rows, config +// to determine which tables to snapshot and what isolation level to use, and +// version to apply any DB2-version-specific behaviour. +func NewSnapshotter(db *sql.DB, config SnapshotConfig, version Version) *Snapshotter { + return &Snapshotter{ + db: db, + config: config, + version: version, + } +} + +// Snapshot reads all rows from each configured table and returns the CSN that +// streaming should resume from (captured before the first table is read). +func (s *Snapshotter) Snapshot(ctx context.Context, handler func(event ChangeEvent) error) (CSN, error) { + for _, schema := range s.config.Schemas { + if !isValidDB2IdentifierInternal(schema) { + return CSN{}, fmt.Errorf("invalid schema %q: must be non-empty uppercase alphanumeric+underscore", schema) + } + } + + // sql.LevelSerializable maps to DB2 "RR" (Repeatable Read — no phantoms). + // sql.LevelRepeatableRead maps to DB2 "RS" (Read Stability — phantoms allowed), + // which is weaker than intended. Use Serializable for correct snapshot isolation. + // Full fix (SET CURRENT ISOLATION RR per statement) is tracked separately. + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: s.config.effectiveIsolationLevel(), + ReadOnly: true, + }) + if err != nil { + return CSN{}, fmt.Errorf("beginning snapshot transaction: %w", err) + } + // Note: this transaction holds the sole connection in db's pool (SetMaxOpenConns(1)). + // Any concurrent caller (schema-change polling, signal processing) blocks until this + // transaction commits. Full fix requires a separate db handle for snapshot queries. + defer tx.Rollback() //nolint:errcheck + + startCSN, err := s.captureCurrentCSN(ctx, tx) + if err != nil { + return CSN{}, fmt.Errorf("capturing starting CSN: %w", err) + } + + if err = s.snapshotTablesSequential(ctx, tx, handler); err != nil { + return CSN{}, fmt.Errorf("snapshot failed: %w", err) + } + + if err := tx.Commit(); err != nil { + return CSN{}, fmt.Errorf("committing snapshot transaction: %w", err) + } + + return startCSN, nil +} + +// captureCurrentCSN reads MAX(SYNCHPOINT) from ASNCDC.IBMSNAP_REGISTER for the +// monitored schema. SYNCHPOINT is the CDC log position the capture daemon has +// confirmed writing; streaming will resume from this value after the snapshot. +// +// SYNCHPOINT (not CURRENT TIMESTAMP or a DB2 log-position function) is used +// because it reflects the capture daemon's confirmed write position — the same +// value that change-table poll queries compare against. +// +// For freshly registered tables SYNCHPOINT may be NULL (the capture daemon has not +// yet processed any transactions). CD_NEW_SYNCHPOINT is always populated at +// ASNCDC.ADDTABLE time, so the UNION over both columns mirrors getUpperBound in +// stream.go and guarantees a non-null watermark even for brand-new registrations. +func (s *Snapshotter) captureCurrentCSN(ctx context.Context, tx *sql.Tx) (CSN, error) { + cdcSchema := s.config.asncdcSchema() + schemasIN := s.config.schemasINClause() + + // Schema names are validated as uppercase alphanumeric + underscore at config-parse + // time (isValidDB2Identifier in input_db2_cdc.go), so direct embedding is safe. + query := fmt.Sprintf(` + SELECT MAX(t.SYNCHPOINT) FROM ( + SELECT CD_NEW_SYNCHPOINT AS SYNCHPOINT + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER IN (%s) + UNION ALL + SELECT SYNCHPOINT + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER IN (%s) + ) t`, + cdcSchema, schemasIN, + cdcSchema, schemasIN, + ) + + var synchpointRaw any + err := tx.QueryRowContext(ctx, query).Scan(&synchpointRaw) + if err != nil { + if errors.Is(err, sql.ErrNoRows) || isObjectNotFoundError(err) { + // CDC tables not yet set up — return a null CSN so the caller + // treats this as "no watermark" rather than "watermark at 0". + return NullCSN(), nil + } + return CSN{}, fmt.Errorf("capturing CSN from IBMSNAP_REGISTER: %w", err) + } + + if synchpointRaw == nil { + return NullCSN(), nil + } + + // SYNCHPOINT is VARCHAR(16) FOR BIT DATA; the DB2 CLI driver may return + // it as a hex-encoded string — getBitDataBytes hex-decodes it if needed. + synchpointBytes := getBitDataBytes(&synchpointRaw) + if len(synchpointBytes) == 0 { + return NullCSN(), nil + } + + return NewCSNFromDBValue(synchpointBytes), nil +} + +// isObjectNotFoundError returns true when err indicates that a table, view, or +// other named object does not exist in the catalog. Checks the structured +// *db2cli.DB2Error SQLSTATE first (exact match, locale-independent); falls back +// to substring matching for errors that have been wrapped into plain strings. +// +// Relevant SQLSTATE codes: +// +// 42704 — DB2 "undefined name" (object not in catalog) +// 42S02 — ODBC "base table or view not found" +func isObjectNotFoundError(err error) bool { + if db2Err, ok := errors.AsType[*db2cli.DB2Error](err); ok { + return db2Err.SQLState == "42704" || db2Err.SQLState == db2cli.SQLSTATE_TABLE_OR_VIEW_NOT_FOUND + } + // Fallback: match the DB2 error string format "DB2 Error [42704] ..." for + // heavily-wrapped errors where AsType cannot unwrap to *db2cli.DB2Error. + // We match only the bracketed SQLSTATE form rather than free English phrases + // ("does not exist", "not found") to avoid swallowing unrelated errors that + // happen to contain those words in their message text. + msg := err.Error() + return strings.Contains(msg, "[42704]") || strings.Contains(msg, "[42S02]") +} + +// snapshotTablesSequential snapshots tables one at a time in the order they +// appear in config.Tables. Parallel snapshotting is not supported because +// sharing a *sql.Tx across goroutines is a data race on the ODBC handle; +// maxSnapshotParallelism enforces this at the config layer. +func (s *Snapshotter) snapshotTablesSequential(ctx context.Context, tx *sql.Tx, handler func(event ChangeEvent) error) error { + for _, schema := range s.config.Schemas { + for _, t := range s.config.Tables { + if s.config.TableFilter != nil && !s.config.TableFilter(t) { + continue + } + if err := s.snapshotTable(ctx, tx, schema, t, handler); err != nil { + return fmt.Errorf("snapshotting table %s.%s: %w", schema, t, err) + } + } + } + return nil +} + +// snapshotTable reads all rows of one table using keyset pagination. +func (s *Snapshotter) snapshotTable(ctx context.Context, tx *sql.Tx, schema, tableName string, handler func(event ChangeEvent) error) error { + pks, err := s.discoverPrimaryKeys(ctx, tx, schema, tableName) + if err != nil { + return fmt.Errorf("discovering primary keys: %w", err) + } + + if len(pks) == 0 { + return fmt.Errorf("table %s.%s has no primary key (required for CDC snapshot)", schema, tableName) + } + for _, pk := range pks { + if !isValidDB2IdentifierInternal(pk) { + return fmt.Errorf("catalog returned unsafe PK column name %q for %s.%s", pk, schema, tableName) + } + } + + columns, err := s.getTableColumns(ctx, tx, schema, tableName) + if err != nil { + return fmt.Errorf("getting table columns: %w", err) + } + for _, col := range columns { + if !isValidDB2IdentifierInternal(col) { + return fmt.Errorf("catalog returned unsafe column name %q for %s.%s", col, schema, tableName) + } + } + + // Pre-compute per-table SQL queries once; avoids repeated quoting/joining across all batch iterations. + quotedCols := make([]string, len(columns)) + for i, col := range columns { + quotedCols[i] = QuoteDB2Identifier(col) + } + quotedPKs := make([]string, len(pks)) + for i, pk := range pks { + quotedPKs[i] = QuoteDB2Identifier(pk) + } + orderByClause := strings.Join(quotedPKs, ", ") + baseSelect := fmt.Sprintf(`SELECT %s FROM %s.%s`, + strings.Join(quotedCols, ", "), + QuoteDB2Identifier(schema), + QuoteDB2Identifier(tableName), + ) + suffix := fmt.Sprintf(" ORDER BY %s FETCH FIRST %d ROWS ONLY", orderByClause, s.config.BatchSize) + queryNoBounds := baseSelect + suffix + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(pks)), ", ") + queryWithBounds := baseSelect + fmt.Sprintf(" WHERE (%s) > (%s)", orderByClause, placeholders) + suffix + + lastKeyValues := make([]any, len(pks)) // allocated once; reused across batch iterations + var hasLastKey bool + var colKindCache []db2ColumnKind // nil on first batch; populated from ColumnTypes() and reused + + for { + var batchKeyArg []any + if hasLastKey { + batchKeyArg = lastKeyValues + } + batchRows, err := s.fetchBatch(ctx, tx, queryNoBounds, queryWithBounds, columns, batchKeyArg, &colKindCache) + if err != nil { + return fmt.Errorf("fetching batch: %w", err) + } + + if len(batchRows) == 0 { + break + } + + for _, row := range batchRows { + event := ChangeEvent{ + Schema: schema, + Table: tableName, + Operation: OpTypeRead, + CSN: NullCSN(), + Data: row, + PKColumns: pks, + } + + if err := handler(event); err != nil { + return fmt.Errorf("handler error: %w", err) + } + } + + lastRow := batchRows[len(batchRows)-1] + for i, pk := range pks { + v := lastRow[pk] + if v == nil { + // NULL in a PK column would produce (pk > NULL) in the next batch's + // WHERE clause, which DB2 evaluates as UNKNOWN — returning zero rows + // and silently truncating the snapshot mid-table. + return fmt.Errorf("snapshot keyset cursor: NULL value in PK column %q for %s.%s; snapshot aborted to prevent row loss", pk, schema, tableName) + } + lastKeyValues[i] = v + } + hasLastKey = true + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + } + + return nil +} + +// discoverPrimaryKeys returns the primary key column names for a table, ordered by sequence. +func (*Snapshotter) discoverPrimaryKeys(ctx context.Context, tx *sql.Tx, schema, tableName string) ([]string, error) { + query := ` + SELECT COLNAME + FROM SYSCAT.KEYCOLUSE + WHERE TABSCHEMA = ? + AND TABNAME = ? + AND CONSTNAME = ( + SELECT CONSTNAME + FROM SYSCAT.TABCONST + WHERE TABSCHEMA = ? + AND TABNAME = ? + AND TYPE = 'P' + ) + ORDER BY COLSEQ + ` + + rows, err := tx.QueryContext(ctx, query, schema, tableName, schema, tableName) + if err != nil { + return nil, fmt.Errorf("querying primary keys: %w", err) + } + defer rows.Close() + + var pks []string + for rows.Next() { + var colName string + if err := rows.Scan(&colName); err != nil { + return nil, err + } + pks = append(pks, strings.TrimSpace(colName)) + } + + return pks, rows.Err() +} + +// getTableColumns returns all column names for a table, in column order. +func (*Snapshotter) getTableColumns(ctx context.Context, tx *sql.Tx, schema, tableName string) ([]string, error) { + query := ` + SELECT COLNAME + FROM SYSCAT.COLUMNS + WHERE TABSCHEMA = ? + AND TABNAME = ? + ORDER BY COLNO + ` + + rows, err := tx.QueryContext(ctx, query, schema, tableName) + if err != nil { + return nil, fmt.Errorf("querying columns: %w", err) + } + defer rows.Close() + + var columns []string + for rows.Next() { + var colName string + if err := rows.Scan(&colName); err != nil { + return nil, err + } + columns = append(columns, strings.TrimSpace(colName)) + } + + if len(columns) == 0 { + return nil, fmt.Errorf("table %s.%s has no columns", schema, tableName) + } + + return columns, rows.Err() +} + +// fetchBatch fetches one page of rows using keyset pagination. +// queryNoBounds and queryWithBounds must be pre-computed by the caller (snapshotTable) +// so this hot-path function avoids per-batch string allocation. +// colKindCache is a write-through pointer: nil on the first call, populated after the +// first call and reused on all subsequent calls so SQLDescribeCol is only invoked once per table. +func (s *Snapshotter) fetchBatch(ctx context.Context, tx *sql.Tx, queryNoBounds, queryWithBounds string, columns []string, lastKeyValues []any, colKindCache *[]db2ColumnKind) ([]map[string]any, error) { + var query string + var args []any + if len(lastKeyValues) > 0 { + query = queryWithBounds + args = lastKeyValues + } else { + query = queryNoBounds + } + + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("executing batch query: %w", err) + } + defer rows.Close() + + // colKinds: computed once on the first batch via SQLDescribeCol and cached for + // all subsequent pages of the same table (column schema is stable across pages). + colKinds := *colKindCache + if colKinds == nil { + columnTypes, err := rows.ColumnTypes() + if err != nil { + return nil, fmt.Errorf("getting column types: %w", err) + } + colKinds = make([]db2ColumnKind, len(columnTypes)) + for i, ct := range columnTypes { + colKinds[i] = columnKind(ct.DatabaseTypeName()) + } + *colKindCache = colKinds + } + + result := make([]map[string]any, 0, s.config.BatchSize) + + // Reuse scanDest and scanPtrs across rows to avoid one allocation per column per row. + scanDest := make([]any, len(columns)) + scanPtrs := make([]any, len(columns)) + for i := range scanDest { + scanPtrs[i] = &scanDest[i] + } + + for rows.Next() { + // Clear previous row values before scanning (avoids stale data on nil columns). + clear(scanDest) + + if err := rows.Scan(scanPtrs...); err != nil { + return nil, fmt.Errorf("scanning row: %w", err) + } + + rowMap := make(map[string]any, len(columns)) + for i, col := range columns { + rowMap[col] = convertDB2Value(scanDest[i], colKinds[i]) + } + + result = append(result, rowMap) + } + + return result, rows.Err() +} + +// convertDB2Value converts a DB2 driver value to a JSON-serializable Go type. +// kind must be pre-computed from the column's DatabaseTypeName() before the row loop. +func convertDB2Value(value any, kind db2ColumnKind) any { + if value == nil { + return nil + } + + if b, ok := value.([]byte); ok { + if kind == db2ColumnKindText { + return string(b) + } + // Binary / unknown types: return raw bytes (JSON-marshalled as base64). + return b + } + + return value +} diff --git a/internal/impl/db2/replication/snapshot_test.go b/internal/impl/db2/replication/snapshot_test.go new file mode 100644 index 0000000000..93574dccbf --- /dev/null +++ b/internal/impl/db2/replication/snapshot_test.go @@ -0,0 +1,918 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "context" + "database/sql/driver" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2cli" +) + +// --------------------------------------------------------------------------- +// Pure-function tests (no DB) +// --------------------------------------------------------------------------- + +func TestConvertDB2Value(t *testing.T) { + t.Parallel() + + // Use the fake driver with colTypes so DatabaseTypeName returns the right values. + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{"NAME", "DATA", "NUM"}, [][]driver.Value{ + {[]byte("Alice"), []byte{0xDE, 0xAD}, int64(42)}, + }, nil + }, + colTypes: func(_ string) map[string]string { + return map[string]string{ + "NAME": "VARCHAR", + "DATA": "BLOB", + "NUM": "INTEGER", + } + }, + }) + + tx, err := db.Begin() + require.NoError(t, err) + defer tx.Rollback() //nolint:errcheck + + rows, err := tx.QueryContext(context.Background(), "SELECT NAME, DATA, NUM FROM T") + require.NoError(t, err) + defer func() { require.NoError(t, rows.Close()); require.NoError(t, rows.Err()) }() + + colTypes, err := rows.ColumnTypes() + require.NoError(t, err) + require.Len(t, colTypes, 3) + + // Derive column kind for each column type (VARCHAR→Text, BLOB→Binary, INTEGER→Other). + // []byte with VARCHAR kind → string + assert.Equal(t, "Alice", convertDB2Value([]byte("Alice"), columnKind(colTypes[0].DatabaseTypeName()))) + // []byte with BLOB kind → raw bytes + assert.Equal(t, []byte{0xDE, 0xAD}, convertDB2Value([]byte{0xDE, 0xAD}, columnKind(colTypes[1].DatabaseTypeName()))) + // int64 passes through unchanged regardless of kind + assert.Equal(t, int64(42), convertDB2Value(int64(42), columnKind(colTypes[2].DatabaseTypeName()))) + // nil returns nil regardless of kind + assert.Nil(t, convertDB2Value(nil, columnKind(colTypes[0].DatabaseTypeName()))) + // []byte with explicit Text kind → string + assert.Equal(t, "abc", convertDB2Value([]byte("abc"), db2ColumnKindText)) +} + +func TestSnapshotConfigAsncdcSchema(t *testing.T) { + t.Parallel() + + c := &SnapshotConfig{} + assert.Equal(t, "ASNCDC", c.asncdcSchema(), "default schema") + + c.AsnCDCSchema = "CUSTOM" + assert.Equal(t, "CUSTOM", c.asncdcSchema(), "explicit schema") +} + +// --------------------------------------------------------------------------- +// Tests requiring a fake database +// --------------------------------------------------------------------------- + +func TestCaptureCurrentCSN(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + queryFunc func(query string, args []driver.Value) ([]string, [][]driver.Value, error) + wantCSN uint64 + wantErr bool + }{ + { + name: "normal SYNCHPOINT bytes", + queryFunc: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + // 8-byte big-endian value for uint64(12345) = 0x0000000000003039 + b := []byte{0, 0, 0, 0, 0, 0, 0x30, 0x39} + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{b}}, nil + }, + wantCSN: 12345, + }, + { + name: "NULL SYNCHPOINT returns zero CSN", + queryFunc: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + }, + wantCSN: 0, + }, + { + name: "table does not exist returns zero CSN (SQLSTATE 42704)", + queryFunc: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + // Real DB2 driver returns SQLSTATE 42704 (undefined name) when the + // IBMSNAP_REGISTER table has not yet been created (CDC not yet set up). + return nil, nil, &db2cli.DB2Error{SQLState: "42704", NativeError: -204, Message: "IBMSNAP_REGISTER is an undefined name"} + }, + wantCSN: 0, + }, + { + name: "table does not exist returns zero CSN (DB2 Error string format)", + queryFunc: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + // Fallback path: wrapped error where AsType cannot extract DB2Error. + return nil, nil, fmt.Errorf("sql: %w", fmt.Errorf("DB2 Error [42704] (Native: -204): IBMSNAP_REGISTER is an undefined name")) + }, + wantCSN: 0, + }, + { + name: "unexpected error propagates", + queryFunc: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return nil, nil, fmt.Errorf("connection timeout") + }, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{query: tc.queryFunc}) + s := NewSnapshotter(db, SnapshotConfig{Schemas: []string{"MYSCHEMA"}}, Version{}) + + tx, err := db.Begin() + require.NoError(t, err) + defer tx.Rollback() //nolint:errcheck + + csn, err := s.captureCurrentCSN(context.Background(), tx) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantCSN, csn.Uint64()) + }) + } +} + +func TestDiscoverPrimaryKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rows [][]driver.Value + wantPKs []string + wantErr bool + }{ + { + name: "single PK column", + rows: [][]driver.Value{{"ID "}}, // trailing spaces trimmed + wantPKs: []string{"ID"}, + }, + { + name: "composite PK", + rows: [][]driver.Value{{"COL_A"}, {"COL_B"}}, + wantPKs: []string{"COL_A", "COL_B"}, + }, + { + name: "no primary key returns empty slice", + rows: nil, + wantPKs: nil, + }, + { + name: "query error propagates", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if tc.wantErr { + return nil, nil, fmt.Errorf("query error") + } + return []string{"COLNAME"}, tc.rows, nil + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{Schemas: []string{"MYSCHEMA"}}, Version{}) + tx, err := db.Begin() + require.NoError(t, err) + defer tx.Rollback() //nolint:errcheck + + pks, err := s.discoverPrimaryKeys(context.Background(), tx, "MYSCHEMA", "EMPLOYEES") + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantPKs, pks) + }) + } +} + +func TestGetTableColumns(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rows [][]driver.Value + wantCols []string + wantErr bool + }{ + { + name: "normal columns", + rows: [][]driver.Value{{"ID"}, {"NAME "}, {"SALARY"}}, + wantCols: []string{"ID", "NAME", "SALARY"}, + }, + { + // rows is non-nil but empty: query succeeds, zero rows → triggers len(columns)==0 guard + name: "no columns returns error", + rows: [][]driver.Value{}, + wantErr: true, + }, + { + // rows is nil: used as sentinel to simulate a query-level failure + name: "query error propagates", + rows: nil, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if tc.rows == nil { + return nil, nil, fmt.Errorf("query failed") + } + return []string{"COLNAME"}, tc.rows, nil + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{Schemas: []string{"MYSCHEMA"}}, Version{}) + tx, err := db.Begin() + require.NoError(t, err) + defer tx.Rollback() //nolint:errcheck + + cols, err := s.getTableColumns(context.Background(), tx, "MYSCHEMA", "EMPLOYEES") + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantCols, cols) + }) + } +} + +func TestFetchBatch(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + columns []string + pks []string + lastKeyValues []any + rows [][]driver.Value + wantCount int + wantQueryFn func(t *testing.T, query string) + }{ + { + name: "first page (no last key)", + columns: []string{"ID", "NAME"}, + pks: []string{"ID"}, + rows: [][]driver.Value{{int64(1), "Alice"}, {int64(2), "Bob"}}, + wantCount: 2, + wantQueryFn: func(t *testing.T, query string) { + t.Helper() + assert.NotContains(t, query, "WHERE") + }, + }, + { + name: "subsequent page uses WHERE clause", + columns: []string{"ID", "NAME"}, + pks: []string{"ID"}, + lastKeyValues: []any{int64(2)}, + rows: [][]driver.Value{{int64(3), "Charlie"}}, + wantCount: 1, + wantQueryFn: func(t *testing.T, query string) { + t.Helper() + assert.Contains(t, query, "WHERE") + assert.Contains(t, query, "?") + }, + }, + { + name: "empty result set", + columns: []string{"ID"}, + pks: []string{"ID"}, + rows: nil, + wantCount: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var capturedQuery string + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + capturedQuery = q + return tc.columns, tc.rows, nil + }, + }) + + const batchSize = 100 + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + BatchSize: batchSize, + }, Version{}) + + tx, err := db.Begin() + require.NoError(t, err) + defer tx.Rollback() //nolint:errcheck + + // Build the pre-computed SQL strings the same way snapshotTable does. + quotedCols := make([]string, len(tc.columns)) + for i, col := range tc.columns { + quotedCols[i] = QuoteDB2Identifier(col) + } + quotedPKs := make([]string, len(tc.pks)) + for i, pk := range tc.pks { + quotedPKs[i] = QuoteDB2Identifier(pk) + } + orderBy := strings.Join(quotedPKs, ", ") + base := fmt.Sprintf(`SELECT %s FROM %s.%s`, + strings.Join(quotedCols, ", "), + QuoteDB2Identifier("MYSCHEMA"), + QuoteDB2Identifier("EMPLOYEES"), + ) + suffix := fmt.Sprintf(" ORDER BY %s FETCH FIRST %d ROWS ONLY", orderBy, batchSize) + qNoBounds := base + suffix + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(tc.pks)), ", ") + qWithBounds := base + fmt.Sprintf(" WHERE (%s) > (%s)", orderBy, placeholders) + suffix + + var colKindCache []db2ColumnKind + result, err := s.fetchBatch(context.Background(), tx, qNoBounds, qWithBounds, tc.columns, tc.lastKeyValues, &colKindCache) + require.NoError(t, err) + assert.Len(t, result, tc.wantCount) + + if tc.wantQueryFn != nil { + tc.wantQueryFn(t, capturedQuery) + } + }) + } +} + +func TestSnapshotEndToEnd(t *testing.T) { + t.Parallel() + + // Simulate: 3 rows in one table, captured in a single batch. + // Query sequence: + // 1. MAX(SYNCHPOINT) → returns CSN bytes for 12345 + // 2. KEYCOLUSE → returns "ID" + // 3. SYSCAT.COLUMNS → returns "ID", "NAME" + // 4. SELECT * → returns 3 rows + // 5. SELECT * → returns empty (end of pagination) + + queryCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + queryCount++ + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0x30, 0x39}}, // 12345 + }, nil + case strings.Contains(q, "KEYCOLUSE"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}, {"NAME"}}, nil + case strings.Contains(q, "FETCH FIRST"): + // Return data on the first page, empty on the second. + if queryCount == 4 { + return []string{"ID", "NAME"}, [][]driver.Value{ + {int64(1), "Alice"}, + {int64(2), "Bob"}, + {int64(3), "Charlie"}, + }, nil + } + return []string{"ID", "NAME"}, nil, nil + default: + return nil, nil, nil + } + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"EMPLOYEES"}, + BatchSize: 1000, + }, Version{Major: 11, Minor: 5}) + + var events []ChangeEvent + startCSN, err := s.Snapshot(context.Background(), func(e ChangeEvent) error { + events = append(events, e) + return nil + }) + + require.NoError(t, err) + assert.Equal(t, uint64(12345), startCSN.Uint64()) + assert.Len(t, events, 3) + for _, e := range events { + assert.Equal(t, OpTypeRead, e.Operation) + assert.Equal(t, "MYSCHEMA", e.Schema) + assert.Equal(t, "EMPLOYEES", e.Table) + assert.True(t, e.CSN.IsNull()) + } +} + +func TestSnapshotSkipsWhenNoPK(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + case strings.Contains(q, "KEYCOLUSE"): + // No PK columns + return []string{"COLNAME"}, nil, nil + default: + return nil, nil, nil + } + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"NOTKEYED"}, + BatchSize: 100, + }, Version{}) + + _, err := s.Snapshot(context.Background(), func(_ ChangeEvent) error { return nil }) + require.Error(t, err) + assert.Contains(t, err.Error(), "primary key") +} + +func TestSnapshotHandlerError(t *testing.T) { + t.Parallel() + + queryCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + queryCount++ + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + case strings.Contains(q, "KEYCOLUSE"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "FETCH FIRST"): + return []string{"ID"}, [][]driver.Value{{int64(1)}}, nil + default: + return nil, nil, nil + } + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"T"}, + BatchSize: 100, + }, Version{}) + + _, err := s.Snapshot(context.Background(), func(_ ChangeEvent) error { + return fmt.Errorf("handler rejected event") + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "handler rejected event") +} + +func TestSnapshotContextCancellation(t *testing.T) { + t.Parallel() + + queryCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + queryCount++ + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + case strings.Contains(q, "KEYCOLUSE"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "FETCH FIRST"): + return []string{"ID"}, [][]driver.Value{{int64(1)}, {int64(2)}, {int64(3)}}, nil + default: + return nil, nil, nil + } + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"T"}, + BatchSize: 100, + }, Version{}) + + ctx, cancel := context.WithCancel(context.Background()) + + count := 0 + _, err := s.Snapshot(ctx, func(_ ChangeEvent) error { + count++ + if count == 1 { + cancel() // cancel after the first event + } + return ctx.Err() + }) + + require.Error(t, err) +} + +// TestSnapshotMultipleTablesSequential verifies that when multiple tables are +// configured, the Snapshotter reads them one at a time in declaration order and +// emits OpTypeRead events with the correct Schema and Table fields for each. +// Every snapshot event must carry a null CSN since it originates outside the +// CDC log stream. +func TestSnapshotMultipleTablesSequential(t *testing.T) { + t.Parallel() + + // The fake handler dispatches by inspecting the query string. KEYCOLUSE and + // SYSCAT.COLUMNS queries embed the table name as a bind parameter (not in + // the SQL text), so we track which table is currently being snapshotted by + // counting how many times we have seen a FETCH FIRST query (data page queries) + // for each table. + // + // Query sequence for 2 tables (EMPLOYEES then ORDERS): + // SYNCHPOINT → CSN 100 (once, at start of Snapshot) + // KEYCOLUSE (args) → "ID" (EMPLOYEES) + // SYSCAT.COLUMNS → "ID","NAME" (EMPLOYEES) + // FETCH FIRST … → 2 rows (EMPLOYEES page 1) + // FETCH FIRST … → empty (EMPLOYEES page 2, end) + // KEYCOLUSE (args) → "ORDER_ID" (ORDERS) + // SYSCAT.COLUMNS → "ORDER_ID","AMOUNT" (ORDERS) + // FETCH FIRST … → 1 row (ORDERS page 1) + // FETCH FIRST … → empty (ORDERS page 2, end) + + fetchCount := 0 + keyColCount := 0 + colCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 100}}, + }, nil + case strings.Contains(q, "KEYCOLUSE"): + keyColCount++ + if keyColCount == 1 { + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + } + return []string{"COLNAME"}, [][]driver.Value{{"ORDER_ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + colCount++ + if colCount == 1 { + return []string{"COLNAME"}, [][]driver.Value{{"ID"}, {"NAME"}}, nil + } + return []string{"COLNAME"}, [][]driver.Value{{"ORDER_ID"}, {"AMOUNT"}}, nil + case strings.Contains(q, "FETCH FIRST"): + fetchCount++ + switch fetchCount { + case 1: // EMPLOYEES page 1 + return []string{"ID", "NAME"}, [][]driver.Value{ + {int64(1), "Alice"}, + {int64(2), "Bob"}, + }, nil + case 2: // EMPLOYEES page 2 → end of table + return []string{"ID", "NAME"}, nil, nil + case 3: // ORDERS page 1 + return []string{"ORDER_ID", "AMOUNT"}, [][]driver.Value{ + {int64(1001), int64(500)}, + }, nil + default: // ORDERS page 2 → end of table + return []string{"ORDER_ID", "AMOUNT"}, nil, nil + } + } + return nil, nil, nil + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"EMPLOYEES", "ORDERS"}, + BatchSize: 1000, + }, Version{Major: 11, Minor: 5}) + + var events []ChangeEvent + startCSN, err := s.Snapshot(context.Background(), func(e ChangeEvent) error { + events = append(events, e) + return nil + }) + + require.NoError(t, err) + assert.Equal(t, uint64(100), startCSN.Uint64()) + + var empEvents, ordEvents []ChangeEvent + for _, e := range events { + assert.Equal(t, OpTypeRead, e.Operation, "all snapshot events must have operation=read") + assert.Equal(t, "MYSCHEMA", e.Schema) + assert.True(t, e.CSN.IsNull(), "snapshot events must carry a null CSN") + switch e.Table { + case "EMPLOYEES": + empEvents = append(empEvents, e) + case "ORDERS": + ordEvents = append(ordEvents, e) + } + } + + assert.Len(t, empEvents, 2, "EMPLOYEES table must produce 2 snapshot events") + assert.Len(t, ordEvents, 1, "ORDERS table must produce 1 snapshot event") +} + +// TestSnapshotNULLColumnValues verifies that NULL column values in the source +// table are represented as nil in the ChangeEvent Data map. A nil value must +// be present as an explicit key (not absent), so that downstream consumers can +// distinguish "value is NULL" from "column was not selected". +func TestSnapshotNULLColumnValues(t *testing.T) { + t.Parallel() + + queryCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + queryCount++ + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + case strings.Contains(q, "KEYCOLUSE"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}, {"MANAGER_ID"}, {"DEPT"}}, nil + case strings.Contains(q, "FETCH FIRST"): + if queryCount == 4 { + // MANAGER_ID and DEPT are NULL for this employee. + return []string{"ID", "MANAGER_ID", "DEPT"}, [][]driver.Value{ + {int64(1), nil, nil}, + }, nil + } + return []string{"ID", "MANAGER_ID", "DEPT"}, nil, nil + } + return nil, nil, nil + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"EMPLOYEES"}, + BatchSize: 100, + }, Version{}) + + var events []ChangeEvent + _, err := s.Snapshot(context.Background(), func(e ChangeEvent) error { + events = append(events, e) + return nil + }) + + require.NoError(t, err) + require.Len(t, events, 1) + + assert.Equal(t, int64(1), events[0].Data["ID"]) + // NULL columns must appear in the Data map as nil, not as absent keys. + managerVal, managerExists := events[0].Data["MANAGER_ID"] + assert.True(t, managerExists, "MANAGER_ID key must be present even when the column value is NULL") + assert.Nil(t, managerVal, "NULL column value must be nil in the Data map") + + deptVal, deptExists := events[0].Data["DEPT"] + assert.True(t, deptExists, "DEPT key must be present even when the column value is NULL") + assert.Nil(t, deptVal, "NULL column value must be nil in the Data map") +} + +// TestSnapshotCompositePKKeysetPagination verifies that tables with composite +// primary keys are paginated correctly using multi-column keyset pagination. +// The WHERE clause must use tuple comparison: (PK1, PK2) > (lastVal1, lastVal2). +// +// This corresponds to the Debezium "composite PK" scenario where a table with +// two PK columns (e.g. id + region) must advance the page cursor using all PK +// columns simultaneously to avoid re-reading rows. +func TestSnapshotCompositePKKeysetPagination(t *testing.T) { + t.Parallel() + + var capturedFetchQueries []string + queryCount := 0 + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + queryCount++ + if strings.Contains(q, "FETCH FIRST") { + capturedFetchQueries = append(capturedFetchQueries, q) + } + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + case strings.Contains(q, "KEYCOLUSE"): + // Composite PK: (REGION_ID, EMP_ID) — two columns + return []string{"COLNAME"}, [][]driver.Value{{"REGION_ID"}, {"EMP_ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + return []string{"COLNAME"}, [][]driver.Value{ + {"REGION_ID"}, {"EMP_ID"}, {"NAME"}, + }, nil + case strings.Contains(q, "FETCH FIRST"): + switch len(capturedFetchQueries) { + case 1: // first page + return []string{"REGION_ID", "EMP_ID", "NAME"}, [][]driver.Value{ + {int64(1), int64(100), "Alice"}, + {int64(1), int64(101), "Bob"}, + }, nil + case 2: // second page + return []string{"REGION_ID", "EMP_ID", "NAME"}, [][]driver.Value{ + {int64(2), int64(200), "Charlie"}, + }, nil + default: // end of data + return []string{"REGION_ID", "EMP_ID", "NAME"}, nil, nil + } + } + return nil, nil, nil + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"EMPLOYEES"}, + BatchSize: 2, + }, Version{}) + + var events []ChangeEvent + _, err := s.Snapshot(context.Background(), func(e ChangeEvent) error { + events = append(events, e) + return nil + }) + + require.NoError(t, err) + assert.Len(t, events, 3, "must receive all 3 rows across two pages") + + require.Len(t, capturedFetchQueries, 3, "must issue 3 SELECT queries") + + // First page: no WHERE clause + assert.NotContains(t, capturedFetchQueries[0], "WHERE", + "first page must not have a WHERE clause") + + // Second page: composite tuple WHERE clause with both PK columns and placeholders + assert.Contains(t, capturedFetchQueries[1], "WHERE", + "second page must have a keyset WHERE clause") + assert.Contains(t, capturedFetchQueries[1], "REGION_ID", + "second page WHERE must reference the first PK column") + assert.Contains(t, capturedFetchQueries[1], "EMP_ID", + "second page WHERE must reference the second PK column") + assert.Contains(t, capturedFetchQueries[1], "?", + "second page WHERE must use parameter placeholders") +} + +// TestSnapshotBlobAndClobColumns verifies that BLOB/binary columns in snapshot +// rows are returned as raw []byte and CLOB/text columns as strings. +// +// This corresponds to the Debezium type propagation test (shouldPropagateSourceTypeByDatatype) +// which verifies that binary and character large objects are decoded correctly. +// Our connector uses convertDB2Value: CHAR/VARCHAR/CLOB/TEXT → string, +// BLOB/binary → raw []byte (to be base64-encoded when marshalled to JSON). +func TestSnapshotBlobAndClobColumns(t *testing.T) { + t.Parallel() + + queryCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + queryCount++ + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + case strings.Contains(q, "KEYCOLUSE"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + return []string{"COLNAME"}, [][]driver.Value{ + {"ID"}, {"DOC_CONTENT"}, {"THUMBNAIL"}, {"TITLE"}, + }, nil + case strings.Contains(q, "FETCH FIRST"): + if queryCount == 4 { + return []string{"ID", "DOC_CONTENT", "THUMBNAIL", "TITLE"}, [][]driver.Value{ + {int64(1), []byte("hello world"), []byte{0xDE, 0xAD, 0xBE, 0xEF}, []byte("My Document")}, + }, nil + } + return []string{"ID", "DOC_CONTENT", "THUMBNAIL", "TITLE"}, nil, nil + } + return nil, nil, nil + }, + colTypes: func(_ string) map[string]string { + return map[string]string{ + "ID": "INTEGER", + "DOC_CONTENT": "CLOB", // text large object → string + "THUMBNAIL": "BLOB", // binary large object → raw bytes + "TITLE": "VARCHAR", // varchar → string + } + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"DOCUMENTS"}, + BatchSize: 100, + }, Version{}) + + var events []ChangeEvent + _, err := s.Snapshot(context.Background(), func(e ChangeEvent) error { + events = append(events, e) + return nil + }) + + require.NoError(t, err) + require.Len(t, events, 1) + + row := events[0].Data + + // CLOB content must be decoded to string (not raw bytes) + assert.Equal(t, "hello world", row["DOC_CONTENT"], + "CLOB columns must be decoded to string via convertDB2Value") + + // BLOB content must remain as raw []byte + assert.Equal(t, []byte{0xDE, 0xAD, 0xBE, 0xEF}, row["THUMBNAIL"], + "BLOB columns must remain as raw []byte in the Data map") + + // VARCHAR must also be decoded to string + assert.Equal(t, "My Document", row["TITLE"], + "VARCHAR columns must be decoded to string") +} + +// TestSnapshotKeysetPaginationTwoPages verifies that large tables are paginated +// using keyset pagination: the first page is fetched without a WHERE clause; +// subsequent pages add "WHERE (PK) > (lastKeyValue)" to advance past rows +// already seen. This avoids OFFSET-based scans which degrade on large tables. +func TestSnapshotKeysetPaginationTwoPages(t *testing.T) { + t.Parallel() + + var capturedFetchQueries []string + queryCount := 0 + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + queryCount++ + if strings.Contains(q, "FETCH FIRST") { + capturedFetchQueries = append(capturedFetchQueries, q) + } + switch { + case strings.Contains(q, "SYNCHPOINT"): + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + case strings.Contains(q, "KEYCOLUSE"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}}, nil + case strings.Contains(q, "SYSCAT.COLUMNS"): + return []string{"COLNAME"}, [][]driver.Value{{"ID"}, {"NAME"}}, nil + case strings.Contains(q, "FETCH FIRST"): + switch len(capturedFetchQueries) { + case 1: // first page — no WHERE clause + return []string{"ID", "NAME"}, [][]driver.Value{ + {int64(1), "Alice"}, + {int64(2), "Bob"}, + }, nil + case 2: // second page — WHERE clause must be present + return []string{"ID", "NAME"}, [][]driver.Value{ + {int64(3), "Charlie"}, + }, nil + default: // third call → empty, end of pagination + return []string{"ID", "NAME"}, nil, nil + } + } + return nil, nil, nil + }, + }) + + s := NewSnapshotter(db, SnapshotConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"EMPLOYEES"}, + BatchSize: 2, // small batch forces multi-page read + }, Version{}) + + var events []ChangeEvent + _, err := s.Snapshot(context.Background(), func(e ChangeEvent) error { + events = append(events, e) + return nil + }) + + require.NoError(t, err) + assert.Len(t, events, 3, "must receive all 3 rows across two pages") + + require.Len(t, capturedFetchQueries, 3, "must issue exactly 3 SELECT queries (page1, page2, empty)") + assert.NotContains(t, capturedFetchQueries[0], "WHERE", + "first page must not have a WHERE clause") + assert.Contains(t, capturedFetchQueries[1], "WHERE", + "second page must add a keyset WHERE clause") + assert.Contains(t, capturedFetchQueries[1], "?", + "second page WHERE clause must use a parameter placeholder for the last key value") +} diff --git a/internal/impl/db2/replication/stream.go b/internal/impl/db2/replication/stream.go new file mode 100644 index 0000000000..56063e2a88 --- /dev/null +++ b/internal/impl/db2/replication/stream.go @@ -0,0 +1,1497 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "cmp" + "context" + "database/sql" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// db2IdentifierRE matches valid DB2 identifiers that are safe to embed in SQL +// without quoting: non-empty, first char letter or underscore, rest alphanumeric or underscore. +var db2IdentifierRE = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*$`) + +// StreamConfig holds configuration for the CDC streaming phase. The Streamer +// polls ASNCDC change tables for rows with IBMSNAP_COMMITSEQ > the last seen +// CSN, then sorts, pairs D+I update rows, and delivers them to the event handler. +type StreamConfig struct { + // Schemas is the list of DB2 source schemas (TABSCHEMA) whose change tables are polled. + // Must have at least one element; each entry must be a valid DB2 identifier + // (uppercase alphanumeric+underscore). For single-schema operation pass a one-element slice. + Schemas []string + + // Tables is the explicit list of table names (without schema prefix) to stream. + // When empty, all CDC-registered tables for Schema are discovered dynamically + // from ASNCDC.IBMSNAP_REGISTER during Initialize. + Tables []string + + // AsnCDCSchema is the schema that owns the SQL Replication control tables + // (IBMSNAP_REGISTER, and the generated change tables). Defaults to "ASNCDC". + // + // IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=tables-ibmsnap-register + AsnCDCSchema string + + // BackoffInterval is the sleep duration between change-table polls when no + // new rows are found. Should be at least the DB2 capture daemon's commit + // interval to avoid wasted round-trips. + BackoffInterval time.Duration + + // PollBatchSize is the FETCH FIRST N ROWS ONLY limit per change-table poll + // query. A full batch (rawCount == PollBatchSize) triggers safe-CSN logic + // to avoid advancing the watermark past a truncated transaction. + PollBatchSize int + + // StartingCSN is the exclusive lower-bound CSN for the first poll. Rows + // with IBMSNAP_COMMITSEQ <= StartingCSN are skipped. A null CSN means start + // from the very beginning of the change tables (all existing rows). + StartingCSN CSN + + // CommitSeqByteLen is the byte length of IBMSNAP_COMMITSEQ in the change + // tables. DB2 ≤ 11.x uses CHAR(10) FOR BIT DATA (10 bytes); DB2 12.1+ uses + // CHAR(16) FOR BIT DATA (16 bytes). Zero triggers auto-detection during + // Initialize via SYSCAT.COLUMNS (the recommended setting). + // + // IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=tables-ibmsnap-uow + CommitSeqByteLen int + + // TableFilter narrows which tables in Tables (or all discovered tables when + // Tables is empty) are actually polled. When nil all candidate tables are + // included. + TableFilter func(string) bool +} + +// asncdcSchema returns the CDC control schema, defaulting to "ASNCDC". +func (c *StreamConfig) asncdcSchema() string { + if c.AsnCDCSchema != "" { + return c.AsnCDCSchema + } + return "ASNCDC" +} + +// schemasINClause returns a SQL IN clause fragment like "'S1', 'S2'". +// Each schema name is validated before embedding so that callers that bypass +// Initialize() cannot inject SQL via an unvalidated Schemas value. +func (c *StreamConfig) schemasINClause() string { + quoted := make([]string, 0, len(c.Schemas)) + for _, s := range c.Schemas { + if !isValidDB2IdentifierInternal(s) { + continue + } + quoted = append(quoted, "'"+s+"'") + } + return strings.Join(quoted, ", ") +} + +// Streamer polls DB2 change tables and emits ChangeEvents in CSN order. +// +// On each poll iteration the Streamer: +// 1. Queries MAX(SYNCHPOINT) from ASNCDC.IBMSNAP_REGISTER as a safe upper +// bound (avoids reading rows inserted by in-progress transactions). +// 2. Issues one SELECT per change table for rows in the window +// (currentCSN, upperCSN]. +// 3. Merges and sorts all events by (CSN, IBMSNAP_INTENTSEQ) before delivery. +// 4. Advances its watermark to the highest CSN seen. +// +// When no new rows are found the loop backs off for StreamConfig.BackoffInterval. +// changeTableEntry is a pre-parsed element of changeTables built at Initialize() +// time. Storing schema and table separately avoids strings.SplitN on every poll. +type changeTableEntry struct { + qualKey string // "SCHEMA.TABLE" — key used in intentSeqByTable + schema string + table string + changeTableName string // qualified CDC change table, e.g. "ASNCDC.CD_ORDERS" + lastRawCount int // raw row count from the last poll; used to pre-size rawEvents next poll +} + +// changeTableMeta caches per-table column metadata derived from rows.Columns() +// and rows.ColumnTypes(). Computed once on the first poll and reused on +// subsequent polls — the change table schema is fixed for the connector lifetime. +type changeTableMeta struct { + columns []string + opIdx int + csnIdx int + intentSeqIdx int + tsIdx int + opcodeIdx int + dataColIdxs []int + dataColNames []string + dataColKinds []db2ColumnKind +} + +// Streamer polls DB2 CDC change tables and delivers ordered ChangeEvents to a handler. +type Streamer struct { + db *sql.DB + config StreamConfig + version Version + changeTables map[string]string // monitored table name -> change table qualified name + // changeTableList is the ordered slice form of changeTables, built once in + // Initialize() so pollChanges can iterate without SplitN on every poll cycle. + changeTableList []changeTableEntry + + // pendingBeforeByTable holds the trailing opTypeUpdateBefore event (opcode 3) + // when a PollBatchSize boundary falls between the D and I rows of an update + // pair. The matching opTypeUpdateAfter row arrives next poll and is inserted + // before it by pollChangeTable before pairOpcodeEvents merges the pair. + // Protected by pendingMu. Reset in Initialize() so stale D-rows from a prior + // connection cannot be matched against I-rows from the new connection. + pendingMu sync.Mutex + pendingBeforeByTable map[string]*ChangeEvent + + // activeWindow is the current open-window deduplication window set by the + // incremental snapshot engine. Protected by windowMu. + windowMu sync.Mutex + activeWindow *DeduplicationWindow + + // currentCSN tracks the latest CSN the Streamer has processed. + // Read by IncrementalSnapshotEngine.CurrentCSN() (snapshot goroutine) and + // written by Stream() (CDC goroutine). Using atomic.Pointer eliminates the + // RWMutex lock/unlock on the hot poll path; each write heap-allocates a CSN + // but that is one allocation per successful poll cycle (~10/s at default config). + currentCSN atomic.Pointer[CSN] + + // columnMeta caches per-change-table column metadata after the first poll. + // Keys are changeTableName (qualified, e.g. "ASNCDC.CD_ORDERS"). Written + // once per table on first poll; read-only after that. Protected by metaMu. + metaMu sync.RWMutex + columnMeta map[string]*changeTableMeta + + // commitSeqByteLen is the detected IBMSNAP_COMMITSEQ byte length (10 or 16). + // Written by Initialize() and read by CommitSeqByteLen() on separate goroutines; + // atomic to avoid a data race between the streaming goroutine (writer) and the + // schema-change poller goroutine (reader). + commitSeqByteLen atomic.Int32 + + // schemasIN is the pre-computed SQL IN clause fragment for config.Schemas, + // e.g. "'S1', 'S2'". It is computed once in NewStreamer and reused on every + // getUpperBound call to avoid repeated slice allocation + strings.Join. + schemasIN string + + // upperBoundQuery is the pre-computed SQL for getUpperBound, built once in + // Initialize() to avoid fmt.Sprintf on every poll cycle. + upperBoundQuery string + + // pollQueryParts maps each changeTableName to a pre-split query template. + // During Initialize the query body is split at its two %s placeholders into + // three static parts; buildPollQuery concatenates them with afterPredicate and + // upperHex without reflection (no fmt.Sprintf). Nil until Initialize() completes + // (tests and direct-use paths fall back to the ad-hoc construction in buildPollQuery). + pollQueryParts map[string][3]string + + // log is used to emit diagnostic warnings (e.g. unmatched D+I pairs). + // A noopLogger is used when nil to keep callers unconditional. + log Logger + + // lastPollEventCount is the observed total ChangeEvent count from the most recent + // successful poll. Used to right-size the allEvents slice on the next poll, + // avoiding repeated slice doublings on hot workloads while keeping idle polls cheap. + // Accessed concurrently by Stream() and any concurrent pollChanges callers; use + // atomic operations to avoid a data race under the -race detector. + lastPollEventCount atomic.Int64 +} + +// NewStreamer creates a new Streamer. +// Pass functional options (e.g. WithStreamerLogger) to configure optional behaviour. +func NewStreamer(db *sql.DB, config StreamConfig, version Version, opts ...func(*Streamer)) *Streamer { + s := &Streamer{ + db: db, + config: config, + version: version, + changeTables: make(map[string]string), + pendingBeforeByTable: make(map[string]*ChangeEvent), + columnMeta: make(map[string]*changeTableMeta), + schemasIN: config.schemasINClause(), + log: noopLogger{}, + } + // currentCSN is an atomic.Pointer; must be initialized via Store after struct + // creation because the struct literal cannot directly initialize atomic types. + startCSN := config.StartingCSN + s.currentCSN.Store(&startCSN) + for _, o := range opts { + o(s) + } + return s +} + +// WithStreamerLogger sets the logger used by the Streamer for diagnostic warnings. +func WithStreamerLogger(l Logger) func(*Streamer) { + return func(s *Streamer) { s.log = l } +} + +// SetWindow installs a DeduplicationWindow on the Streamer. +// CDC events within the window's CSN bracket will evict matching snapshot rows. +// When the stream passes window.HighCSN, the surviving rows are prepended to +// the batch as OpTypeRead events and the window is cleared. +func (s *Streamer) SetWindow(w *DeduplicationWindow) { + // Do NOT reset pendingBeforeByTable here. A D+I UPDATE pair can be split + // across the poll boundary that installs the window; discarding the pending + // D-row would cause the matching I-row to be re-classified as an INSERT, + // losing the before-image. The pending map is only cleared at Initialize() + // time (reconnect / startup), which is safe because those D-rows belong to + // a prior connection whose matching I-rows will never arrive on this stream. + s.windowMu.Lock() + s.activeWindow = w + s.windowMu.Unlock() +} + +// ClearWindow removes w from the Streamer's activeWindow if it is still +// current. Call this when the incremental snapshot engine cancels a window +// before it reaches HighCSN (e.g. context deadline) to prevent the stale +// window from evicting rows in a future snapshot round. +func (s *Streamer) ClearWindow(w *DeduplicationWindow) { + s.windowMu.Lock() + if s.activeWindow == w { + s.activeWindow = nil + } + s.windowMu.Unlock() +} + +// hasPending reports whether there is a pending before-image for tableName. +// pendingBeforeByTable is only written by pollChangeTable on the single Poll +// goroutine, so the lock is not strictly required for this read. It is kept +// for clarity and to guard against future callers from other goroutines. +func (s *Streamer) hasPending(tableName string) bool { + s.pendingMu.Lock() + defer s.pendingMu.Unlock() + return s.pendingBeforeByTable[tableName] != nil +} + +// CurrentCSN returns the latest committed CSN the Streamer has processed. +func (s *Streamer) CurrentCSN() CSN { + if ptr := s.currentCSN.Load(); ptr != nil { + return *ptr + } + return NullCSN() +} + +// CommitSeqByteLen returns the detected byte length of IBMSNAP_COMMITSEQ (10 or 16). +// The value is set by Initialize(); callers outside the Streamer should read it here +// rather than from the StreamConfig copy they passed at construction time. +// Uses an atomic load to be safe for concurrent reads from the schema-change poller goroutine. +func (s *Streamer) CommitSeqByteLen() int { + if n := s.commitSeqByteLen.Load(); n > 0 { + return int(n) + } + // Initialize() has not yet run (pre-connect or test path); return the default. + return 10 +} + +// InvalidateColumnMetaForSource drops the cached changeTableMeta for the change table +// corresponding to the given source owner and table. The next poll re-derives the meta +// from the live column descriptor, picking up any schema changes (e.g., ADDCOLUMN). +// Safe to call concurrently with Stream(). +func (s *Streamer) InvalidateColumnMetaForSource(srcOwner, srcTable string) { + key := strings.ToUpper(srcOwner) + "." + strings.ToUpper(srcTable) + // changeTables is written only during Initialize() and never modified after. + // The Streamer is published to callers only after Initialize() returns (via + // runStreaming's d.streamer assignment under d.mu), so any concurrent caller + // of InvalidateColumnMetaForSource already observes the fully-initialized map. + changeTable, ok := s.changeTables[key] + if !ok { + return + } + s.metaMu.Lock() + delete(s.columnMeta, changeTable) + s.metaMu.Unlock() +} + +// isValidDB2IdentifierInternal reports whether s is safe to embed in SQL strings: +// non-empty, first char letter or underscore, remaining chars alphanumeric or underscore. +func isValidDB2IdentifierInternal(s string) bool { + return db2IdentifierRE.MatchString(s) +} + +// Initialize discovers the change tables for all monitored tables from IBMSNAP_REGISTER. +// When config.Tables is empty, all registered tables for the schema are discovered dynamically. +// config.TableFilter (if set) narrows the discovered or configured set. +func (s *Streamer) Initialize(ctx context.Context) error { + for _, schema := range s.config.Schemas { + if !isValidDB2IdentifierInternal(schema) { + return fmt.Errorf("invalid schema %q: must be non-empty uppercase alphanumeric+underscore", schema) + } + } + + // Reset on reconnect: if Initialize is called again (e.g. after a transient + // error), stale entries from deregistered tables must not persist. A stale + // pending D-row would produce a spurious warning on the next poll when its + // expected I-row never arrives (different connection = different CSN space). + s.changeTables = make(map[string]string) + s.changeTableList = nil + s.pendingMu.Lock() + s.pendingBeforeByTable = make(map[string]*ChangeEvent) + s.pendingMu.Unlock() + s.metaMu.Lock() + s.columnMeta = make(map[string]*changeTableMeta) + s.metaMu.Unlock() + + cdcSchema := s.config.asncdcSchema() + + query := fmt.Sprintf(` + SELECT SOURCE_OWNER, SOURCE_TABLE, CD_OWNER, CD_TABLE + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER IN (%s) + `, cdcSchema, s.schemasIN) + + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return fmt.Errorf("querying CDC registration: %w", err) + } + defer rows.Close() + + dynamicDiscovery := len(s.config.Tables) == 0 + + registered := make(map[string]bool) + + for rows.Next() { + var sourceOwner, sourceTable, cdOwner, cdTable string + if err := rows.Scan(&sourceOwner, &sourceTable, &cdOwner, &cdTable); err != nil { + return fmt.Errorf("scanning registration row: %w", err) + } + + sourceOwner = strings.TrimSpace(sourceOwner) + sourceTable = strings.TrimSpace(sourceTable) + cdOwner = strings.TrimSpace(cdOwner) + cdTable = strings.TrimSpace(cdTable) + // Validate catalog-sourced identifiers before embedding in SQL as table names. + // Non-conformant values (spaces, quotes, semicolons) would allow SQL injection + // via a rogue IBMSNAP_REGISTER row; skip them rather than produce broken queries. + if !isValidDB2IdentifierInternal(cdOwner) || !isValidDB2IdentifierInternal(cdTable) { + continue + } + changeKey := strings.ToUpper(sourceOwner) + "." + strings.ToUpper(sourceTable) + changeTableName := cdOwner + "." + cdTable + + if dynamicDiscovery { + // Accept all registered tables, apply filter below. + if s.config.TableFilter == nil || s.config.TableFilter(sourceTable) { + s.changeTables[changeKey] = changeTableName + // Force a new backing array to avoid aliasing the caller's slice + // (on reconnect, the same StreamConfig is reused with spare capacity). + s.config.Tables = append(s.config.Tables[:len(s.config.Tables):len(s.config.Tables)], sourceTable) + registered[sourceTable] = true + } + } else { + for _, monitoredTable := range s.config.Tables { + if strings.EqualFold(sourceTable, monitoredTable) { + if s.config.TableFilter == nil || s.config.TableFilter(monitoredTable) { + s.changeTables[changeKey] = changeTableName + } + registered[monitoredTable] = true + break + } + } + } + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("error iterating registration rows: %w", err) + } + + if !dynamicDiscovery { + for _, table := range s.config.Tables { + if !registered[table] { + return fmt.Errorf("table %s is not registered for CDC (run ASNCDC.ADDTABLE)", table) + } + } + } + + if len(s.changeTables) == 0 { + return fmt.Errorf("no CDC-registered tables found for schemas %s (check ASNCDC.IBMSNAP_REGISTER)", s.schemasIN) + } + + // Pre-parse changeTables into changeTableList so pollChanges can iterate + // without calling strings.SplitN on every poll cycle, and so that + // detectCommitSeqByteLen (below) can iterate it in deterministic order. + s.changeTableList = make([]changeTableEntry, 0, len(s.changeTables)) + for qualKey, cdTableName := range s.changeTables { + parts := strings.SplitN(qualKey, ".", 2) + if len(parts) != 2 { + continue + } + s.changeTableList = append(s.changeTableList, changeTableEntry{ + qualKey: qualKey, + schema: parts[0], + table: parts[1], + changeTableName: cdTableName, + }) + } + + // Auto-detect IBMSNAP_COMMITSEQ byte length from the registered CD tables. + // Must run after changeTableList is built (detectCommitSeqByteLen iterates it). + if s.config.CommitSeqByteLen <= 0 { + if byteLen, err := s.detectCommitSeqByteLen(ctx); err == nil { + s.config.CommitSeqByteLen = byteLen + } else { + s.config.CommitSeqByteLen = 10 // safe default for older DB2 + } + } + // Mirror into the atomic so CommitSeqByteLen() and buildPollQuery are + // race-free when read by the schema-change poller goroutine concurrently. + s.commitSeqByteLen.Store(int32(s.config.CommitSeqByteLen)) + + // Pre-compute the getUpperBound query string. It is structurally invariant — + // cdcSchema and schemasIN never change after Initialize — so rebuilding it with + // fmt.Sprintf on every poll cycle is unnecessary allocation. + s.upperBoundQuery = fmt.Sprintf(` + SELECT MAX(t.SYNCHPOINT) FROM ( + SELECT CD_NEW_SYNCHPOINT AS SYNCHPOINT + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER IN (%s) + UNION ALL + SELECT SYNCHPOINT + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER IN (%s) + ) t`, + cdcSchema, s.schemasIN, + cdcSchema, s.schemasIN, + ) + + // Pre-compute per-table poll query templates. changeTableName and PollBatchSize + // are baked in once here; only afterPredicate (%[1]s) and upperHex (%[2]s) are + // substituted at query time, eliminating the large fmt.Sprintf on every poll. + const pollQueryBody = ` + SELECT CASE + WHEN cdc.IBMSNAP_OPERATION = 'D' + AND LEAD(cdc.IBMSNAP_OPERATION,1,'X') OVER (PARTITION BY cdc.IBMSNAP_COMMITSEQ ORDER BY cdc.IBMSNAP_INTENTSEQ) = 'I' + THEN 3 + WHEN cdc.IBMSNAP_OPERATION = 'I' + AND LAG(cdc.IBMSNAP_OPERATION,1,'X') OVER (PARTITION BY cdc.IBMSNAP_COMMITSEQ ORDER BY cdc.IBMSNAP_INTENTSEQ) = 'D' + THEN 4 + WHEN cdc.IBMSNAP_OPERATION = 'D' THEN 1 + WHEN cdc.IBMSNAP_OPERATION = 'I' THEN 2 + ELSE 0 + END AS IBMSNAP_OPCODE, + cdc.* + FROM %s cdc + WHERE %%s + AND cdc.IBMSNAP_COMMITSEQ <= X'%%s' + ORDER BY cdc.IBMSNAP_COMMITSEQ, cdc.IBMSNAP_INTENTSEQ + FETCH FIRST %d ROWS ONLY + ` + s.pollQueryParts = make(map[string][3]string, len(s.changeTableList)) + for _, entry := range s.changeTableList { + tmpl := fmt.Sprintf(pollQueryBody, entry.changeTableName, s.config.PollBatchSize) + // Split at the two %s placeholders so buildPollQuery can substitute + // afterPredicate and upperHex via plain concatenation (no fmt.Sprintf). + parts := strings.SplitN(tmpl, "%s", 3) + if len(parts) == 3 { + s.pollQueryParts[entry.changeTableName] = [3]string{parts[0], parts[1], parts[2]} + } + } + + return nil +} + +// detectCommitSeqByteLen queries SYSCAT.COLUMNS to determine the actual byte +// length of IBMSNAP_COMMITSEQ in the registered CD tables. +// DB2 ≤ 11.x uses CHAR(10) FOR BIT DATA; DB2 12.1+ uses CHAR(16) FOR BIT DATA. +func (s *Streamer) detectCommitSeqByteLen(ctx context.Context) (int, error) { + // Iterate changeTableList (deterministic order) rather than the changeTables + // map to avoid non-deterministic query ordering in tests and production. + for _, entry := range s.changeTableList { + parts := strings.SplitN(entry.changeTableName, ".", 2) + if len(parts) != 2 { + continue + } + var length int + err := s.db.QueryRowContext(ctx, + "SELECT LENGTH FROM SYSCAT.COLUMNS WHERE TABSCHEMA=? AND TABNAME=? AND COLNAME='IBMSNAP_COMMITSEQ'", + strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), + ).Scan(&length) + // DB2 only defines CHAR(10) for ≤11.x and CHAR(16) for 12.1+. + // Accept only these two values to reject malformed or malicious catalog rows + // that could cause oversized SQL hex literals or excessive buffer allocations. + if err == nil && (length == 10 || length == 16) { + return length, nil + } + } + return 0, errors.New("could not detect IBMSNAP_COMMITSEQ byte length from registered CD tables") +} + +// Stream continuously polls change tables and sends events to handler until ctx is cancelled. +func (s *Streamer) Stream(ctx context.Context, handler func(event ChangeEvent) error) error { + currentCSN := s.config.StartingCSN + // intentSeqByTable tracks the last IBMSNAP_INTENTSEQ seen per table at currentCSN, + // enabling composite (CSN, IntentSeq) pagination when a batch boundary lands + // mid-transaction at the same CSN across multiple tables. + intentSeqByTable := make(map[string]int64) + + // Reuse a single backoff timer rather than allocating a new time.Timer per idle cycle. + backoffTimer := time.NewTimer(s.config.BackoffInterval) + defer backoffTimer.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + events, maxCSN, newIntentSeqs, err := s.pollChanges(ctx, currentCSN, intentSeqByTable) + if err != nil { + return fmt.Errorf("polling changes: %w", err) + } + + for _, event := range events { + if err := handler(event); err != nil { + return fmt.Errorf("handler error: %w", err) + } + } + + if len(events) > 0 { + currentCSN = maxCSN + csnCopy := maxCSN + s.currentCSN.Store(&csnCopy) + intentSeqByTable = newIntentSeqs + } else { + // Drain the channel before Reset: if the timer fired between the last + // select exit and now, Reset would not reset the fire-state and the next + // select would return immediately, producing a zero-sleep spin cycle. + if !backoffTimer.Stop() { + select { + case <-backoffTimer.C: + default: + } + } + backoffTimer.Reset(s.config.BackoffInterval) + select { + case <-ctx.Done(): + return ctx.Err() + case <-backoffTimer.C: + } + } + } +} + +// tableResult holds one table's poll output. +type tableResult struct { + tableName string + events []ChangeEvent + // full is true when the raw SQL row count equalled PollBatchSize (before + // pairOpcodeEvents merging) OR when the table has a pending before-image + // that must be matched next poll. Either condition means the watermark must + // not advance past this table's last event CSN. + full bool + // heldCSN is non-nil when pollChangeTable stored a new pending D-row + // (opTypeUpdateBefore) at the end of this poll cycle. The matching I-row + // arrives on the next poll; anyFullAtReturnCSN composite pagination in + // pollChanges ensures the I-row remains reachable on the next poll via + // "(COMMITSEQ = heldCSN AND INTENTSEQ > maxSeenIntentSeq)". + heldCSN *CSN +} + +// computeSafeCSN returns the highest CSN that is safe to advance to across all +// tables in a single poll round. +// +// For each full table (raw row count == PollBatchSize, or pending before-image +// held), the batch may have been truncated mid-transaction. The safe per-table +// ceiling is the penultimate distinct CSN within that table's own batch; events +// at the max CSN must be re-fetched. Edge case: all rows in the batch share one +// CSN — advance to tableMax anyway (at-least-once; caller deduplicates on +// csn+intentSeq). +// +// Non-full tables impose no constraint: all rows up to upperCSN were returned. +// +// The global safe ceiling is the minimum of all per-table ceilings, starting +// from upperCSN (no full table → no narrowing). +// +// afterCSN is the watermark of the last committed poll; it is used as a +// no-advance floor when a table holds a pending D-row but returned no other +// events — the matching I-row has not arrived yet and must not be skipped. +func computeSafeCSN(results []tableResult, upperCSN, afterCSN CSN) CSN { + safeCSN := upperCSN + for _, r := range results { + if !r.full { + continue // not full — no constraint from this table + } + if len(r.events) == 0 { + if r.heldCSN != nil { + // A pending D-row was created this poll with no other events in + // this batch. The matching I-row arrives next poll; must not + // advance past afterCSN so the I-row's CSN (== heldCSN) remains + // within the next poll window's [afterCSN+1, upperCSN] range. + if afterCSN.Less(safeCSN) { + safeCSN = afterCSN + } + } + continue + } + tableMax := r.events[len(r.events)-1].CSN + // Default: edge case where all rows share one CSN — advance to tableMax. + tableSafe := tableMax + for i := len(r.events) - 2; i >= 0; i-- { + if !r.events[i].CSN.Equal(tableMax) { + tableSafe = r.events[i].CSN // found penultimate distinct CSN + break + } + } + if tableSafe.Less(safeCSN) { + safeCSN = tableSafe + } + // NOTE: no extra cap for r.heldCSN here. + // + // When all events in a full batch share one CSN (tableSafe == heldCSN == + // tableMax), the old cap "safeCSN = afterCSN" trimmed every event and caused + // an infinite loop. The I-row reachability concern (next poll "COMMITSEQ > + // tableSafe" would skip the I-row that shares heldCSN) is already addressed by + // anyFullAtReturnCSN composite pagination: pollChanges sets newIntentSeqs[table] + // = max(INTENTSEQ in this batch) so the next poll uses: + // "(COMMITSEQ > safeCSN) OR (COMMITSEQ = safeCSN AND INTENTSEQ > maxSeen)" + // which reaches the I-row at the same CSN with a higher INTENTSEQ. + // + // When events have mixed CSNs (tableSafe < heldCSN = tableMax), the condition + // "!tableSafe.Less(heldCSN)" was already false, so the old block never fired + // in that case — removing it has no effect on mixed-CSN batches. + } + return safeCSN +} + +// computeReturnCSN caps safeCSN at the actual max event CSN so that sparse +// event windows do not advance the watermark past the last observed event. +// When no table is full, safeCSN equals upperCSN which may be much higher than +// any event in the batch; returning upperCSN would silently skip future events +// that land before the next poll window opens. +func computeReturnCSN(safeCSN CSN, sorted []ChangeEvent) CSN { + if len(sorted) == 0 { + return safeCSN + } + maxEventCSN := sorted[len(sorted)-1].CSN + if maxEventCSN.Less(safeCSN) { + return maxEventCSN + } + return safeCSN +} + +// trimToSafeCSN removes events with CSN strictly above safeCSN (right-trims the +// sorted slice). If every event exceeds safeCSN — the heldCSN edge case where +// safeCSN equals afterCSN and no safe event exists — the slice is empty so the +// watermark does not advance and the next poll re-fetches the same window. +func trimToSafeCSN(sorted []ChangeEvent, safeCSN CSN) []ChangeEvent { + if len(sorted) == 0 || !sorted[len(sorted)-1].CSN.Greater(safeCSN) { + return sorted // fast path: all events within safe window (common case) + } + // Binary search for the first event at or after safeCSN: O(log n) vs O(n). + // Use a proper 3-way comparator so BinarySearchFunc lands at the first event + // with CSN == safeCSN; then advance past equal events to include them. + cut, _ := slices.BinarySearchFunc(sorted, safeCSN, func(e ChangeEvent, s CSN) int { + return e.CSN.Compare(s) + }) + for cut < len(sorted) && sorted[cut].CSN.Equal(safeCSN) { + cut++ + } + return sorted[:cut] +} + +// pollChanges polls all change tables and merges events ordered by CSN. +// +// Watermark safety: IBMSNAP_COMMITSEQ identifies a DB2 transaction; all rows +// from that transaction share the same CSN. If any change table returned +// exactly PollBatchSize rows the batch may have been truncated in the middle +// of a transaction, so we must NOT advance currentCSN past that last CSN — the +// remaining rows would be permanently skipped by the `> afterCSN` predicate. +// +// We compute the safe ceiling per-table (not globally) and take the minimum. +// Using the global penultimate would cause events from a full table at a lower +// CSN to be permanently skipped when another table has events at a higher CSN. +// +// intentSeqByTable holds the per-table last IBMSNAP_INTENTSEQ seen at afterCSN. +// When any table has a non-zero entry, composite (CSN, IntentSeq) pagination is +// used for that table so the batch does not re-deliver already-seen rows. +// A global afterIntentSeq would incorrectly skip table B's events when table A +// defines the high-water mark for a shared CSN. +func (s *Streamer) pollChanges(ctx context.Context, afterCSN CSN, intentSeqByTable map[string]int64) ([]ChangeEvent, CSN, map[string]int64, error) { + upperCSN, err := s.getUpperBound(ctx) + if err != nil { + return nil, CSN{}, nil, fmt.Errorf("getting upper bound: %w", err) + } + // Skip the poll when the capture daemon hasn't advanced past our watermark, + // but only when no table has mid-CSN pending rows (intentSeq > 0). + // Any table with intentSeq > 0 means we are mid-transaction and must continue. + anyPending := false + for _, seq := range intentSeqByTable { + if seq > 0 { + anyPending = true + break + } + } + if !upperCSN.Greater(afterCSN) && !anyPending { + // If there is an active snapshot window whose HighCSN is null (the table + // was just registered and asncap hasn't captured any transactions yet), + // flush the window immediately so the snapshot rows are returned rather + // than blocking forever waiting for a CDC event to advance past HighCSN. + s.windowMu.Lock() + w := s.activeWindow + s.windowMu.Unlock() + if w != nil && (w.HighCSN.IsNull() || !upperCSN.Less(w.HighCSN)) { + flushed := w.Flush() + // Only clear activeWindow if it is still the same window we just flushed. + // SetWindow may have installed a new window between our Flush call and + // this lock acquisition; clearing it unconditionally would lose that window. + s.windowMu.Lock() + if s.activeWindow == w { + s.activeWindow = nil + } + s.windowMu.Unlock() + if len(flushed) > 0 { + return flushed, afterCSN, intentSeqByTable, nil + } + } + return nil, afterCSN, intentSeqByTable, nil + } + + // changeTableList is normally pre-built by Initialize(). Fall back to + // rebuilding it inline for callers (e.g. unit tests) that populate + // changeTables directly without calling Initialize(). + if len(s.changeTableList) == 0 && len(s.changeTables) > 0 { + s.changeTableList = make([]changeTableEntry, 0, len(s.changeTables)) + for qualKey, cdTableName := range s.changeTables { + parts := strings.SplitN(qualKey, ".", 2) + if len(parts) != 2 { + continue + } + cdParts := strings.SplitN(cdTableName, ".", 2) + if len(cdParts) != 2 || !isValidDB2IdentifierInternal(cdParts[0]) || !isValidDB2IdentifierInternal(cdParts[1]) { + s.log.Warnf("changeTableList fallback: skipping invalid CD table name %q", cdTableName) + continue + } + s.changeTableList = append(s.changeTableList, changeTableEntry{ + qualKey: qualKey, schema: parts[0], table: parts[1], changeTableName: cdTableName, + }) + } + } + n := len(s.changeTableList) + results := make([]tableResult, 0, n) + // Size allEvents based on last observed total to avoid repeated doublings on + // hot workloads while keeping idle polls cheap (default n*16 cap). + allocCap := int(s.lastPollEventCount.Load()) + if minCap := n * 16; allocCap < minCap { + allocCap = minCap + } + allEvents := make([]ChangeEvent, 0, allocCap) + + for i := range s.changeTableList { + entry := &s.changeTableList[i] + tableIntentSeq := intentSeqByTable[entry.qualKey] + // Capture hasPending BEFORE pollChangeTable; the call may consume the old + // pending entry and set a NEW pending D-row at the batch tail. + wasPending := s.hasPending(entry.qualKey) + events, rawCount, err := s.pollChangeTable(ctx, entry.qualKey, entry.schema, entry.table, entry.changeTableName, afterCSN, tableIntentSeq, upperCSN, entry.lastRawCount) + entry.lastRawCount = rawCount + if err != nil { + return nil, CSN{}, nil, fmt.Errorf("polling change table %s: %w", entry.changeTableName, err) + } + // Snapshot any NEW pending D-row written by pollChangeTable this cycle. + // Its CSN is the lower bound that safeCSN must not advance to or past. + var heldCSN *CSN + s.pendingMu.Lock() + if ev := s.pendingBeforeByTable[entry.qualKey]; ev != nil { + c := ev.CSN + heldCSN = &c + } + s.pendingMu.Unlock() + // full=true when: raw row count hit the batch limit (batch was truncated), + // OR a pending before-image was held before this poll (matching I row must + // arrive next poll), OR a new pending D-row was just stored this poll + // (matching I row must arrive next poll — watermark must not advance to heldCSN). + full := rawCount == s.config.PollBatchSize || wasPending || heldCSN != nil + results = append(results, tableResult{tableName: entry.qualKey, events: events, full: full, heldCSN: heldCSN}) + allEvents = append(allEvents, events...) + } + + sorted := s.sortEventsByCSN(allEvents) + if len(sorted) == 0 { + // Even with no CDC events this poll, the capture daemon may have advanced + // past the active window's HighCSN without producing events for the tracked + // tables (e.g. other-table transactions advanced the watermark). Flush the + // window so the snapshot rows are not blocked indefinitely. + s.windowMu.Lock() + w := s.activeWindow + s.windowMu.Unlock() + if w != nil && (w.HighCSN.IsNull() || upperCSN.Greater(w.HighCSN) || upperCSN.Equal(w.HighCSN)) { + flushed := w.Flush() + // Only clear activeWindow if it is still the same window we flushed. + s.windowMu.Lock() + if s.activeWindow == w { + s.activeWindow = nil + } + s.windowMu.Unlock() + if len(flushed) > 0 { + return flushed, afterCSN, intentSeqByTable, nil + } + } + return nil, afterCSN, intentSeqByTable, nil + } + + safeCSN := computeSafeCSN(results, upperCSN, afterCSN) + sorted = trimToSafeCSN(sorted, safeCSN) + + if len(sorted) == 0 { + return nil, afterCSN, intentSeqByTable, nil + } + + returnCSN := computeReturnCSN(safeCSN, sorted) + + // Deduplication window: evict rows whose PK was modified by in-window CDC events, + // then flush surviving snapshot rows when the batch passes the window's HighCSN. + s.windowMu.Lock() + w := s.activeWindow + s.windowMu.Unlock() + if w != nil { + for i := range sorted { + if !sorted[i].CSN.IsNull() { + w.Evict(sorted[i].Schema, sorted[i].Table, sorted[i].Data, sorted[i].CSN) + // For UPDATE events also evict using the before-image so that a + // PK-modifying UPDATE evicts the old PK from the snapshot window. + if sorted[i].Operation == OpTypeUpdate && len(sorted[i].BeforeData) > 0 { + w.Evict(sorted[i].Schema, sorted[i].Table, sorted[i].BeforeData, sorted[i].CSN) + } + } + } + // If the last event in the batch has reached or passed HighCSN, flush the + // surviving snapshot rows and prepend them before the CDC events. + // Use GreaterOrEqual so a batch whose last event lands exactly on HighCSN + // triggers the flush immediately rather than waiting for the next poll. + if sorted[len(sorted)-1].CSN.Greater(w.HighCSN) || sorted[len(sorted)-1].CSN.Equal(w.HighCSN) { + flushed := w.Flush() + if len(flushed) > 0 { + sorted = append(flushed, sorted...) + } + // Only clear activeWindow if it is still the same window we flushed. + s.windowMu.Lock() + if s.activeWindow == w { + s.activeWindow = nil + } + s.windowMu.Unlock() + } + } + + // Build per-table max intentSeq at returnCSN for composite pagination. + // + // Normally: if returnCSN advanced past afterCSN, intent seqs reset to nil so + // the next poll uses the simple "COMMITSEQ > returnCSN" predicate. + // + // Exception — large transaction: when a full table's entire batch shares a + // single CSN (e.g. 5000 UPDATE rows all at CSN=N, PollBatchSize=1000), the + // loop in computeSafeCSN finds no penultimate CSN so tableSafe = tableMax = N. + // returnCSN = N > afterCSN, which would reset newIntentSeqs to nil. The next + // poll would then use "COMMITSEQ > N", permanently skipping the remaining 4000 + // rows at N. To prevent this, we detect when any full table's last event lands + // at returnCSN and populate newIntentSeqs so the next poll uses composite + // pagination: "(COMMITSEQ > N OR (COMMITSEQ = N AND INTENTSEQ > maxSeen))". + anyFullAtReturnCSN := false + for _, r := range results { + if r.full && len(r.events) > 0 && r.events[len(r.events)-1].CSN.Equal(returnCSN) { + anyFullAtReturnCSN = true + break + } + } + + var newIntentSeqs map[string]int64 + if returnCSN.Greater(afterCSN) && !anyFullAtReturnCSN { + // nil signals "all seqs reset"; avoids one map allocation per poll cycle. + } else { + newIntentSeqs = make(map[string]int64, len(intentSeqByTable)) + for i := len(sorted) - 1; i >= 0; i-- { + ev := sorted[i] + if !ev.CSN.Equal(returnCSN) { + break + } + if ev.IntentSeq > newIntentSeqs[ev.QualKey] { + newIntentSeqs[ev.QualKey] = ev.IntentSeq + } + } + } + // Update adaptive capacity hint for the next poll's allEvents allocation. + s.lastPollEventCount.Store(int64(len(sorted))) + return sorted, returnCSN, newIntentSeqs, nil +} + +// getUpperBound returns the highest log position the DB2 capture daemon has +// durably written, using MAX over both CD_NEW_SYNCHPOINT (set at ADDTABLE time, +// always non-null) and SYNCHPOINT (updated after each captured transaction). +// This prevents reading change rows that belong to an in-progress transaction. +func (s *Streamer) getUpperBound(ctx context.Context) (CSN, error) { + // Use the pre-computed query when Initialize has been called; otherwise build + // it on the fly (test or direct-use path where Initialize was skipped). + query := s.upperBoundQuery + if query == "" { + cdcSchema := s.config.asncdcSchema() + query = fmt.Sprintf(` + SELECT MAX(t.SYNCHPOINT) FROM ( + SELECT CD_NEW_SYNCHPOINT AS SYNCHPOINT + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER IN (%s) + UNION ALL + SELECT SYNCHPOINT + FROM %s.IBMSNAP_REGISTER + WHERE SOURCE_OWNER IN (%s) + ) t`, + cdcSchema, s.schemasIN, + cdcSchema, s.schemasIN, + ) + } + + var synchpointRaw any + err := s.db.QueryRowContext(ctx, query).Scan(&synchpointRaw) + if err != nil { + return CSN{}, fmt.Errorf("getting SYNCHPOINT: %w", err) + } + + if synchpointRaw == nil { + return NullCSN(), nil + } + + // SYNCHPOINT is CHAR(n) FOR BIT DATA; the DB2 CLI driver may return + // it as a hex-encoded string — getBitDataBytes hex-decodes it if needed. + synchpointBytes := getBitDataBytes(&synchpointRaw) + if len(synchpointBytes) == 0 { + return NullCSN(), nil + } + + return NewCSNFromDBValue(synchpointBytes), nil +} + +// buildPollQuery returns the SQL used to fetch a batch of change rows. +// +// IBMSNAP_COMMITSEQ is typed CHAR(n) FOR BIT DATA — a raw binary column. +// Reliable parameter binding for binary columns via SQL_C_BINARY would require +// exact n-byte buffers and careful length handling. To avoid this complexity, +// CSN bounds are embedded directly as hex literals sized to match the column +// (X'<2n chars>'). This is safe because CSN values come from our own state, +// never from user input. +// +// When afterIntentSeq > 0 the query uses composite (CSN, IntentSeq) pagination +// so that a PollBatchSize cutoff mid-CSN resumes from where it left off rather +// than re-reading or skipping the tail of the same transaction. +func (s *Streamer) buildPollQuery(changeTableName string, afterCSN CSN, afterIntentSeq int64, upperCSN CSN) string { + byteLen := int(s.commitSeqByteLen.Load()) + if byteLen <= 0 { + byteLen = 10 // Initialize() has not yet run; fall back to DB2 ≤11.x default. + } + afterHex := afterCSN.SQLHex(byteLen) + upperHex := upperCSN.SQLHex(byteLen) + + var afterPredicate string + if afterIntentSeq > 0 { + // Composite pagination: resume within the same CSN using IntentSeq. + // IBMSNAP_INTENTSEQ is CHAR(10) FOR BIT DATA — must compare using a + // binary hex literal X'...' not a decimal integer. + // + // Encoding: pack the int64 into bytes [0:8] (big-endian), matching how + // getInt64 decodes it (reads bytes [0:8] of the 10-byte column value). + // Pad the trailing bytes [8:9] to 0xFF so that the 10-byte binary + // comparison correctly skips ALL rows whose leading 8-byte value equals + // afterIntentSeq (regardless of their trailing 2 bytes). Without 0xFF + // padding, rows at the same int64 but with trailing bytes > 0x00 would + // satisfy the predicate, re-delivering already-seen rows. + var intentBytes [10]byte + binary.BigEndian.PutUint64(intentBytes[:8], uint64(afterIntentSeq)) + intentBytes[8] = 0xFF + intentBytes[9] = 0xFF + // Encode directly to uppercase hex, avoiding the two-alloc + // strings.ToUpper(hex.EncodeToString(...)) pattern. + const upperHex = "0123456789ABCDEF" + var intentHexBuf [20]byte + for i, b := range intentBytes { + intentHexBuf[2*i] = upperHex[b>>4] + intentHexBuf[2*i+1] = upperHex[b&0x0F] + } + intentHex := string(intentHexBuf[:]) + // Assemble without fmt to avoid reflection overhead; all parts are hex-safe strings. + afterPredicate = "(cdc.IBMSNAP_COMMITSEQ > X'" + afterHex + + "' OR (cdc.IBMSNAP_COMMITSEQ = X'" + afterHex + + "' AND cdc.IBMSNAP_INTENTSEQ > X'" + intentHex + "'))" + } else { + afterPredicate = "cdc.IBMSNAP_COMMITSEQ > X'" + afterHex + "'" + } + + // Use the pre-split per-table parts when available (normal production path after + // Initialize()). Concatenate three static parts with afterPredicate and upperHex + // directly — no fmt.Sprintf reflection on the hot poll path. + if parts, ok := s.pollQueryParts[changeTableName]; ok { + return parts[0] + afterPredicate + parts[1] + upperHex + parts[2] + } + + // Fallback: build the query ad-hoc for tests and direct-use paths where + // Initialize() was not called. LEAD/LAG window functions classify each D/I row: + // OPCODE 1 = standalone DELETE OPCODE 2 = standalone INSERT + // OPCODE 3 = before-image of UPDATE (D row immediately before I) + // OPCODE 4 = after-image of UPDATE (I row immediately after D) + // Ported from Debezium LuwPlatform.java CHANGE_TABLE_DATA_COLUMNS_QUERY. + return fmt.Sprintf(` + SELECT CASE + WHEN cdc.IBMSNAP_OPERATION = 'D' + AND LEAD(cdc.IBMSNAP_OPERATION,1,'X') OVER (PARTITION BY cdc.IBMSNAP_COMMITSEQ ORDER BY cdc.IBMSNAP_INTENTSEQ) = 'I' + THEN 3 + WHEN cdc.IBMSNAP_OPERATION = 'I' + AND LAG(cdc.IBMSNAP_OPERATION,1,'X') OVER (PARTITION BY cdc.IBMSNAP_COMMITSEQ ORDER BY cdc.IBMSNAP_INTENTSEQ) = 'D' + THEN 4 + WHEN cdc.IBMSNAP_OPERATION = 'D' THEN 1 + WHEN cdc.IBMSNAP_OPERATION = 'I' THEN 2 + ELSE 0 + END AS IBMSNAP_OPCODE, + cdc.* + FROM %s cdc + WHERE %s + AND cdc.IBMSNAP_COMMITSEQ <= X'%s' + ORDER BY cdc.IBMSNAP_COMMITSEQ, cdc.IBMSNAP_INTENTSEQ + FETCH FIRST %d ROWS ONLY + `, changeTableName, afterPredicate, upperHex, s.config.PollBatchSize) +} + +// pollChangeTable queries a single change table for events in the CSN window (afterCSN, upperCSN]. +// +// Returns the merged events, the raw SQL row count (before pairOpcodeEvents merging), and any error. +// The raw row count is used by the caller to determine whether the batch was full (truncated). +// +// Pending before-image handling: when the LEAD/LAG query places a D+I pair at the PollBatchSize +// boundary, the opTypeUpdateBefore (D) row is saved in s.pendingBeforeByTable[tableName] instead +// of being emitted as a phantom delete. On the next poll, it is prepended to the raw events before +// pairOpcodeEvents processes the complete pair. +func (s *Streamer) pollChangeTable(ctx context.Context, tableKey, schema, tableName, changeTableName string, afterCSN CSN, afterIntentSeq int64, upperCSN CSN, rawCountHint int) ([]ChangeEvent, int, error) { + query := s.buildPollQuery(changeTableName, afterCSN, afterIntentSeq, upperCSN) + + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return nil, 0, fmt.Errorf("querying change table %s: %w", changeTableName, err) + } + defer rows.Close() + + // Look up cached column metadata; derive and cache on first poll or after + // an explicit InvalidateColumnMetaForSource call (e.g., on OpTypeSchemaChange). + s.metaMu.RLock() + meta := s.columnMeta[changeTableName] + s.metaMu.RUnlock() + + if meta == nil { + columns, err := rows.Columns() + if err != nil { + return nil, 0, fmt.Errorf("getting columns for %s: %w", changeTableName, err) + } + + columnTypes, err := rows.ColumnTypes() + if err != nil { + return nil, 0, fmt.Errorf("getting column types for %s: %w", changeTableName, err) + } + + m := &changeTableMeta{ + columns: columns, + opIdx: -1, + csnIdx: -1, + intentSeqIdx: -1, + tsIdx: -1, + opcodeIdx: -1, + } + for i, col := range columns { + switch col { + case "IBMSNAP_OPERATION": + m.opIdx = i + case "IBMSNAP_COMMITSEQ": + m.csnIdx = i + case "IBMSNAP_INTENTSEQ": + m.intentSeqIdx = i + case "IBMSNAP_LOGMARKER": + m.tsIdx = i + case "IBMSNAP_OPCODE": + m.opcodeIdx = i + default: + if !strings.HasPrefix(col, "IBMSNAP_") { + m.dataColIdxs = append(m.dataColIdxs, i) + m.dataColNames = append(m.dataColNames, col) + } + } + } + if m.opIdx < 0 || m.csnIdx < 0 || m.intentSeqIdx < 0 { + // Sanitize column names before embedding in error string: DB2 catalog + // names are normally pure ASCII identifiers, but quoted names can contain + // arbitrary bytes. Replace any non-printable-ASCII byte with '?'. + safe := make([]string, len(columns)) + for i, c := range columns { + safe[i] = strings.Map(func(r rune) rune { + if r >= 0x20 && r < 0x7F { + return r + } + return '?' + }, c) + } + return nil, 0, fmt.Errorf("change table %s is missing required IBMSNAP_ columns (found: %v)", changeTableName, safe) + } + m.dataColKinds = make([]db2ColumnKind, len(m.dataColIdxs)) + for i, idx := range m.dataColIdxs { + m.dataColKinds[i] = columnKind(columnTypes[idx].DatabaseTypeName()) + } + s.metaMu.Lock() + s.columnMeta[changeTableName] = m + s.metaMu.Unlock() + meta = m + } + + opIdx, csnIdx, intentSeqIdx, tsIdx, opcodeIdx := meta.opIdx, meta.csnIdx, meta.intentSeqIdx, meta.tsIdx, meta.opcodeIdx + dataColIdxs, dataColNames, dataColKinds := meta.dataColIdxs, meta.dataColNames, meta.dataColKinds + + // Use the last observed row count as a capacity hint to avoid repeated slice + // doublings on active tables. Fall back to 16 for the first poll or idle tables. + rawEvents := make([]ChangeEvent, 0, max(rawCountHint, 16)) + var rawCount int + + // Reuse scanDest and scanPtrs across rows to avoid one allocation per column per row. + scanDest := make([]any, len(meta.columns)) + scanPtrs := make([]any, len(meta.columns)) + for i := range scanDest { + scanPtrs[i] = &scanDest[i] + } + + for rows.Next() { + // Clear previous row values before scanning (avoids stale data on nil columns). + clear(scanDest) + + if err := rows.Scan(scanPtrs...); err != nil { + return nil, 0, fmt.Errorf("scanning row from %s: %w", changeTableName, err) + } + + // IBMSNAP_COMMITSEQ is CHAR(n) FOR BIT DATA; the DB2 CLI driver may + // return it as a hex-encoded string — getBitDataBytes handles both. + csnBytes := getBitDataBytes(&scanDest[csnIdx]) + intentSeq := getInt64(&scanDest[intentSeqIdx]) + + var timestamp time.Time + if tsIdx >= 0 { + timestamp = getTime(&scanDest[tsIdx]) + } + + csn := NewCSNFromDBValue(csnBytes) + + var opType OpType + var opErr error + if opcodeIdx >= 0 { + code := getInt64(&scanDest[opcodeIdx]) + opType, opErr = fromOpcodeInt(code) + } else { + operation := getString(&scanDest[opIdx]) + opType, opErr = FromDB2Op(operation) + } + // Count every row the DB returned so the fullness signal accurately + // reflects whether the batch was truncated by FETCH FIRST. Skipping + // unknown opcodes after this point is safe; they produce no events. + rawCount++ + if opErr != nil { + // Skip unknown operation types rather than failing the whole batch. + continue + } + + data := make(map[string]any, len(dataColNames)) + for i, idx := range dataColIdxs { + data[dataColNames[i]] = convertDB2Value(scanDest[idx], dataColKinds[i]) + } + + rawEvents = append(rawEvents, ChangeEvent{ + Schema: schema, + Table: tableName, + QualKey: tableKey, + Operation: opType, + CSN: csn, + IntentSeq: intentSeq, + Timestamp: timestamp, + Data: data, + }) + } + + if err := rows.Err(); err != nil { + return nil, rawCount, err + } + + // Cross-poll D+I pairing: read, update, and clear the pending before-image + // for this table in a single lock acquisition (H4: was 3 separate acquires). + // pollChangeTable runs on the Streamer goroutine; the only concurrent writer + // is SetWindow (signals goroutine), which replaces the entire map, so all + // reads and writes here must be under pendingMu. + s.pendingMu.Lock() + pending := s.pendingBeforeByTable[tableKey] + if pending != nil { + // Scan all events for the matching I-row (not just index 0) — composite + // pagination can produce a batch like [evt@7, I@8] where the match is not + // at position 0. + // Two-pass scan: prefer opTypeUpdateAfter over OpTypeInsert at the same + // CSN. If composite pagination splits a D+I pair across batches, LEAD/LAG + // classifies the I-row as OPCODE=2 (OpTypeInsert) because its D-row was in + // the previous batch. We promote it — but only if no opTypeUpdateAfter at + // the same CSN also matches, to avoid misidentifying a concurrent standalone + // INSERT (same COMMITSEQ, higher IntentSeq) as the UPDATE after-image. + matchIdx := -1 + for i, ev := range rawEvents { + if ev.Operation == opTypeUpdateAfter && + ev.CSN.Equal(pending.CSN) && ev.IntentSeq > pending.IntentSeq { + matchIdx = i + break // opTypeUpdateAfter is the authoritative match; no need to scan further + } + } + if matchIdx < 0 { + // No opTypeUpdateAfter found; accept OpTypeInsert as fallback (cross-poll + // D+I split where LEAD/LAG downgraded the I-row to standalone INSERT). + for i, ev := range rawEvents { + if ev.Operation == OpTypeInsert && + ev.CSN.Equal(pending.CSN) && ev.IntentSeq > pending.IntentSeq { + matchIdx = i + rawEvents[i].Operation = opTypeUpdateAfter // ensure pairOpcodeEvents merges it + break + } + } + } + if matchIdx >= 0 { + // Insert the before-image immediately before its matching after-image. + rawEvents = append(rawEvents, ChangeEvent{}) // grow by 1 + copy(rawEvents[matchIdx+1:], rawEvents[matchIdx:]) + rawEvents[matchIdx] = *pending + } else { + // The expected opTypeUpdateAfter row never arrived. This "shouldn't happen" + // with DB2 SQL Replication but would silently drop one UPDATE event if it + // does. Log a warning so operators can correlate in IBMSNAP_APPLYTRACE. + s.log.Warnf("db2 streamer: unmatched UPDATE before-image for table %q at CSN %s — expected opTypeUpdateAfter did not arrive; one UPDATE event dropped", + tableKey, pending.CSN) + } + delete(s.pendingBeforeByTable, tableKey) // clear whether matched or not + } + // If the last raw event is an unmatched opTypeUpdateBefore, hold it for + // the next poll. pairOpcodeEvents will see a complete pair next time. + if len(rawEvents) > 0 && rawEvents[len(rawEvents)-1].Operation == opTypeUpdateBefore { + ev := rawEvents[len(rawEvents)-1] + s.pendingBeforeByTable[tableKey] = &ev + rawEvents = rawEvents[:len(rawEvents)-1] + } + s.pendingMu.Unlock() + + return pairOpcodeEvents(rawEvents), rawCount, nil +} + +// pairOpcodeEvents merges consecutive opTypeUpdateBefore + opTypeUpdateAfter pairs +// (produced by the LEAD/LAG query) into a single OpTypeUpdate event with BeforeData +// populated. Pairs must be consecutive and share the same CSN (guaranteed by the +// LEAD/LAG window function and computeSafeCSN pagination). +// +// Cross-batch D+I pairs are handled upstream: pollChangeTable injects the +// pending before-image at the front and strips the trailing before-image before +// calling this function, so pairOpcodeEvents only sees complete pairs or +// non-update events. +func pairOpcodeEvents(events []ChangeEvent) []ChangeEvent { + if len(events) == 0 { + return events + } + // Fast path: skip the allocation when no UPDATE before-images are present. + // opTypeUpdateBefore is rare relative to INSERT/DELETE events; avoid the + // make([]ChangeEvent, ...) cost for the common no-pairs case. + hasPairs := false + for i := range events { + if events[i].Operation == opTypeUpdateBefore { + hasPairs = true + break + } + } + if !hasPairs { + return events + } + + out := make([]ChangeEvent, 0, len(events)) + for i := 0; i < len(events); i++ { + ev := events[i] + if ev.Operation == opTypeUpdateBefore { + if i+1 < len(events) { + next := events[i+1] + if next.Operation == opTypeUpdateAfter && next.CSN.Equal(ev.CSN) { + out = append(out, ChangeEvent{ + Schema: next.Schema, + Table: next.Table, + QualKey: next.QualKey, + Operation: OpTypeUpdate, + CSN: next.CSN, + IntentSeq: next.IntentSeq, + Timestamp: next.Timestamp, + Data: next.Data, + BeforeData: ev.Data, + }) + i++ // skip the after-image row + continue + } + } + // Orphaned before-image with no matching after-image. + // pollChangeTable strips the trailing case into pendingBeforeByTable; + // this guard discards a mid-stream orphan (DB2 anomaly) rather than + // leaking the internal opTypeUpdateBefore opcode downstream. + continue + } + // Orphaned after-image (DB2 anomaly: CSN mismatch with preceding before-image). + // We have the new row state but not the before state; emit as INSERT so + // consumers see the current row value rather than an unrecognised opcode. + if ev.Operation == opTypeUpdateAfter { + ev.Operation = OpTypeInsert + } + out = append(out, ev) + } + return out +} + +// sortEventsByCSN sorts events in-place by (CSN, IntentSeq) and returns the slice. +// (CSN, IntentSeq) is a total order — DB2 guarantees IntentSeq is unique within +// a CommitSeq — so SortFunc (unstable, O(n log n)) is safe and faster than SortStableFunc. +func (*Streamer) sortEventsByCSN(events []ChangeEvent) []ChangeEvent { + slices.SortFunc(events, func(a, b ChangeEvent) int { + if c := a.CSN.Compare(b.CSN); c != 0 { + return c + } + return cmp.Compare(a.IntentSeq, b.IntentSeq) + }) + return events +} + +// getString extracts a string value from a scanned any pointer. +func getString(dest any) string { + value := *(dest.(*any)) + if value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case []byte: + return string(v) + default: + return fmt.Sprintf("%v", v) + } +} + +// getBytes extracts raw bytes from a scanned any pointer. +func getBytes(dest any) []byte { + value := *(dest.(*any)) + if value == nil { + return nil + } + switch v := value.(type) { + case []byte: + return v + case string: + return []byte(v) + default: + return nil + } +} + +// getBitDataBytes extracts raw bytes from a scanned any pointer for a DB2 +// FOR BIT DATA column. The DB2 CLI driver returns VARCHAR(n) FOR BIT DATA +// columns as hex-encoded strings (e.g. "00000021B8...") rather than as binary +// []byte. This function hex-decodes strings so callers always receive the +// actual binary content. +func getBitDataBytes(dest any) []byte { + value := *(dest.(*any)) + if value == nil { + return nil + } + switch v := value.(type) { + case []byte: + return v + case string: + b, err := hex.DecodeString(v) + if err != nil { + // Not a valid hex string — return raw bytes as fallback. + return []byte(v) + } + return b + default: + return nil + } +} + +// getInt64 extracts an int64 from a scanned any pointer. +// IBMSNAP_INTENTSEQ is typed CHAR(n) FOR BIT DATA on DB2 LUW; the CLI driver +// returns it as []byte. We interpret the bytes as a big-endian binary offset +// (up to the first 8 bytes) so that composite (CSN, IntentSeq) pagination +// correctly resumes mid-transaction without skipping or duplicating rows. +func getInt64(dest any) int64 { + value := *(dest.(*any)) + if value == nil { + return 0 + } + switch v := value.(type) { + case int64: + return v + case int32: + return int64(v) + case int: + return int64(v) + case float64: + return int64(v) + case string: + // Some drivers return numeric columns as strings. + if n, err := strconv.ParseInt(v, 10, 64); err == nil { + return n + } + // IBMSNAP_INTENTSEQ may arrive as a raw-byte string from some driver + // versions. Decode as big-endian binary, same as the []byte case. + // encoding/binary.BigEndian.Uint64 is the idiomatic Go approach. + // Note: the bytes package (bytes.NewReader, etc.) does not provide + // integer-from-slice helpers; encoding/binary is the correct stdlib path. + b := []byte(v) + if len(b) >= 8 { + return int64(binary.BigEndian.Uint64(b[:8])) + } + var n int64 + for _, bv := range b { + n = (n << 8) | int64(bv) + } + return n + case []byte: + // IBMSNAP_INTENTSEQ arrives as []byte (CHAR(10) FOR BIT DATA). Decode + // the leading 8 bytes as a big-endian uint64 for comparison purposes. + if len(v) >= 8 { + return int64(binary.BigEndian.Uint64(v[:8])) + } + // Shorter than 8 bytes (should not happen for IBMSNAP_INTENTSEQ, which is + // always CHAR(10)): accumulate as big-endian integer. + var n int64 + for _, b := range v { + n = (n << 8) | int64(b) + } + return n + default: + return 0 + } +} + +// getTime extracts a time.Time from a scanned any pointer. +// The DB2 CLI driver returns TIMESTAMP columns as strings via SQL_C_CHAR +// binding, so we fall back to string parsing when the native time.Time +// assertion fails. +func getTime(dest any) time.Time { + value := *(dest.(*any)) + if value == nil { + return time.Time{} + } + if t, ok := value.(time.Time); ok { + return t + } + if s, ok := value.(string); ok { + // DB2 native format: "2006-01-02 15:04:05.999999" + if t, err := time.Parse("2006-01-02 15:04:05.999999", s); err == nil { + return t + } + if t, err := time.Parse(time.RFC3339, s); err == nil { + return t + } + } + return time.Time{} +} diff --git a/internal/impl/db2/replication/stream_db_test.go b/internal/impl/db2/replication/stream_db_test.go new file mode 100644 index 0000000000..dc49229b8e --- /dev/null +++ b/internal/impl/db2/replication/stream_db_test.go @@ -0,0 +1,1213 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "context" + "database/sql/driver" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// Helper function tests (no DB) +// --------------------------------------------------------------------------- + +func TestGetStringHelper(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + value any + want string + }{ + {"nil", nil, ""}, + {"string", "hello", "hello"}, + {"[]byte", []byte("world"), "world"}, + {"int64", int64(42), "42"}, + {"float64", float64(3.14), "3.14"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + v := tc.value + dest := &v + assert.Equal(t, tc.want, getString(dest)) + }) + } +} + +func TestGetTimeHelper(t *testing.T) { + t.Parallel() + + now := time.Now().Truncate(time.Second) + + tests := []struct { + name string + value any + wantZero bool + wantTime time.Time + }{ + {"nil", nil, true, time.Time{}}, + {"time.Time", now, false, now}, + {"string (not a time)", "not-a-time", true, time.Time{}}, + {"int64 (not a time)", int64(12345), true, time.Time{}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + v := tc.value + result := getTime(&v) + if tc.wantZero { + assert.True(t, result.IsZero()) + } else { + assert.Equal(t, tc.wantTime, result) + } + }) + } +} + +func TestStreamConfigAsncdcSchema(t *testing.T) { + t.Parallel() + + c := &StreamConfig{} + assert.Equal(t, "ASNCDC", c.asncdcSchema(), "default schema") + + c.AsnCDCSchema = "CUSTOM" + assert.Equal(t, "CUSTOM", c.asncdcSchema(), "explicit schema") +} + +// --------------------------------------------------------------------------- +// DB-dependent tests +// --------------------------------------------------------------------------- + +func TestGetUpperBound(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rows [][]driver.Value + wantCSN uint64 + wantNull bool + wantErr bool + }{ + { + name: "normal SYNCHPOINT", + rows: [][]driver.Value{{[]byte{0, 0, 0, 0, 0, 0, 0x30, 0x39}}}, // 12345 + wantCSN: 12345, + }, + { + name: "nil SYNCHPOINT returns NullCSN", + rows: [][]driver.Value{{nil}}, + wantNull: true, + }, + { + name: "query error propagates", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if tc.wantErr { + return nil, nil, fmt.Errorf("SYNCHPOINT query failed") + } + return []string{"MAX(SYNCHPOINT)"}, tc.rows, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"MYSCHEMA"}}, Version{}) + csn, err := s.getUpperBound(context.Background()) + + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.wantNull { + assert.True(t, csn.IsNull()) + } else { + assert.Equal(t, tc.wantCSN, csn.Uint64()) + } + }) + } +} + +func TestInitialize(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rows [][]driver.Value + tables []string + wantErr bool + errSubstr string + }{ + { + name: "single table registered", + rows: [][]driver.Value{{"MYSCHEMA", "EMPLOYEES", "ASNCDC", "EMPLOYEES_CT"}}, + tables: []string{"EMPLOYEES"}, + }, + { + name: "case-insensitive match", + rows: [][]driver.Value{{"MYSCHEMA", "employees", "ASNCDC", "EMPLOYEES_CT"}}, + tables: []string{"EMPLOYEES"}, + }, + { + name: "multiple tables", + rows: [][]driver.Value{ + {"MYSCHEMA", "EMPLOYEES", "ASNCDC", "EMPLOYEES_CT"}, + {"MYSCHEMA", "ORDERS", "ASNCDC", "ORDERS_CT"}, + }, + tables: []string{"EMPLOYEES", "ORDERS"}, + }, + { + name: "table not registered returns error", + rows: [][]driver.Value{{"MYSCHEMA", "EMPLOYEES", "ASNCDC", "EMPLOYEES_CT"}}, + tables: []string{"EMPLOYEES", "MISSING_TABLE"}, + wantErr: true, + errSubstr: "MISSING_TABLE", + }, + { + name: "query error propagates", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if tc.wantErr && tc.rows == nil { + return nil, nil, fmt.Errorf("registration query failed") + } + return []string{"SOURCE_OWNER", "SOURCE_TABLE", "CD_OWNER", "CD_TABLE"}, tc.rows, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: tc.tables, + }, Version{}) + + err := s.Initialize(context.Background()) + if tc.wantErr { + require.Error(t, err) + if tc.errSubstr != "" { + assert.Contains(t, err.Error(), tc.errSubstr) + } + return + } + require.NoError(t, err) + + // Verify change tables were populated (keys are schema-qualified). + for _, table := range tc.tables { + _, ok := s.changeTables["MYSCHEMA."+strings.ToUpper(table)] + assert.True(t, ok, "change table for %s should be registered", table) + } + }) + } +} + +func TestPollChangeTable(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + + tests := []struct { + name string + rows [][]driver.Value + columns []string + wantCount int + wantErr bool + errSubstr string + }{ + { + name: "one insert event", + columns: []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "EMP_ID", "EMP_NAME", + }, + rows: [][]driver.Value{{ + []byte{0, 0, 0, 0, 0, 0, 0x30, 0x39}, // CSN=12345 + int64(1), // INTENTSEQ + "I", // INSERT + ts, // LOGMARKER + int64(42), // EMP_ID + "Alice", // EMP_NAME + }}, + wantCount: 1, + }, + { + name: "delete event", + columns: []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", + }, + rows: [][]driver.Value{{ + []byte{0, 0, 0, 0, 0, 0, 0, 1}, + int64(1), + "D", + ts, + int64(1), + }}, + wantCount: 1, + }, + { + name: "unknown operation skipped", + columns: []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", + }, + rows: [][]driver.Value{{ + []byte{0, 0, 0, 0, 0, 0, 0, 1}, + int64(1), + "Z", // unknown operation + ts, + int64(1), + }}, + wantCount: 0, + }, + { + name: "missing IBMSNAP_ columns returns error", + columns: []string{"COL1", "COL2"}, + rows: [][]driver.Value{{"val1", "val2"}}, + wantErr: true, + errSubstr: "missing required IBMSNAP_", + }, + { + name: "query error propagates", + wantErr: true, + }, + { + name: "empty result set", + columns: []string{"IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER"}, + rows: nil, + wantCount: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if tc.wantErr && tc.columns == nil { + return nil, nil, fmt.Errorf("change table query failed") + } + return tc.columns, tc.rows, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 100, + }, Version{}) + + events, _, err := s.pollChangeTable(context.Background(), "MYSCHEMA.EMPLOYEES", "MYSCHEMA", "EMPLOYEES", "ASNCDC.EMPLOYEES_CT", + NewCSN(0), 0, NewCSN(99999), 0) + + if tc.wantErr { + require.Error(t, err) + if tc.errSubstr != "" { + assert.Contains(t, err.Error(), tc.errSubstr) + } + return + } + require.NoError(t, err) + assert.Len(t, events, tc.wantCount) + }) + } +} + +func TestPollChanges(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + afterCSN CSN + upperCSN []byte // bytes returned for SYNCHPOINT; nil = NullCSN + changeRows [][]driver.Value + changeCols []string + wantCount int + wantErr bool + }{ + { + name: "no new changes (upper <= after)", + afterCSN: NewCSN(100), + upperCSN: []byte{0, 0, 0, 0, 0, 0, 0, 99}, // 99 < 100 + wantCount: 0, + }, + { + name: "new events returned", + afterCSN: NewCSN(0), + upperCSN: []byte{0, 0, 0, 0, 0, 0, 0x30, 0x39}, // 12345 + changeCols: []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", + }, + changeRows: [][]driver.Value{{ + []byte{0, 0, 0, 0, 0, 0, 0, 50}, + int64(1), + "I", + time.Now(), + int64(1), + }}, + wantCount: 1, + }, + { + name: "upper bound query error", + afterCSN: NewCSN(0), + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + callCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + callCount++ + if tc.wantErr && callCount == 1 { + return nil, nil, fmt.Errorf("SYNCHPOINT error") + } + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{tc.upperCSN}}, nil + } + // Change table query + return tc.changeCols, tc.changeRows, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 100, + }, Version{}) + + if !tc.wantErr { + // Register a fake change table. + s.changeTables["MYSCHEMA.EMPLOYEES"] = "ASNCDC.EMPLOYEES_CT" + } + + events, maxCSN, _, err := s.pollChanges(context.Background(), tc.afterCSN, nil) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Len(t, events, tc.wantCount) + if tc.wantCount > 0 { + assert.True(t, maxCSN.Greater(tc.afterCSN) || maxCSN.Equal(tc.afterCSN)) + } + }) + } +} + +// TestPollChangesLargeTransactionCompositePageination verifies that when a full batch +// contains rows all sharing a single CSN (large transaction), pollChanges populates +// newIntentSeqs so the next call uses composite (CSN, IntentSeq) pagination rather +// than resetting to a plain "COMMITSEQ > N" predicate that would skip remaining rows. +func TestPollChangesLargeTransactionCompositePagination(t *testing.T) { + t.Parallel() + + // PollBatchSize = 3, all 3 rows share CSN = 50 → full batch, single CSN. + // After the fix, pollChanges must return newIntentSeqs with max intentSeq = 3 + // so the next poll uses "(COMMITSEQ > 50 OR (COMMITSEQ = 50 AND INTENTSEQ > 3))". + upperCSN := []byte{0, 0, 0, 0, 0, 0, 0, 100} + changeCols := []string{"IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", "ID"} + sharedCSN := []byte{0, 0, 0, 0, 0, 0, 0, 50} + changeRows := [][]driver.Value{ + {sharedCSN, int64(1), "I", time.Now(), int64(1)}, + {sharedCSN, int64(2), "I", time.Now(), int64(2)}, + {sharedCSN, int64(3), "I", time.Now(), int64(3)}, + } + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{upperCSN}}, nil + } + return changeCols, changeRows, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 3, // full batch when all 3 rows returned + }, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + events, returnCSN, newIntentSeqs, err := s.pollChanges(context.Background(), NewCSN(0), nil) + require.NoError(t, err) + require.Len(t, events, 3, "all 3 rows must be returned") + assert.Equal(t, uint64(50), returnCSN.Uint64(), "returnCSN must equal the shared CSN") + + // The critical assertion: newIntentSeqs must be non-nil and populated so the + // next poll uses composite pagination. Before the fix this was nil, causing + // any remaining rows at CSN=50 to be skipped. + require.NotNil(t, newIntentSeqs, "newIntentSeqs must be non-nil for large-transaction composite pagination") + assert.Equal(t, int64(3), newIntentSeqs["MYSCHEMA.T"], "max intentSeq at returnCSN must be 3") +} + +func TestStreamContextCancel(t *testing.T) { + t.Parallel() + + // Stream should exit cleanly when context is cancelled. + callCount := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + callCount++ + // Return a SYNCHPOINT that's always less than afterCSN to avoid + // actually processing events. Stream will back off and then exit via ctx. + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{{nil}}, nil + }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + BackoffInterval: 10 * time.Millisecond, + PollBatchSize: 100, + StartingCSN: NewCSN(0), + }, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + done := make(chan error, 1) + go func() { + done <- s.Stream(ctx, func(_ ChangeEvent) error { return nil }) + }() + + // Cancel after a short delay to let the Stream loop spin at least once. + time.Sleep(30 * time.Millisecond) + cancel() + + err := <-done + assert.ErrorIs(t, err, context.Canceled) +} + +// TestPollChangeTableDeleteEvent verifies that a 'D' operation row is decoded +// into a ChangeEvent with OpTypeDelete and the row data in the Data map. +// In DB2 LUW SQL Replication, DELETE rows are written to the change table with +// IBMSNAP_OPERATION='D'. The before-image column values are present in the row. +func TestPollChangeTableDeleteEvent(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + csnBytes := []byte{0, 0, 0, 0, 0, 0, 0, 42} // CSN = 42 + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "EMP_ID", "EMP_NAME", + }, [][]driver.Value{{ + csnBytes, int64(1), "D", ts, + int64(99), "Alice", + }}, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"MYSCHEMA"}, PollBatchSize: 100}, Version{}) + events, _, err := s.pollChangeTable(context.Background(), "MYSCHEMA.EMPLOYEES", "MYSCHEMA", "EMPLOYEES", "ASNCDC.EMPLOYEES_CT", + NewCSN(0), 0, NewCSN(99999), 0) + + require.NoError(t, err) + require.Len(t, events, 1) + + evt := events[0] + assert.Equal(t, OpTypeDelete, evt.Operation) + assert.Equal(t, uint64(42), evt.CSN.Uint64()) + assert.Equal(t, int64(1), evt.IntentSeq) + assert.Equal(t, ts, evt.Timestamp) + assert.Equal(t, "MYSCHEMA", evt.Schema) + assert.Equal(t, "EMPLOYEES", evt.Table) + + // Data map must contain row columns but not any IBMSNAP_* control columns. + assert.Equal(t, int64(99), evt.Data["EMP_ID"]) + assert.Equal(t, "Alice", evt.Data["EMP_NAME"]) + _, hasIBM := evt.Data["IBMSNAP_OPERATION"] + assert.False(t, hasIBM, "IBMSNAP_* control columns must not appear in Data") +} + +// TestPollChangeTableUpdateAsDIPair verifies that pollChangeTable correctly maps +// IBMSNAP_OPERATION='D' and 'I' rows to OpTypeDelete and OpTypeInsert when no +// IBMSNAP_OPCODE column is present (the non-LEAD/LAG fallback path). +// +// In production the buildPollQuery LEAD/LAG subquery emits IBMSNAP_OPCODE 3/4 for +// update pairs; pairOpcodeEvents then merges them into a single OpTypeUpdate event. +// This test validates the fallback: when the result set has no IBMSNAP_OPCODE column, +// pollChangeTable maps raw D/I IBMSNAP_OPERATION codes to OpTypeDelete/OpTypeInsert. +func TestPollChangeTableUpdateAsDIPair(t *testing.T) { + t.Parallel() + + sharedCSN := []byte{0, 0, 0, 0, 0, 0, 0, 77} // both rows share this CSN + ts := time.Now().Truncate(time.Second) + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", "SALARY", + }, [][]driver.Value{ + // D row: before-image (intentseq=1, old salary) + {sharedCSN, int64(1), "D", ts, int64(5), int64(50000)}, + // I row: after-image (intentseq=2, new salary) + {sharedCSN, int64(2), "I", ts, int64(5), int64(60000)}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"MYSCHEMA"}, PollBatchSize: 100}, Version{}) + events, _, err := s.pollChangeTable(context.Background(), "MYSCHEMA.EMPLOYEES", "MYSCHEMA", "EMPLOYEES", "ASNCDC.EMPLOYEES_CT", + NewCSN(0), 0, NewCSN(99999), 0) + + require.NoError(t, err) + require.Len(t, events, 2, "UPDATE must produce exactly 2 events (D + I)") + + // First event: delete with before-image salary. + assert.Equal(t, OpTypeDelete, events[0].Operation) + assert.Equal(t, uint64(77), events[0].CSN.Uint64()) + assert.Equal(t, int64(1), events[0].IntentSeq) + assert.Equal(t, int64(50000), events[0].Data["SALARY"]) + + // Second event: insert with after-image salary. + assert.Equal(t, OpTypeInsert, events[1].Operation) + assert.Equal(t, uint64(77), events[1].CSN.Uint64()) + assert.Equal(t, int64(2), events[1].IntentSeq) + assert.Equal(t, int64(60000), events[1].Data["SALARY"]) + + // Both events must share the same CSN — this is what identifies them as + // the two halves of a single UPDATE operation. + assert.True(t, events[0].CSN.Equal(events[1].CSN), + "D and I events for an UPDATE must share the same IBMSNAP_COMMITSEQ") +} + +// TestPollChangesMultiTableCSNOrdering verifies that events from multiple change +// tables are merged and returned in strict (CSN, IntentSeq) order regardless of +// which table they originated from. +// +// Two change tables (ORDERS_CT with rows at CSN=10,30 and EMPLOYEES_CT with a +// row at CSN=20) must be merged so the output order is CSN 10 → 20 → 30. +func TestPollChangesMultiTableCSNOrdering(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 50}}, // upper bound = 50 + }, nil + } + cols := []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", "ID", + } + if strings.Contains(q, "ORDERS_CT") { + return cols, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 10}, int64(1), "I", ts, int64(1)}, + {[]byte{0, 0, 0, 0, 0, 0, 0, 30}, int64(1), "I", ts, int64(3)}, + }, nil + } + // EMPLOYEES_CT + return cols, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 20}, int64(1), "I", ts, int64(2)}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + PollBatchSize: 100, + }, Version{}) + s.changeTables["MYSCHEMA.ORDERS"] = "ASNCDC.ORDERS_CT" + s.changeTables["MYSCHEMA.EMPLOYEES"] = "ASNCDC.EMPLOYEES_CT" + + events, maxCSN, _, err := s.pollChanges(context.Background(), NewCSN(0), nil) + + require.NoError(t, err) + require.Len(t, events, 3) + + assert.Equal(t, uint64(10), events[0].CSN.Uint64(), "first event must have lowest CSN") + assert.Equal(t, uint64(20), events[1].CSN.Uint64(), "second event must have middle CSN") + assert.Equal(t, uint64(30), events[2].CSN.Uint64(), "third event must have highest CSN") + assert.Equal(t, uint64(30), maxCSN.Uint64()) +} + +// TestInitializeCaseSensitiveTableNames verifies that Initialize handles mixed- +// case table names from IBMSNAP_REGISTER correctly. DB2 may store table names +// in the register with different casing; the connector matches them +// case-insensitively against the configured tables list. +func TestInitializeCaseSensitiveTableNames(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + registeredAs string // value in SOURCE_TABLE column + configuredAs string // value in StreamConfig.Tables + wantRegistered bool + }{ + { + name: "exact match uppercase", + registeredAs: "EMPLOYEES", + configuredAs: "EMPLOYEES", + wantRegistered: true, + }, + { + name: "registry lowercase, config uppercase", + registeredAs: "employees", + configuredAs: "EMPLOYEES", + wantRegistered: true, + }, + { + name: "registry uppercase, config lowercase", + registeredAs: "EMPLOYEES", + configuredAs: "employees", + wantRegistered: true, + }, + { + name: "mixed case in registry", + registeredAs: "Employees", + configuredAs: "EMPLOYEES", + wantRegistered: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{"SOURCE_OWNER", "SOURCE_TABLE", "CD_OWNER", "CD_TABLE"}, + [][]driver.Value{ + {"MYSCHEMA", tc.registeredAs, "ASNCDC", "EMPLOYEES_CT"}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{tc.configuredAs}, + }, Version{}) + + err := s.Initialize(context.Background()) + if tc.wantRegistered { + require.NoError(t, err) + assert.Len(t, s.changeTables, 1) + } else { + require.Error(t, err) + } + }) + } +} + +// TestInitializeCustomCDCSchema verifies that a non-default CDC schema +// (AsnCDCSchema config field) is used in the IBMSNAP_REGISTER query instead of +// the hardcoded default. This allows operators who installed SQL Replication +// under a custom schema to use the connector. +func TestInitializeCustomCDCSchema(t *testing.T) { + t.Parallel() + + var capturedQueries []string + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + capturedQueries = append(capturedQueries, q) + if strings.Contains(q, "SOURCE_OWNER") { + return []string{"SOURCE_OWNER", "SOURCE_TABLE", "CD_OWNER", "CD_TABLE"}, + [][]driver.Value{ + {"MYSCHEMA", "EMPLOYEES", "CDCSCHEMA", "EMPLOYEES_CT"}, + }, nil + } + // detectCommitSeqByteLen queries SYSCAT.COLUMNS — return a valid length. + return []string{"LENGTH"}, [][]driver.Value{{int64(16)}}, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + Tables: []string{"EMPLOYEES"}, + AsnCDCSchema: "CDCSCHEMA", // non-default CDC schema + }, Version{}) + + err := s.Initialize(context.Background()) + require.NoError(t, err) + + allQueries := strings.Join(capturedQueries, "\n") + assert.Contains(t, allQueries, "CDCSCHEMA.IBMSNAP_REGISTER", + "Initialize must query the custom CDC schema, not the hardcoded default") + assert.NotContains(t, allQueries, "ASNCDC.IBMSNAP_REGISTER", + "default schema name must not appear when a custom schema is configured") +} + +func TestStreamHandlerError(t *testing.T) { + t.Parallel() + + ts := time.Now() + callCount := 0 + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + callCount++ + if strings.Contains(q, "SYNCHPOINT") { + // Return a large SYNCHPOINT so pollChanges sees new events. + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0xFF, 0xFF}}, + }, nil + } + // Change table returns one event. + return []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", + }, [][]driver.Value{{ + []byte{0, 0, 0, 0, 0, 0, 0, 50}, + int64(1), + "I", + ts, + int64(1), + }}, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + BackoffInterval: time.Millisecond, + PollBatchSize: 100, + StartingCSN: NewCSN(0), + }, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + err := s.Stream(context.Background(), func(_ ChangeEvent) error { + return fmt.Errorf("handler refused event") + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "handler refused event") +} + +// TestPollChangeTableFullBatchWithPairedUpdates verifies that tableResult.full is +// set from the raw SQL row count BEFORE pairOpcodeEvents merging (C1 fix). +// +// Scenario: PollBatchSize=4, feed exactly 4 rows as 2 D+I pairs (opcodes 3+4). +// pairOpcodeEvents reduces them to 2 OpTypeUpdate events. tableResult.full must +// still be true because the raw row count was 4 == PollBatchSize, even though +// len(events) == 2 after merging. Without C1, len(events)==2 != batchSize==4 +// → full=false → computeSafeCSN would not protect the watermark. +func TestPollChangeTableFullBatchWithPairedUpdates(t *testing.T) { + t.Parallel() + + sharedCSN := []byte{0, 0, 0, 0, 0, 0, 0, 42} + ts := time.Now().Truncate(time.Second) + + // 4 rows: 2 D+I pairs using IBMSNAP_OPCODE (3=update-before, 4=update-after). + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 99}}, + }, nil + } + return []string{ + "IBMSNAP_OPCODE", + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", "VAL", + }, [][]driver.Value{ + {int64(3), sharedCSN, int64(1), "D", ts, int64(1), int64(100)}, // pair 1 before + {int64(4), sharedCSN, int64(2), "I", ts, int64(1), int64(101)}, // pair 1 after + {int64(3), sharedCSN, int64(3), "D", ts, int64(2), int64(200)}, // pair 2 before + {int64(4), sharedCSN, int64(4), "I", ts, int64(2), int64(201)}, // pair 2 after + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"MYSCHEMA"}, PollBatchSize: 4}, Version{}) + s.changeTables["MYSCHEMA.T"] = "ASNCDC.T_CT" + + events, rawCount, err := s.pollChangeTable(context.Background(), "MYSCHEMA.T", "MYSCHEMA", "T", "ASNCDC.T_CT", + NewCSN(0), 0, NewCSN(99), 0) + require.NoError(t, err) + require.Len(t, events, 2, "two D+I pairs must merge to 2 update events") + assert.Equal(t, 4, rawCount, "raw row count must be 4 even though only 2 events after merging") + assert.Equal(t, OpTypeUpdate, events[0].Operation) + assert.Equal(t, OpTypeUpdate, events[1].Operation) + + // Verify via pollChanges that full=true is propagated correctly. + events2, _, _, err := s.pollChanges(context.Background(), NewCSN(0), nil) + require.NoError(t, err) + require.Len(t, events2, 2, "pollChanges must return 2 merged update events") +} + +// TestPollChangeTableMultipleDeletesSameTable verifies that multiple DELETE +// operations on the same table within one poll batch each produce an +// independent OpTypeDelete event with the correct before-image data. +// +// This corresponds to Debezium's deleteWithoutTombstone test, which executes +// "DELETE FROM tableB" to remove N rows in a single transaction and asserts +// that each delete event carries the deleted row's column values in the +// before-image. In our model the before-image is in Data (not BeforeData) +// for DELETE events — consistent with the Debezium "before" field mapping. +func TestPollChangeTableMultipleDeletesSameTable(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + csnBytes := []byte{0, 0, 0, 0, 0, 0, 0, 10} // single transaction CSN + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", "COLB", + }, [][]driver.Value{ + // 5 rows deleted in the same transaction (same CSN, ascending IntentSeq) + {csnBytes, int64(1), "D", ts, int64(10), "b"}, + {csnBytes, int64(2), "D", ts, int64(11), "b"}, + {csnBytes, int64(3), "D", ts, int64(12), "b"}, + {csnBytes, int64(4), "D", ts, int64(13), "b"}, + {csnBytes, int64(5), "D", ts, int64(14), "b"}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"DB2INST1"}, PollBatchSize: 100}, Version{}) + events, _, err := s.pollChangeTable(context.Background(), "DB2INST1.TABLEB", "DB2INST1", "TABLEB", "ASNCDC.TABLEB_CT", + NewCSN(0), 0, NewCSN(99999), 0) + + require.NoError(t, err) + require.Len(t, events, 5, "each deleted row must produce one OpTypeDelete event") + + for i, evt := range events { + assert.Equal(t, OpTypeDelete, evt.Operation, "event %d must be a delete", i) + assert.Equal(t, uint64(10), evt.CSN.Uint64(), "all deletes share the same transaction CSN") + assert.Equal(t, int64(i+1), evt.IntentSeq, "intent seq must be sequential") + assert.Equal(t, int64(10+i), evt.Data["ID"], "before-image ID must match deleted row") + assert.Equal(t, "b", evt.Data["COLB"], "before-image COLB must match deleted row") + assert.Nil(t, evt.BeforeData, "DELETE events must have nil BeforeData (before-image is in Data)") + } +} + +// TestPollChangeTablePrimaryKeyUpdate verifies that an UPDATE which changes the +// primary key column is correctly captured as a D+I pair (not as a phantom delete +// followed by an unrelated insert). +// +// This mirrors Debezium's updatePrimaryKey test: UPDATE tablea SET id=100 WHERE id=1 +// produces a D row with the old pk value and an I row with the new pk value, both +// sharing the same IBMSNAP_COMMITSEQ. The connector emits these as OpTypeDelete +// (before-image, old pk=1) and OpTypeInsert (after-image, new pk=100). +// +// Note: DB2 LUW SQL Replication does NOT encode PK updates differently from +// regular updates. An UPDATE to a PK column still produces a D+I pair in the +// change table with the same COMMITSEQ. pairOpcodeEvents then merges the pair +// into OpTypeUpdate with BeforeData populated. +func TestPollChangeTablePrimaryKeyUpdate(t *testing.T) { + t.Parallel() + + sharedCSN := []byte{0, 0, 0, 0, 0, 0, 0, 77} + ts := time.Now().Truncate(time.Second) + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + // IBMSNAP_OPCODE=3 (update-before) + 4 (update-after) detect a PK-update pair. + return []string{ + "IBMSNAP_OPCODE", + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", "COLA", + }, [][]driver.Value{ + {int64(3), sharedCSN, int64(1), "D", ts, int64(1), "a"}, // before: id=1 + {int64(4), sharedCSN, int64(2), "I", ts, int64(100), "a"}, // after: id=100 + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"DB2INST1"}, PollBatchSize: 100}, Version{}) + events, _, err := s.pollChangeTable(context.Background(), "DB2INST1.TABLEA", "DB2INST1", "TABLEA", "ASNCDC.TABLEA_CT", + NewCSN(0), 0, NewCSN(99999), 0) + + require.NoError(t, err) + require.Len(t, events, 1, "D+I pair for PK update must merge into a single OpTypeUpdate event") + + evt := events[0] + assert.Equal(t, OpTypeUpdate, evt.Operation) + assert.Equal(t, uint64(77), evt.CSN.Uint64()) + + // After-image: new pk=100 + assert.Equal(t, int64(100), evt.Data["ID"], "Data must contain the after-image (new pk)") + assert.Equal(t, "a", evt.Data["COLA"]) + + // Before-image: old pk=1 + require.NotNil(t, evt.BeforeData, "BeforeData must be populated for PK update") + assert.Equal(t, int64(1), evt.BeforeData["ID"], "BeforeData must contain the before-image (old pk)") + assert.Equal(t, "a", evt.BeforeData["COLA"]) +} + +// TestPollChangeTableNullColumnsInCDCRows verifies that NULL column values in +// CDC change table rows (not snapshot rows) are represented as nil in the +// Data map. This is distinct from TestSnapshotNULLColumnValues which tests +// snapshot null handling. Change table rows also carry NULL for optional columns. +// +// Scenario: UPDATE employees SET manager_id = NULL WHERE id = 5. +// The after-image I row has manager_id = NULL; the before-image D row had a value. +func TestPollChangeTableNullColumnsInCDCRows(t *testing.T) { + t.Parallel() + + ts := time.Now().Truncate(time.Second) + csnBytes := []byte{0, 0, 0, 0, 0, 0, 0, 55} + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{ + "IBMSNAP_OPCODE", + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", "MANAGER_ID", "DEPT", + }, [][]driver.Value{ + // before: manager_id was 42, dept was "ENG" + {int64(3), csnBytes, int64(1), "D", ts, int64(5), int64(42), "ENG"}, + // after: manager_id set to NULL, dept set to NULL + {int64(4), csnBytes, int64(2), "I", ts, int64(5), nil, nil}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"MYSCHEMA"}, PollBatchSize: 100}, Version{}) + events, _, err := s.pollChangeTable(context.Background(), "MYSCHEMA.EMPLOYEES", "MYSCHEMA", "EMPLOYEES", "ASNCDC.EMPLOYEES_CT", + NewCSN(0), 0, NewCSN(99999), 0) + + require.NoError(t, err) + require.Len(t, events, 1, "D+I pair must merge into one update event") + + evt := events[0] + assert.Equal(t, OpTypeUpdate, evt.Operation) + + // After-image: NULL columns must appear as nil (present key, nil value). + managerAfter, managerAfterExists := evt.Data["MANAGER_ID"] + assert.True(t, managerAfterExists, "MANAGER_ID must be present in Data even when NULL") + assert.Nil(t, managerAfter, "NULL after-image value must be nil in Data") + + deptAfter, deptAfterExists := evt.Data["DEPT"] + assert.True(t, deptAfterExists, "DEPT must be present in Data even when NULL") + assert.Nil(t, deptAfter, "NULL after-image value must be nil in Data") + + // Before-image: non-null values must be preserved in BeforeData. + require.NotNil(t, evt.BeforeData) + assert.Equal(t, int64(42), evt.BeforeData["MANAGER_ID"], "before-image must preserve the original manager_id") + assert.Equal(t, "ENG", evt.BeforeData["DEPT"]) +} + +// TestPollChangeTableMultipleInsertsSharedCSN verifies that N INSERT events +// sharing the same IBMSNAP_COMMITSEQ (a single transaction with N inserts) are +// all emitted in ascending IBMSNAP_INTENTSEQ order. This mirrors the Debezium +// test pattern where a loop inserts N rows in a single commit and expects N +// sequential INSERT events ordered by their position within the transaction. +func TestPollChangeTableMultipleInsertsSharedCSN(t *testing.T) { + t.Parallel() + + sharedCSN := []byte{0, 0, 0, 0, 0, 0, 0, 99} + ts := time.Now().Truncate(time.Second) + + db := openFakeDB(t, &replFakeHandlers{ + query: func(_ string, _ []driver.Value) ([]string, [][]driver.Value, error) { + return []string{ + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", "COLA", + }, [][]driver.Value{ + {sharedCSN, int64(1), "I", ts, int64(10), "a"}, + {sharedCSN, int64(2), "I", ts, int64(11), "a"}, + {sharedCSN, int64(3), "I", ts, int64(12), "a"}, + {sharedCSN, int64(4), "I", ts, int64(13), "a"}, + {sharedCSN, int64(5), "I", ts, int64(14), "a"}, + }, nil + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"DB2INST1"}, PollBatchSize: 100}, Version{}) + events, rawCount, err := s.pollChangeTable(context.Background(), "DB2INST1.TABLEA", "DB2INST1", "TABLEA", "ASNCDC.TABLEA_CT", + NewCSN(0), 0, NewCSN(99999), 0) + + require.NoError(t, err) + assert.Equal(t, 5, rawCount, "raw row count must equal the number of rows returned") + require.Len(t, events, 5, "all 5 inserts from the same transaction must be emitted") + + for i, evt := range events { + assert.Equal(t, OpTypeInsert, evt.Operation, "event %d must be an insert", i) + assert.Equal(t, uint64(99), evt.CSN.Uint64(), "all events share the transaction CSN") + assert.Equal(t, int64(i+1), evt.IntentSeq, "events must be ordered by IntentSeq") + assert.Equal(t, int64(10+i), evt.Data["ID"]) + } +} + +// TestInitializeWithTableFilter verifies that a TableFilter applied during +// Initialize restricts which registered tables are monitored. Tables that +// match the filter become CDC-monitored; those that do not are silently skipped +// even when present in IBMSNAP_REGISTER. +// +// This mirrors Debezium's testTableIncludeList/testTableExcludeList behaviour: +// the connector only tracks tables that pass the filter predicate. +func TestInitializeWithTableFilter(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + registered [][]driver.Value // rows from IBMSNAP_REGISTER + filter func(string) bool + wantTables []string + wantFiltered []string // tables that must NOT appear in changeTables + wantErr bool + }{ + { + name: "filter allows only one table", + registered: [][]driver.Value{ + {"MYSCHEMA", "TABLEA", "ASNCDC", "TABLEA_CT"}, + {"MYSCHEMA", "TABLEB", "ASNCDC", "TABLEB_CT"}, + }, + filter: func(t string) bool { return t == "TABLEA" }, + wantTables: []string{"TABLEA"}, + wantFiltered: []string{"TABLEB"}, + }, + { + name: "filter rejects all tables returns error", + registered: [][]driver.Value{ + {"MYSCHEMA", "TABLEA", "ASNCDC", "TABLEA_CT"}, + }, + filter: func(_ string) bool { return false }, + wantErr: true, + }, + { + name: "nil filter allows all tables", + registered: [][]driver.Value{ + {"MYSCHEMA", "TABLEA", "ASNCDC", "TABLEA_CT"}, + {"MYSCHEMA", "TABLEB", "ASNCDC", "TABLEB_CT"}, + }, + filter: nil, + wantTables: []string{"TABLEA", "TABLEB"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SOURCE_OWNER") { + return []string{"SOURCE_OWNER", "SOURCE_TABLE", "CD_OWNER", "CD_TABLE"}, + tc.registered, nil + } + // detectCommitSeqByteLen + return []string{"LENGTH"}, [][]driver.Value{{int64(10)}}, nil + }, + }) + + s := NewStreamer(db, StreamConfig{ + Schemas: []string{"MYSCHEMA"}, + TableFilter: tc.filter, + // Tables is empty → dynamic discovery + }, Version{}) + + err := s.Initialize(context.Background()) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + for _, table := range tc.wantTables { + _, ok := s.changeTables["MYSCHEMA."+strings.ToUpper(table)] + assert.True(t, ok, "table %s must be registered after Initialize", table) + } + for _, table := range tc.wantFiltered { + _, ok := s.changeTables["MYSCHEMA."+strings.ToUpper(table)] + assert.False(t, ok, "table %s must NOT be registered (filtered out)", table) + } + }) + } +} + +// TestPollChangeTableCrossBatchUpdatePairing verifies cross-poll D+I pair merging (C3 fix). +// +// When PollBatchSize=1 and a D+I pair straddles two polls: +// - Poll 1 returns only the D row (opcode 3). It must NOT be emitted as a delete. +// The pending before-image is held in pendingBeforeByTable. +// - Poll 2 returns only the I row (opcode 4). It must be merged with the pending +// before-image into a single OpTypeUpdate event with BeforeData populated. +func TestPollChangeTableCrossBatchUpdatePairing(t *testing.T) { + t.Parallel() + + sharedCSN := []byte{0, 0, 0, 0, 0, 0, 0, 55} + ts := time.Now().Truncate(time.Second) + + callNum := 0 + db := openFakeDB(t, &replFakeHandlers{ + query: func(q string, _ []driver.Value) ([]string, [][]driver.Value, error) { + if strings.Contains(q, "SYNCHPOINT") { + return []string{"MAX(SYNCHPOINT)"}, [][]driver.Value{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 99}}, + }, nil + } + callNum++ + cols := []string{ + "IBMSNAP_OPCODE", + "IBMSNAP_COMMITSEQ", "IBMSNAP_INTENTSEQ", "IBMSNAP_OPERATION", "IBMSNAP_LOGMARKER", + "ID", "VAL", + } + switch callNum { + case 1: + // Batch 1: only the before-image (opcode 3) + return cols, [][]driver.Value{ + {int64(3), sharedCSN, int64(1), "D", ts, int64(7), int64(10)}, + }, nil + case 2: + // Batch 2: only the after-image (opcode 4) + return cols, [][]driver.Value{ + {int64(4), sharedCSN, int64(2), "I", ts, int64(7), int64(20)}, + }, nil + default: + return cols, nil, nil + } + }, + }) + + s := NewStreamer(db, StreamConfig{Schemas: []string{"MYSCHEMA"}, PollBatchSize: 1}, Version{}) + + // Poll 1: D row (opcode 3) arrives. Must be held as pending — 0 events emitted. + events1, rawCount1, err := s.pollChangeTable(context.Background(), "MYSCHEMA.T", "MYSCHEMA", "T", "ASNCDC.T_CT", + NewCSN(0), 0, NewCSN(99), 0) + require.NoError(t, err) + assert.Empty(t, events1, "D row (opcode 3) must be held as pending, not emitted") + assert.Equal(t, 1, rawCount1, "raw count must be 1") + assert.NotNil(t, s.pendingBeforeByTable["MYSCHEMA.T"], "pending before-image must be held") + + // Poll 2: I row (opcode 4) arrives. Must be merged with pending into 1 update event. + events2, rawCount2, err := s.pollChangeTable(context.Background(), "MYSCHEMA.T", "MYSCHEMA", "T", "ASNCDC.T_CT", + NewCSN(0), 0, NewCSN(99), 0) + require.NoError(t, err) + require.Len(t, events2, 1, "pending before-image + I row must merge to 1 update event") + assert.Equal(t, 1, rawCount2, "raw count must be 1 (only the I row)") + assert.Nil(t, s.pendingBeforeByTable["MYSCHEMA.T"], "pending state must be cleared after merge") + + update := events2[0] + assert.Equal(t, OpTypeUpdate, update.Operation) + assert.Equal(t, int64(20), update.Data["VAL"], "after-image value must be in Data") + assert.Equal(t, int64(10), update.BeforeData["VAL"], "before-image value must be in BeforeData") +} diff --git a/internal/impl/db2/replication/stream_test.go b/internal/impl/db2/replication/stream_test.go new file mode 100644 index 0000000000..f18e1e3bf8 --- /dev/null +++ b/internal/impl/db2/replication/stream_test.go @@ -0,0 +1,450 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildPollQuery(t *testing.T) { + s := &Streamer{config: StreamConfig{PollBatchSize: 100}} + afterCSN := NewCSN(0) + upperCSN := NewCSN(12345) + + query := s.buildPollQuery("ASNCDC.EMPLOYEES_CT", afterCSN, 0, upperCSN) + + // New LEAD/LAG query: selects computed IBMSNAP_OPCODE column then cdc.* + assert.Contains(t, query, "IBMSNAP_OPCODE") + assert.Contains(t, query, "LEAD(cdc.IBMSNAP_OPERATION") + assert.Contains(t, query, "LAG(cdc.IBMSNAP_OPERATION") + assert.Contains(t, query, "FROM ASNCDC.EMPLOYEES_CT cdc") + assert.Contains(t, query, "cdc.IBMSNAP_COMMITSEQ >") + assert.Contains(t, query, "cdc.IBMSNAP_COMMITSEQ <=") + assert.Contains(t, query, "cdc.IBMSNAP_COMMITSEQ, cdc.IBMSNAP_INTENTSEQ") + assert.Contains(t, query, "FETCH FIRST 100 ROWS ONLY") + assert.Contains(t, query, "X'00000000000000000000'") // afterCSN = 0 + // 12345 = 0x3039 → uint64 big-endian in bytes 0-7: [0,0,0,0,0,0,0x30,0x39]; bytes 8-9: zeros + assert.Contains(t, query, "X'00000000000030390000'") // 12345 = 0x3039 +} + +func TestBuildPollQueryHexPadding(t *testing.T) { + s := &Streamer{config: StreamConfig{PollBatchSize: 1000}} + afterCSN := NewCSN(255) // 0xFF + upperCSN := NewCSN(65535) // 0xFFFF + + query := s.buildPollQuery("SCHEMA.TABLE_CT", afterCSN, 0, upperCSN) + + // uint64 big-endian in bytes 0-7; bytes 8-9 are zero padding. + // 0xFF → bytes 0-6: zero, byte 7: 0xFF, bytes 8-9: zero → "00000000000000FF0000" + assert.Contains(t, query, "X'00000000000000FF0000'") // 255 + // 0xFFFF → byte 6: 0xFF, byte 7: 0xFF, others: zero → "000000000000FFFF0000" + assert.Contains(t, query, "X'000000000000FFFF0000'") // 65535 +} + +func TestSortEventsByCSN(t *testing.T) { + s := &Streamer{} + events := []ChangeEvent{ + {CSN: NewCSN(300), IntentSeq: 1, Operation: OpTypeInsert}, + {CSN: NewCSN(100), IntentSeq: 2, Operation: OpTypeDelete}, + {CSN: NewCSN(200), IntentSeq: 1, Operation: OpTypeDelete}, + {CSN: NewCSN(100), IntentSeq: 1, Operation: OpTypeInsert}, + } + + sorted := s.sortEventsByCSN(events) + + require.Len(t, sorted, 4) + assert.Equal(t, uint64(100), sorted[0].CSN.Uint64()) + assert.Equal(t, int64(1), sorted[0].IntentSeq) + assert.Equal(t, uint64(100), sorted[1].CSN.Uint64()) + assert.Equal(t, int64(2), sorted[1].IntentSeq) + assert.Equal(t, uint64(200), sorted[2].CSN.Uint64()) + assert.Equal(t, uint64(300), sorted[3].CSN.Uint64()) +} + +func TestSortEventsByCSNEmpty(t *testing.T) { + s := &Streamer{} + result := s.sortEventsByCSN(nil) + assert.Nil(t, result) + + result = s.sortEventsByCSN([]ChangeEvent{}) + assert.Empty(t, result) +} + +func TestGetBytesHelper(t *testing.T) { + // nil pointer + var v1 any = nil + dest1 := &v1 + assert.Nil(t, getBytes(dest1)) + + // []byte value + expected := []byte{0x01, 0x02, 0x03} + var v2 any = expected + dest2 := &v2 + assert.Equal(t, expected, getBytes(dest2)) + + // string value + var v3 any = "hello" + dest3 := &v3 + assert.Equal(t, []byte("hello"), getBytes(dest3)) +} + +func TestGetInt64Helper(t *testing.T) { + var v1 any = int64(42) + assert.Equal(t, int64(42), getInt64(&v1)) + + var v2 any = int32(99) + assert.Equal(t, int64(99), getInt64(&v2)) + + var v3 any = "123" + assert.Equal(t, int64(123), getInt64(&v3)) + + var v4 any = nil + assert.Equal(t, int64(0), getInt64(&v4)) +} + +// TestComputeSafeCSN verifies that the per-table watermark uses the minimum +// safe CSN, not the global penultimate. +// +// Bug scenario: TableA is full at CSN 50, TableB is not full with events up to +// CSN 100. The old code used the global penultimate (90) as maxCSN, causing +// the next poll (COMMITSEQ > 90) to permanently skip TableA's remaining rows +// at CSN 50. The fix computes per-table safe CSNs and takes the minimum. +func TestComputeSafeCSN(t *testing.T) { + t.Parallel() + upper := NewCSN(1000) + + t.Run("full table with all-same-CSN (edge case) narrows to tableMax", func(t *testing.T) { + t.Parallel() + // TableA: full batch, all rows at CSN 50 (edge case — advance anyway). + // TableB: not full — no constraint. + results := []tableResult{ + {events: []ChangeEvent{{CSN: NewCSN(50)}, {CSN: NewCSN(50)}}, full: true}, + {events: []ChangeEvent{{CSN: NewCSN(80)}}, full: false}, + } + safe := computeSafeCSN(results, upper, NullCSN()) + assert.Equal(t, uint64(50), safe.Uint64()) + }) + + t.Run("full table with distinct penultimate CSN narrows to penultimate", func(t *testing.T) { + t.Parallel() + // TableA: full batch, rows at CSN 40 and 50 → penultimate = 40. + // TableB: not full, events up to CSN 80 → no constraint. + results := []tableResult{ + {events: []ChangeEvent{{CSN: NewCSN(40)}, {CSN: NewCSN(50)}}, full: true}, + {events: []ChangeEvent{{CSN: NewCSN(80)}}, full: false}, + } + safe := computeSafeCSN(results, upper, NullCSN()) + assert.Equal(t, uint64(40), safe.Uint64()) + }) + + t.Run("no full tables — safe CSN equals upperCSN", func(t *testing.T) { + t.Parallel() + results := []tableResult{ + {events: []ChangeEvent{{CSN: NewCSN(50)}}, full: false}, + {events: []ChangeEvent{{CSN: NewCSN(80)}}, full: false}, + } + safe := computeSafeCSN(results, upper, NullCSN()) + assert.Equal(t, upper.Uint64(), safe.Uint64()) + }) + + t.Run("two full tables — minimum per-table safe CSN wins", func(t *testing.T) { + t.Parallel() + // TableA: full, penultimate = 30. TableB: full, penultimate = 70. + // Global safe = min(30, 70) = 30. + results := []tableResult{ + {events: []ChangeEvent{{CSN: NewCSN(30)}, {CSN: NewCSN(40)}}, full: true}, + {events: []ChangeEvent{{CSN: NewCSN(70)}, {CSN: NewCSN(90)}}, full: true}, + } + safe := computeSafeCSN(results, upper, NullCSN()) + assert.Equal(t, uint64(30), safe.Uint64()) + }) + + t.Run("full table with heldCSN and no events — safe CSN capped at afterCSN", func(t *testing.T) { + t.Parallel() + // Table has only a D-row at CSN 8; the D-row was stripped to pendingBeforeByTable. + // events is empty, heldCSN = 8. afterCSN = 5. Safe must not advance past 5 so + // the matching I@8 can be fetched next poll. + held := NewCSN(8) + after := NewCSN(5) + results := []tableResult{ + {events: nil, full: true, heldCSN: &held}, + } + safe := computeSafeCSN(results, upper, after) + assert.Equal(t, uint64(5), safe.Uint64()) + }) + + t.Run("full table with heldCSN and events below it — standard penultimate logic", func(t *testing.T) { + t.Parallel() + // Table has [evt@5, D@8]; D stripped to pending. events=[evt@5], heldCSN=8. + // Penultimate logic still governs: tableSafe = 5 because there's only one event. + held := NewCSN(8) + after := NewCSN(3) + results := []tableResult{ + {events: []ChangeEvent{{CSN: NewCSN(5)}}, full: true, heldCSN: &held}, + } + safe := computeSafeCSN(results, upper, after) + assert.Equal(t, uint64(5), safe.Uint64()) + }) + + t.Run("full table with heldCSN and events at same CSN — advance to tableMax, composite pagination handles I-row", func(t *testing.T) { + t.Parallel() + // Table has [evt@8, D@8]; D stripped to pending. events=[evt@8], heldCSN=8. + // tableSafe = tableMax = 8 (all rows share one CSN → edge case advances to max). + // The matching I-row at CSN=8 remains reachable via anyFullAtReturnCSN composite + // pagination: next poll uses "(COMMITSEQ = 8 AND INTENTSEQ > maxSeen)" to fetch it. + // Capping at afterCSN=5 was the OLD behavior and caused an infinite loop when the + // full batch (PollBatchSize rows) all shared one CSN — every event was trimmed. + held := NewCSN(8) + after := NewCSN(5) + results := []tableResult{ + {events: []ChangeEvent{{CSN: NewCSN(8)}}, full: true, heldCSN: &held}, + } + safe := computeSafeCSN(results, upper, after) + assert.Equal(t, uint64(8), safe.Uint64()) + }) + + t.Run("PollBatchSize events all same CSN with heldCSN — no infinite loop regression", func(t *testing.T) { + t.Parallel() + // Regression test for: large transaction produces ≥ PollBatchSize rows at CSN=5, + // with D-row at tail (stripped to heldCSN). events = 999 rows all at CSN=5. + // Old code capped safeCSN at afterCSN=4, trimming all 999 events → infinite loop. + // New code: tableSafe = tableMax = 5; I-row reachable via composite pagination. + events := make([]ChangeEvent, 999) + for i := range events { + events[i] = ChangeEvent{CSN: NewCSN(5)} + } + held := NewCSN(5) + after := NewCSN(4) + results := []tableResult{ + {events: events, full: true, heldCSN: &held}, + } + safe := computeSafeCSN(results, upper, after) + // Must be 5, not 4 — advancing to tableMax allows all 999 events to be emitted. + assert.Equal(t, uint64(5), safe.Uint64()) + }) +} + +// TestComputeReturnCSN verifies that the watermark returned from a poll round is +// the minimum of the safe per-table ceiling and the actual max event CSN. +func TestComputeReturnCSN(t *testing.T) { + t.Parallel() + + t.Run("no full tables — capped at max event CSN not upper", func(t *testing.T) { + t.Parallel() + safe := NewCSN(1000) // safeCSN == upperCSN when no table is full + sorted := []ChangeEvent{{CSN: NewCSN(50)}, {CSN: NewCSN(80)}} + assert.Equal(t, uint64(80), computeReturnCSN(safe, sorted).Uint64()) + }) + + t.Run("full table — safe ceiling is below max event, safe wins", func(t *testing.T) { + t.Parallel() + safe := NewCSN(40) // penultimate from a full batch + sorted := []ChangeEvent{{CSN: NewCSN(30)}, {CSN: NewCSN(40)}} + assert.Equal(t, uint64(40), computeReturnCSN(safe, sorted).Uint64()) + }) + + t.Run("empty sorted slice — returns safeCSN unchanged", func(t *testing.T) { + t.Parallel() + safe := NewCSN(100) + assert.Equal(t, uint64(100), computeReturnCSN(safe, nil).Uint64()) + }) + + t.Run("single event below safe — returns event CSN", func(t *testing.T) { + t.Parallel() + safe := NewCSN(500) + sorted := []ChangeEvent{{CSN: NewCSN(200)}} + assert.Equal(t, uint64(200), computeReturnCSN(safe, sorted).Uint64()) + }) +} + +// TestTrimToSafeCSN verifies the safe-ceiling trim helper. +// +// Regression: the old inline code initialised trimIdx = len(sorted) instead of 0, +// so when every event exceeded safeCSN (heldCSN edge case where safeCSN == afterCSN) +// the loop found no safe event and the full slice was incorrectly returned — emitting +// events above the safe ceiling and permanently advancing the watermark. +func TestTrimToSafeCSN(t *testing.T) { + t.Parallel() + + t.Run("all events below safe — no trimming", func(t *testing.T) { + t.Parallel() + events := []ChangeEvent{{CSN: NewCSN(10)}, {CSN: NewCSN(20)}} + got := trimToSafeCSN(events, NewCSN(50)) + assert.Equal(t, events, got) + }) + + t.Run("some events above safe — tail trimmed", func(t *testing.T) { + t.Parallel() + events := []ChangeEvent{{CSN: NewCSN(10)}, {CSN: NewCSN(20)}, {CSN: NewCSN(30)}} + got := trimToSafeCSN(events, NewCSN(20)) + require.Len(t, got, 2) + assert.Equal(t, uint64(10), got[0].CSN.Uint64()) + assert.Equal(t, uint64(20), got[1].CSN.Uint64()) + }) + + t.Run("all events above safe (heldCSN edge case) — returns empty", func(t *testing.T) { + t.Parallel() + // Bug scenario: safeCSN == afterCSN (= 5), all events are at CSN 10, 15, 20. + // Old code (trimIdx = len(sorted)) would return all events unchanged. + // Fixed code (trimIdx = 0) returns empty slice. + events := []ChangeEvent{{CSN: NewCSN(10)}, {CSN: NewCSN(15)}, {CSN: NewCSN(20)}} + got := trimToSafeCSN(events, NewCSN(5)) + assert.Empty(t, got, "all events above safe ceiling must be trimmed") + }) + + t.Run("empty slice — no-op", func(t *testing.T) { + t.Parallel() + assert.Nil(t, trimToSafeCSN(nil, NewCSN(100))) + }) + + t.Run("single event at safe — kept", func(t *testing.T) { + t.Parallel() + events := []ChangeEvent{{CSN: NewCSN(50)}} + got := trimToSafeCSN(events, NewCSN(50)) + require.Len(t, got, 1) + assert.Equal(t, uint64(50), got[0].CSN.Uint64()) + }) +} + +// TestBuildPollQueryBinaryColumnComparison verifies the X'...' hex literal +// format embedded in the poll query. The literal must be exactly 2*byteLen +// characters to avoid SQLSTATE 22001. +// +// When CommitSeqByteLen is 0 (undetected), buildPollQuery defaults to 10 bytes. +// CSNs created via NewCSN (no rawBytes) write the uint64 big-endian into the +// first 8 bytes of the N-byte buffer; bytes 8..N-1 are zero. +func TestBuildPollQueryBinaryColumnComparison(t *testing.T) { + t.Parallel() + + // CommitSeqByteLen=0 → buildPollQuery falls back to 10-byte default. + s := &Streamer{config: StreamConfig{PollBatchSize: 50}} + + tests := []struct { + name string + afterCSN CSN + upperCSN CSN + wantAfter string + wantUpper string + }{ + { + name: "zero bounds", + afterCSN: NewCSN(0), + upperCSN: NewCSN(0), + wantAfter: "X'00000000000000000000'", + wantUpper: "X'00000000000000000000'", + }, + { + // uint64 big-endian in bytes 0-7, zero-pad bytes 8-9. + // 0x3E8 → bytes 0-7: [0,0,0,0,0,0,0x03,0xE8]; bytes 8-9: [0,0] + // hex: "00000000000003E8" + "0000" = "00000000000003E80000" + name: "typical CDC range", + afterCSN: NewCSN(1000), // 0x3E8 + upperCSN: NewCSN(2000), // 0x7D0 + wantAfter: "X'00000000000003E80000'", + wantUpper: "X'00000000000007D00000'", + }, + { + // 0xFFFFFFFF → bytes 0-7: [0,0,0,0,0xFF,0xFF,0xFF,0xFF]; bytes 8-9: [0,0] + // 0x100000000 → bytes 0-7: [0,0,0,0x01,0,0,0,0]; bytes 8-9: [0,0] + name: "large CSN spanning 32-bit boundary", + afterCSN: NewCSN(0xFFFFFFFF), + upperCSN: NewCSN(0x100000000), + wantAfter: "X'00000000FFFFFFFF0000'", + wantUpper: "X'00000001000000000000'", + }, + { + // rawBytes path: CSN from DB read uses rawBytes verbatim. + // [0,0,0,0,0,0,0x17,0x14,0xAB,0x00] → "0000000000001714AB00" + name: "rawBytes path (10-byte DB2 CSN)", + afterCSN: NewCSNFromBytes([]byte{0, 0, 0, 0, 0, 0, 0x17, 0x14, 0xAB, 0x00}), + upperCSN: NewCSNFromBytes([]byte{0, 0, 0, 0, 0, 0, 0x17, 0x15, 0x00, 0x00}), + // rawBytes has 10 bytes → SQLHex(10) = all 10 bytes verbatim + wantAfter: "X'0000000000001714AB00'", + wantUpper: "X'00000000000017150000'", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + q := s.buildPollQuery("ASNCDC.T_CT", tc.afterCSN, 0, tc.upperCSN) + assert.Contains(t, q, tc.wantAfter, + "afterCSN hex literal must be 20-char for CHAR(10) FOR BIT DATA") + assert.Contains(t, q, tc.wantUpper, + "upperCSN hex literal must be 20-char for CHAR(10) FOR BIT DATA") + }) + } +} + +// TestBuildPollQueryIntentSeqHexLiteral verifies that composite (CSN, IntentSeq) pagination +// encodes afterIntentSeq as a CHAR(10) FOR BIT DATA hex literal — not a decimal integer. +// IBMSNAP_INTENTSEQ is typed CHAR(10) FOR BIT DATA on DB2 LUW; comparing it to a decimal +// integer literal is a type mismatch that DB2 would reject with SQLSTATE 42818. +func TestBuildPollQueryIntentSeqHexLiteral(t *testing.T) { + t.Parallel() + + s := &Streamer{config: StreamConfig{PollBatchSize: 50}} + afterCSN := NewCSN(5) + upperCSN := NewCSN(10) + + tests := []struct { + name string + afterIntentSeq int64 + wantHex string // expected X'...' fragment in the query + wantNotDecimal string // decimal form that must NOT appear verbatim + }{ + { + name: "intentSeq=0 — simple pagination, no INTENTSEQ predicate", + afterIntentSeq: 0, + wantHex: "", // no composite predicate at all + }, + { + // int64=1 into bytes[0:8] = 00 00 00 00 00 00 00 01, + // then 0xFF 0xFF padding → 00 00 00 00 00 00 00 01 FF FF (10 bytes, 20 hex chars) + name: "intentSeq=1", + afterIntentSeq: 1, + wantHex: "X'0000000000000001FFFF'", + wantNotDecimal: "> 1)", + }, + { + // 999 = 0x3E7; bytes[0:8] = 00 00 00 00 00 00 03 E7, + // then 0xFF 0xFF → 00 00 00 00 00 00 03 E7 FF FF (20 hex chars) + name: "intentSeq=999", + afterIntentSeq: 999, + wantHex: "X'00000000000003E7FFFF'", + wantNotDecimal: "> 999)", + }, + { + // 65536 = 0x10000; bytes[0:8] = 00 00 00 00 00 01 00 00, + // then 0xFF 0xFF → 00 00 00 00 00 01 00 00 FF FF (20 hex chars) + name: "intentSeq=65536", + afterIntentSeq: 65536, + wantHex: "X'0000000000010000FFFF'", + wantNotDecimal: "> 65536)", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + q := s.buildPollQuery("ASNCDC.T_CT", afterCSN, tc.afterIntentSeq, upperCSN) + if tc.wantHex != "" { + assert.Contains(t, q, tc.wantHex, + "IBMSNAP_INTENTSEQ must be compared via X'...' hex literal (CHAR(10) FOR BIT DATA)") + } + if tc.wantNotDecimal != "" { + assert.NotContains(t, q, tc.wantNotDecimal, + "decimal comparison to CHAR(10) FOR BIT DATA is a type mismatch (SQLSTATE 42818)") + } + }) + } +} diff --git a/internal/impl/db2/replication/types.go b/internal/impl/db2/replication/types.go new file mode 100644 index 0000000000..4d840127f6 --- /dev/null +++ b/internal/impl/db2/replication/types.go @@ -0,0 +1,674 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// CSN represents a DB2 Commit Sequence Number — the binary log position stored +// in the IBMSNAP_COMMITSEQ column (type CHAR(10) FOR BIT DATA) of every change +// table. The 10-byte big-endian value is held internally as a uint64 for +// efficient comparison and arithmetic; the upper 2 bytes are unused in practice. +// +// CSN values are monotonically increasing: a higher value means a later +// committed transaction. Events with the same CSN belong to the same DB2 +// transaction and are ordered by IBMSNAP_INTENTSEQ. +// +// String representation: "CSN:<16-digit-hex>", e.g. "CSN:000000000000C350". +// When embedding in a SQL WHERE clause use the 20-hex-char literal form +// X'%020X' to match the CHAR(10) column without a type cast. +// +// The null CSN (returned by NullCSN) marks snapshot events that have no +// associated log position. +type CSN struct { + value uint64 + isNull bool + rawBytes []byte // Original bytes from DB2 +} + +// NewCSN creates a CSN from a numeric uint64 value. Use this when constructing +// synthetic CSNs (e.g. checkpoints loaded from integer storage, unit tests). +// For CSNs read from DB2 change tables use NewCSNFromDBValue instead, which +// correctly handles the ASCII-hex encoding returned by the DB2 CLI driver. +func NewCSN(value uint64) CSN { + return CSN{ + value: value, + isNull: false, + } +} + +// NullCSN returns an unset CSN used to mark snapshot events that were read +// outside the CDC log stream and therefore have no commit sequence position. +func NullCSN() CSN { + return CSN{isNull: true} +} + +// NewCSNFromBytes creates a CSN from raw binary bytes (big-endian). +// For values received from the DB2 CLI driver, use NewCSNFromDBValue instead, +// since the driver returns CHAR(n) FOR BIT DATA as ASCII hex strings. +func NewCSNFromBytes(data []byte) CSN { + if len(data) == 0 { + return CSN{isNull: true} + } + + // Convert bytes to uint64 (big-endian) + var value uint64 + for i := 0; i < len(data) && i < 8; i++ { + value = (value << 8) | uint64(data[i]) + } + + return CSN{ + value: value, + isNull: false, + rawBytes: data, + } +} + +// NewCSNFromDBValue creates a CSN from a value as returned by the DB2 CLI driver. +// +// The DB2 CLI library (SQL_C_CHAR) returns CHAR(n) FOR BIT DATA columns as +// uppercase ASCII hex strings — e.g. a 10-byte LSN yields a 20-character +// string like "0000000000001714AB00". Scanning that into []byte gives the +// ASCII bytes of the hex string, not raw binary. This function hex-decodes the +// value before constructing the CSN. If the bytes are not valid hex (e.g. raw +// bytes from internal construction), they are interpreted as raw big-endian. +func NewCSNFromDBValue(data []byte) CSN { + if len(data) == 0 { + return CSN{isNull: true} + } + // Try hex decoding (DB2 CLI SQL_C_CHAR path). + if len(data)%2 == 0 { + isHex := true + for _, b := range data { + if (b < '0' || b > '9') && (b < 'A' || b > 'F') && (b < 'a' || b > 'f') { + isHex = false + break + } + } + if isHex { + if decoded, err := hex.DecodeString(string(data)); err == nil { + return NewCSNFromBytes(decoded) + } + } + } + return NewCSNFromBytes(data) +} + +// NewCSNFromHex creates a CSN by hex-decoding hexStr (e.g. "0000000000001A2B"). +// An optional "0x" or "0X" prefix is stripped. Returns an error if hexStr +// contains non-hex characters or has an odd number of characters. +func NewCSNFromHex(hexStr string) (CSN, error) { + // Remove any "0x" prefix + hexStr = strings.TrimPrefix(hexStr, "0x") + hexStr = strings.TrimPrefix(hexStr, "0X") + + // Decode hex to bytes + data, err := hex.DecodeString(hexStr) + if err != nil { + return CSN{}, fmt.Errorf("invalid hex CSN: %w", err) + } + + return NewCSNFromBytes(data), nil +} + +// ParseCSN parses a CSN from a string (handles multiple formats) +// Formats: "CSN:", "", "0x" +func ParseCSN(s string) (CSN, error) { + if s == "" { + return CSN{isNull: true}, nil + } + + // Handle "CSN:" prefix + if hexStr, ok := strings.CutPrefix(s, "CSN:"); ok { + return NewCSNFromHex(hexStr) + } + + // Handle hex with "0x" prefix + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + return NewCSNFromHex(s) + } + + // Try parsing as decimal + value, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return CSN{}, fmt.Errorf("invalid CSN format: %s", s) + } + + return NewCSN(value), nil +} + +// String returns the string representation of the CSN. +// +// When rawBytes is present (CSN was read directly from DB2), all bytes are +// encoded so that ParseCSN(c.String()).Equal(c) holds for 16-byte DB2 12.1 CSNs. +// The %016X form truncates to 8 bytes and would lose bytes 8-15 of a 16-byte LSN. +func (c CSN) String() string { + if c.isNull { + return "" + } + if len(c.rawBytes) > 0 { + // Encode directly to uppercase to avoid hex.EncodeToString (lowercase) + + // strings.ToUpper (second copy) + "CSN:" concat (third copy) = 3 allocs. + // One allocation for the result string. + const upperHex = "0123456789ABCDEF" + enc := make([]byte, 4+2*len(c.rawBytes)) // "CSN:" + hex digits + copy(enc, "CSN:") + for i, b := range c.rawBytes { + enc[4+2*i] = upperHex[b>>4] + enc[4+2*i+1] = upperHex[b&0x0F] + } + return string(enc) + } + return fmt.Sprintf("CSN:%016X", c.value) +} + +// Uint64 returns the internal uint64 representation of the CSN. For CSNs read +// from DB2 change tables, only the first 8 bytes of the raw value are stored in +// the uint64; use rawBytes (via String) for the full 16-byte representation on +// DB2 12.1+. +func (c CSN) Uint64() uint64 { + return c.value +} + +// HexLiteral returns the uppercase hex encoding of this CSN for use in an +// SQL X'...' literal. size specifies the desired byte count (not hex chars). +// +// DB2 12 stores IBMSNAP_COMMITSEQ as VARCHAR(16) FOR BIT DATA — a full 16-byte +// log LSN. Passing size=16 produces the correct 32-char literal for these +// columns so that COMMITSEQ > X'...' comparisons work correctly. +// +// When rawBytes are available (CSN came from a DB2 query) they are used +// directly, which preserves the full binary precision of the log position. +// When rawBytes are absent (e.g. NewCSN(0) for a fresh start), the uint64 +// value is encoded big-endian in the first 8 bytes and the remaining bytes +// are zero-padded on the right. +func (c CSN) HexLiteral(size int) string { + if len(c.rawBytes) >= size { + return strings.ToUpper(hex.EncodeToString(c.rawBytes[:size])) + } + b := make([]byte, size) + if len(c.rawBytes) > 0 { + copy(b, c.rawBytes) + } else { + binary.BigEndian.PutUint64(b[:8], c.value) + } + return strings.ToUpper(hex.EncodeToString(b)) +} + +// IsNull returns true if the CSN is null/unset +func (c CSN) IsNull() bool { + return c.isNull +} + +// Compare compares two CSNs. +// Returns: -1 if c < other, 0 if c == other, 1 if c > other +// +// When both CSNs have rawBytes (i.e. were read directly from DB2), a full +// byte-by-byte comparison is used. This is required for DB2 12.1 which uses +// 16-byte LSNs: nearby transactions on the same log page share the same +// first-8-byte uint64 value but differ only in the trailing bytes. Comparing only +// the uint64 (bytes 0–7) would treat such CSNs as equal, preventing the streamer +// from advancing past the initial SYNCHPOINT. +// +// Shorter rawBytes are left-aligned (trailing positions treated as zero), matching +// DB2 ≤11.x's CHAR(10) layout where bytes 8–9 are always zero. +func (c CSN) Compare(other CSN) int { + if c.isNull && other.isNull { + return 0 + } + if c.isNull { + return -1 + } + if other.isNull { + return 1 + } + + // Full raw-bytes comparison when both CSNs were read from DB2. + // DB2 ≤11.x stores the uint64 in bytes 0-7 with trailing zero bytes (8-9). + // Left-align shorter rawBytes: the missing trailing bytes are treated as zero, + // which is correct because DB2 trailing bytes are always zero for 10-byte CSNs. + if len(c.rawBytes) > 0 && len(other.rawBytes) > 0 { + // Fast path: equal length (the common case after auto-detect stabilises). + if len(c.rawBytes) == len(other.rawBytes) { + return bytes.Compare(c.rawBytes, other.rawBytes) + } + // Unequal lengths: compare common prefix, then treat the trailing bytes of + // the longer side against implicit zeros. + n := min(len(c.rawBytes), len(other.rawBytes)) + if cmp := bytes.Compare(c.rawBytes[:n], other.rawBytes[:n]); cmp != 0 { + return cmp + } + // Common prefix is equal. Check whether the longer side's tail is all zeros. + if len(c.rawBytes) > n { + for _, b := range c.rawBytes[n:] { + if b != 0 { + return 1 + } + } + return 0 + } + for _, b := range other.rawBytes[n:] { + if b != 0 { + return -1 + } + } + return 0 + } + + // Fall back to uint64 for synthetic CSNs (checkpoints, unit tests). + if c.value < other.value { + return -1 + } + if c.value > other.value { + return 1 + } + return 0 +} + +// Less returns true if c < other +func (c CSN) Less(other CSN) bool { + return c.Compare(other) < 0 +} + +// Equal returns true if c == other +func (c CSN) Equal(other CSN) bool { + return c.Compare(other) == 0 +} + +// Greater returns true if c > other +func (c CSN) Greater(other CSN) bool { + return c.Compare(other) > 0 +} + +// Next returns the CSN incremented by one. Useful when computing an exclusive +// lower bound for poll queries that use IBMSNAP_COMMITSEQ > X'...' predicates. +// On a null CSN, Next() returns NewCSN(1). +func (c CSN) Next() CSN { + if c.isNull { + return NewCSN(1) + } + return NewCSN(c.value + 1) +} + +// SQLHex returns an uppercase hex string for embedding in X'...' SQL literals. +// +// lsnByteLen is the byte length of the target CHAR(n) FOR BIT DATA column +// (10 for DB2 ≤ 11.x, 16 for DB2 12.1+). +// +// DB2 ≤11.x stores the CSN as a uint64 in bytes 0–7 (big-endian) followed by +// 2 trailing zero bytes (bytes 8–9) in the CHAR(10) column. rawBytes is +// left-aligned in buf: copy to buf[0:], trailing bytes remain zero. +// +// When only the uint64 value is available (e.g. restored from checkpoint), +// the value is written big-endian into bytes 0–7 of buf; trailing bytes are zero. +func (c CSN) SQLHex(lsnByteLen int) string { + if lsnByteLen <= 0 { + lsnByteLen = 10 + } + // Use a stack-allocated backing array for the common ≤16-byte case to avoid + // a heap allocation per call (DB2 IBMSNAP_COMMITSEQ is 10 bytes in practice). + var backing [16]byte + var buf []byte + if lsnByteLen <= len(backing) { + buf = backing[:lsnByteLen] + } else { + buf = make([]byte, lsnByteLen) + } + if len(c.rawBytes) > 0 { + // Left-align: rawBytes at buf[0:], trailing positions stay zero. + n := min(len(c.rawBytes), lsnByteLen) + copy(buf, c.rawBytes[:n]) + } else { + // uint64 big-endian in bytes 0-7; bytes 8+ are zero padding. + limit := min(lsnByteLen, 8) + v := c.value + for i := limit - 1; i >= 0; i-- { + buf[i] = byte(v) + v >>= 8 + } + } + // Encode directly to uppercase hex to avoid the two-allocation path of + // hex.EncodeToString (produces lowercase) + strings.ToUpper (produces uppercase). + const upperHex = "0123456789ABCDEF" + var encBacking [32]byte + var enc []byte + if 2*lsnByteLen <= len(encBacking) { + enc = encBacking[:2*lsnByteLen] + } else { + enc = make([]byte, 2*lsnByteLen) + } + for i, b := range buf { + enc[2*i] = upperHex[b>>4] + enc[2*i+1] = upperHex[b&0x0F] + } + return string(enc) +} + +// OpType represents a CDC operation type. +type OpType string + +// OpType constants for CDC operations visible to callers of the replication package. +const ( + // OpTypeRead marks a snapshot row read during the initial (or incremental) + // snapshot phase. These events have a null CSN because they are not sourced + // from the DB2 transaction log. Debezium op code: "r". + OpTypeRead OpType = "read" + + // OpTypeInsert marks a new row inserted into the source table. Corresponds + // to IBMSNAP_OPERATION='I' (standalone, not part of a D+I update pair). + // Debezium op code: "c" (create). + OpTypeInsert OpType = "insert" + + // OpTypeDelete marks a row deleted from the source table. Corresponds to + // IBMSNAP_OPERATION='D' that is NOT followed by a matching 'I' in the same + // IBMSNAP_COMMITSEQ. Debezium op code: "d". + OpTypeDelete OpType = "delete" + + // OpTypeUpdate is emitted by pairOpcodeEvents when the LEAD/LAG window query + // (buildPollQuery) detects a D+I pair sharing the same IBMSNAP_COMMITSEQ. + // DB2 LUW SQL Replication encodes UPDATE as a DELETE record (before-image) + // immediately followed by an INSERT record (after-image). BeforeData holds + // the pre-update row; Data holds the post-update row. Debezium op code: "u". + // + // IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=tables-ibmsnap-uow + OpTypeUpdate OpType = "update" +) + +// opTypeUpdateBefore and opTypeUpdateAfter are internal-only intermediate +// values emitted by the LEAD/LAG SQL query. They are never exported as part +// of a ChangeEvent; pairOpcodeEvents merges them into OpTypeUpdate. +const ( + opTypeUpdateBefore OpType = "_update_before" // IBMSNAP_OPCODE=3 + opTypeUpdateAfter OpType = "_update_after" // IBMSNAP_OPCODE=4 +) + +// OpTypeHeartbeat is emitted periodically when no CDC changes are available. +// Used to keep downstream consumers alive on low-traffic tables. +const OpTypeHeartbeat OpType = "heartbeat" + +// OpTypeSchemaChange is emitted when a new table is added to SQL Replication +// (i.e., when ASNCDC.ADDTABLE is called while the connector is running). +const OpTypeSchemaChange OpType = "schema_change" + +// FromDB2Op converts a DB2 IBMSNAP_OPERATION code to an OpType. +// +// DB2 LUW SQL Replication uses three operation codes: +// - 'I' (INSERT): a new row was inserted. +// - 'D' (DELETE): an existing row was deleted. +// - 'U' (UPDATE): an existing row was modified. Note that UPDATE events are +// NOT encoded as 'B' (before-image) + 'U' pairs — that encoding is used by +// IBM Q-Replication only. In LUW SQL Replication each UPDATE produces a 'D' +// record (the before-image) followed immediately by an 'I' record (the +// after-image) with the same IBMSNAP_COMMITSEQ value. +// +// Returns an error for empty strings or unrecognised codes. +func FromDB2Op(dbOp string) (OpType, error) { + if len(dbOp) == 0 { + return "", errors.New("empty operation type") + } + + switch dbOp[0] { + case 'I', 'i': + return OpTypeInsert, nil + case 'U', 'u': + return OpTypeUpdate, nil + case 'D', 'd': + return OpTypeDelete, nil + default: + return "", fmt.Errorf("unknown DB2 operation: %s", dbOp) + } +} + +// fromOpcodeInt maps an IBMSNAP_OPCODE integer (from the LEAD/LAG subquery) +// to an OpType. Returns an error for unknown values. +func fromOpcodeInt(code int64) (OpType, error) { + switch code { + case 1: + return OpTypeDelete, nil + case 2: + return OpTypeInsert, nil + case 3: + return opTypeUpdateBefore, nil + case 4: + return opTypeUpdateAfter, nil + default: + return "", fmt.Errorf("unknown IBMSNAP_OPCODE %d", code) + } +} + +// Version represents a DB2 version +type Version struct { + Major int + Minor int + Mod int + Fix int + Raw string +} + +// ParseVersion parses a DB2 version string. +// Examples: "SQL11050", "DB2 v11.5.0.0", "11.5" +// All known DB2 versions have Major >= 9; a zero Major indicates a parse anomaly +// and produces an "unknown version" error from SupportsCDC callers. +func ParseVersion(versionStr string) (Version, error) { + v := Version{Raw: versionStr} + + // Handle "SQL11050" format + if rest, ok := strings.CutPrefix(versionStr, "SQL"); ok { + versionStr = rest + if len(versionStr) >= 5 { + major, _ := strconv.Atoi(versionStr[0:2]) + minor, _ := strconv.Atoi(versionStr[2:4]) + mod, _ := strconv.Atoi(versionStr[4:5]) + v.Major = major + v.Minor = minor + v.Mod = mod + return v, nil + } + } + + // Handle "DB2 v11.5.0.0" format + versionStr = strings.TrimPrefix(versionStr, "DB2 ") + versionStr = strings.TrimPrefix(versionStr, "v") + + // Parse dotted version "11.5.0.0" + parts := strings.Split(versionStr, ".") + if len(parts) >= 1 { + v.Major, _ = strconv.Atoi(strings.TrimSpace(parts[0])) + } + if len(parts) >= 2 { + v.Minor, _ = strconv.Atoi(strings.TrimSpace(parts[1])) + } + if len(parts) >= 3 { + v.Mod, _ = strconv.Atoi(strings.TrimSpace(parts[2])) + } + if len(parts) >= 4 { + v.Fix, _ = strconv.Atoi(strings.TrimSpace(parts[3])) + } + + if v.Major == 0 { + return v, fmt.Errorf("parsing DB2 version: %s", versionStr) + } + + return v, nil +} + +// String returns a string representation of the version +func (v Version) String() string { + return fmt.Sprintf("%d.%d.%d.%d", v.Major, v.Minor, v.Mod, v.Fix) +} + +// Compare compares two versions +// Returns: -1 if v < other, 0 if v == other, 1 if v > other +func (v Version) Compare(other Version) int { + if v.Major != other.Major { + if v.Major < other.Major { + return -1 + } + return 1 + } + if v.Minor != other.Minor { + if v.Minor < other.Minor { + return -1 + } + return 1 + } + if v.Mod != other.Mod { + if v.Mod < other.Mod { + return -1 + } + return 1 + } + if v.Fix != other.Fix { + if v.Fix < other.Fix { + return -1 + } + return 1 + } + return 0 +} + +// AtLeast returns true if v >= other +func (v Version) AtLeast(major, minor int) bool { + if v.Major > major { + return true + } + if v.Major == major && v.Minor >= minor { + return true + } + return false +} + +// SupportsCDC reports whether this DB2 version supports SQL Replication CDC. +// DB2 10.1 introduced the CSN-based IBMSNAP_COMMITSEQ column that this connector +// depends on. Versions before 10.1 used a different replication subsystem and +// are not supported. +// +// IBM reference: https://www.ibm.com/docs/en/db2/11.5?topic=replication-sql-capture-program +func (v Version) SupportsCDC() bool { + return v.AtLeast(10, 1) +} + +// ChangeEvent is a single CDC event produced by the DB2 SQL Replication capture +// daemon and emitted as a Redpanda Connect message. +// +// For snapshot events (Operation == OpTypeRead) CSN is the null CSN and +// BeforeData is nil. For streaming events CSN carries the +// IBMSNAP_COMMITSEQ value; IntentSeq (IBMSNAP_INTENTSEQ) orders rows within +// the same transaction. +// +// For update events (Operation == OpTypeUpdate), pairOpcodeEvents merges the +// D+I pair detected by the LEAD/LAG window query: BeforeData holds the +// pre-update row and Data holds the post-update row. +type ChangeEvent struct { + Schema string `json:"schema"` // DB2 source schema (TABSCHEMA) + Table string `json:"table"` // DB2 source table name + Operation OpType `json:"operation"` // insert / delete / read / update + CSN CSN `json:"csn"` // log position; NullCSN for snapshot rows + IntentSeq int64 `json:"intent_seq"` // IBMSNAP_INTENTSEQ, tie-breaks within a CSN + Timestamp time.Time `json:"timestamp"` // IBMSNAP_LOGMARKER from the change table + Data map[string]any `json:"data"` // row column values; for INSERT/UPDATE: new row; for DELETE: deleted row + BeforeData map[string]any `json:"before_data,omitempty"` // pre-update row for OpTypeUpdate; nil for all other operations + // PKColumns is set for OpTypeRead (snapshot) events and lists the primary-key + // column names. The idempotency key uses only PK columns so that non-PK columns + // (which may contain sensitive values) do not appear in message metadata. + PKColumns []string `json:"pk_columns,omitempty"` + // QualKey is "SCHEMA.TABLE" set once at construction time in pollChangeTable. + // Stored to avoid repeated string concatenation in hot inner loops (e.g. the + // newIntentSeqs scan). Excluded from JSON output — use Schema/Table instead. + QualKey string `json:"-"` +} + +// String returns a JSON representation of the event +func (e ChangeEvent) String() string { + data, err := json.Marshal(e) + if err != nil { + // Fallback: JSON-encode each string field individually to avoid injection. + schema, _ := json.Marshal(e.Schema) + table, _ := json.Marshal(e.Table) + op, _ := json.Marshal(string(e.Operation)) + csn, _ := json.Marshal(e.CSN.String()) + return fmt.Sprintf(`{"schema":%s,"table":%s,"operation":%s,"csn":%s,"intent_seq":%d}`, + schema, table, op, csn, e.IntentSeq) + } + return string(data) +} + +// MarshalJSON implements json.Marshaler for proper JSON serialization +func (e ChangeEvent) MarshalJSON() ([]byte, error) { + type Alias ChangeEvent + return json.Marshal(&struct { + CSN string `json:"csn"` + Timestamp string `json:"timestamp,omitempty"` + *Alias + }{ + CSN: e.CSN.String(), + Timestamp: formatTimestamp(e.Timestamp), + Alias: (*Alias)(&e), + }) +} + +// formatTimestamp formats a timestamp for JSON output +func formatTimestamp(t time.Time) string { + if t.IsZero() { + return "" + } + return t.Format(time.RFC3339Nano) +} + +// QuoteDB2Identifier wraps a DB2 identifier in double-quote delimiters, +// escaping any embedded double-quotes by doubling them (SQL standard §5.3). +// Use this for every schema, table, or column name embedded in a SQL string +// via fmt.Sprintf — never inline the pattern manually. +func QuoteDB2Identifier(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` +} + +// db2ColumnKind classifies a DB2 column's data family for value conversion. +// Using a typed constant rather than a plain bool lets callers be explicit +// about which family they are handling and makes the match exhaustive if a +// new kind is ever added. +type db2ColumnKind uint8 + +const ( + db2ColumnKindOther db2ColumnKind = iota // numerics, date/time, boolean, … + db2ColumnKindText // CHAR, VARCHAR, GRAPHIC, CLOB, TEXT, … + db2ColumnKindBinary // BINARY, VARBINARY, BLOB +) + +// columnKind maps a DB2 column type name (sql.ColumnType.DatabaseTypeName) to +// its conversion family. +// +// Covered families: +// - db2ColumnKindText: CHAR, VARCHAR, LONG VARCHAR, NCHAR, NVARCHAR, +// CLOB, NCLOB, DBCLOB, GRAPHIC, VARGRAPHIC, LONG VARGRAPHIC, TEXT +// - db2ColumnKindBinary: BINARY, VARBINARY, BLOB +// - db2ColumnKindOther: INTEGER, BIGINT, DECIMAL, DATE, TIMESTAMP, … +func columnKind(typeName string) db2ColumnKind { + t := strings.ToUpper(typeName) + switch { + case strings.Contains(t, "CHAR") || + strings.Contains(t, "CLOB") || + strings.Contains(t, "TEXT") || + strings.Contains(t, "GRAPHIC"): + return db2ColumnKindText + case strings.Contains(t, "BINARY") || strings.Contains(t, "BLOB"): + return db2ColumnKindBinary + default: + return db2ColumnKindOther + } +} diff --git a/internal/impl/db2/replication/types_test.go b/internal/impl/db2/replication/types_test.go new file mode 100644 index 0000000000..8d12711744 --- /dev/null +++ b/internal/impl/db2/replication/types_test.go @@ -0,0 +1,671 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package replication + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCSN(t *testing.T) { + t.Run("create from uint64", func(t *testing.T) { + csn := NewCSN(12345) + assert.False(t, csn.IsNull()) + assert.Equal(t, uint64(12345), csn.Uint64()) + assert.Equal(t, "CSN:0000000000003039", csn.String()) + }) + + t.Run("create from bytes", func(t *testing.T) { + data := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x39} + csn := NewCSNFromBytes(data) + assert.False(t, csn.IsNull()) + assert.Equal(t, uint64(12345), csn.Uint64()) + }) + + t.Run("create from hex string", func(t *testing.T) { + csn, err := NewCSNFromHex("0000000000003039") + require.NoError(t, err) + assert.Equal(t, uint64(12345), csn.Uint64()) + + csn2, err := NewCSNFromHex("0x3039") + require.NoError(t, err) + assert.Equal(t, uint64(12345), csn2.Uint64()) + }) + + t.Run("parse CSN", func(t *testing.T) { + tests := []struct { + input string + expected uint64 + wantErr bool + }{ + {"CSN:0000000000003039", 12345, false}, + {"0x3039", 12345, false}, + {"12345", 12345, false}, + {"", 0, false}, // Null CSN + {"invalid", 0, true}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + csn, err := ParseCSN(tt.input) + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + if tt.input == "" { + assert.True(t, csn.IsNull()) + } else { + assert.Equal(t, tt.expected, csn.Uint64()) + } + } + }) + } + }) + + t.Run("comparison", func(t *testing.T) { + csn1 := NewCSN(100) + csn2 := NewCSN(200) + csn3 := NewCSN(100) + + assert.True(t, csn1.Less(csn2)) + assert.False(t, csn2.Less(csn1)) + assert.False(t, csn1.Less(csn3)) + + assert.True(t, csn1.Equal(csn3)) + assert.False(t, csn1.Equal(csn2)) + + assert.True(t, csn2.Greater(csn1)) + assert.False(t, csn1.Greater(csn2)) + + assert.Equal(t, -1, csn1.Compare(csn2)) + assert.Equal(t, 1, csn2.Compare(csn1)) + assert.Equal(t, 0, csn1.Compare(csn3)) + }) + + t.Run("null CSN", func(t *testing.T) { + nullCSN := CSN{isNull: true} + csn := NewCSN(100) + + assert.True(t, nullCSN.IsNull()) + assert.True(t, nullCSN.Less(csn)) + assert.False(t, csn.Less(nullCSN)) + }) + + t.Run("next", func(t *testing.T) { + csn := NewCSN(100) + next := csn.Next() + assert.Equal(t, uint64(101), next.Uint64()) + + nullCSN := CSN{isNull: true} + nextNull := nullCSN.Next() + assert.Equal(t, uint64(1), nextNull.Uint64()) + }) +} + +func TestOpType(t *testing.T) { + t.Run("from DB2 operation", func(t *testing.T) { + tests := []struct { + dbOp string + expected OpType + wantErr bool + }{ + {"I", OpTypeInsert, false}, + {"i", OpTypeInsert, false}, + {"U", OpTypeUpdate, false}, + {"u", OpTypeUpdate, false}, + {"D", OpTypeDelete, false}, + {"d", OpTypeDelete, false}, + {"B", "", true}, // 'B' is Q-Replication only; LUW SQL Replication uses D+I pairs + {"b", "", true}, + {"", "", true}, + {"X", "", true}, + } + + for _, tt := range tests { + t.Run(tt.dbOp, func(t *testing.T) { + op, err := FromDB2Op(tt.dbOp) + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expected, op) + } + }) + } + }) +} + +func TestVersion(t *testing.T) { + t.Run("parse version", func(t *testing.T) { + tests := []struct { + input string + major int + minor int + mod int + fix int + }{ + {"SQL11050", 11, 5, 0, 0}, + {"DB2 v11.5.0.0", 11, 5, 0, 0}, + {"v10.1.0.5", 10, 1, 0, 5}, + {"11.5", 11, 5, 0, 0}, + {"10.5.0", 10, 5, 0, 0}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + v, err := ParseVersion(tt.input) + require.NoError(t, err) + assert.Equal(t, tt.major, v.Major) + assert.Equal(t, tt.minor, v.Minor) + assert.Equal(t, tt.mod, v.Mod) + assert.Equal(t, tt.fix, v.Fix) + }) + } + }) + + t.Run("version comparison", func(t *testing.T) { + v10_1 := Version{Major: 10, Minor: 1} + v10_5 := Version{Major: 10, Minor: 5} + v11_5 := Version{Major: 11, Minor: 5} + v11_5_dup := Version{Major: 11, Minor: 5} + + assert.Equal(t, -1, v10_1.Compare(v10_5)) + assert.Equal(t, -1, v10_5.Compare(v11_5)) + assert.Equal(t, 0, v11_5.Compare(v11_5_dup)) + assert.Equal(t, 1, v11_5.Compare(v10_1)) + }) + + t.Run("version feature support", func(t *testing.T) { + v9_7 := Version{Major: 9, Minor: 7} + v10_1 := Version{Major: 10, Minor: 1} + v11_5 := Version{Major: 11, Minor: 5} + + assert.False(t, v9_7.SupportsCDC()) + assert.True(t, v10_1.SupportsCDC()) + assert.True(t, v11_5.SupportsCDC()) + }) + + t.Run("AtLeast", func(t *testing.T) { + v11_5 := Version{Major: 11, Minor: 5} + + assert.True(t, v11_5.AtLeast(11, 5)) + assert.True(t, v11_5.AtLeast(11, 0)) + assert.True(t, v11_5.AtLeast(10, 5)) + assert.False(t, v11_5.AtLeast(11, 6)) + assert.False(t, v11_5.AtLeast(12, 0)) + }) + + t.Run("string representation", func(t *testing.T) { + v := Version{Major: 11, Minor: 5, Mod: 0, Fix: 1} + assert.Equal(t, "11.5.0.1", v.String()) + }) +} + +func TestChangeEvent(t *testing.T) { + t.Run("JSON serialization", func(t *testing.T) { + timestamp := time.Date(2024, 1, 15, 10, 30, 45, 123456789, time.UTC) + event := ChangeEvent{ + Schema: "DB2ADMIN", + Table: "EMPLOYEES", + Operation: OpTypeInsert, + CSN: NewCSN(12345), + IntentSeq: 1, + Timestamp: timestamp, + Data: map[string]any{ + "id": 123, + "name": "John Doe", + "salary": 75000.00, + }, + } + + // Test String() method returns valid JSON + str := event.String() + assert.NotEmpty(t, str) + + // Verify it's valid JSON + var decoded map[string]any + err := json.Unmarshal([]byte(str), &decoded) + require.NoError(t, err) + + // Check required fields + assert.Equal(t, "DB2ADMIN", decoded["schema"]) + assert.Equal(t, "EMPLOYEES", decoded["table"]) + assert.Equal(t, "insert", decoded["operation"]) + assert.Equal(t, "CSN:0000000000003039", decoded["csn"]) + assert.Equal(t, float64(1), decoded["intent_seq"]) + assert.NotEmpty(t, decoded["timestamp"]) + assert.NotNil(t, decoded["data"]) + }) + + t.Run("JSON serialization with before_data", func(t *testing.T) { + event := ChangeEvent{ + Schema: "DB2ADMIN", + Table: "EMPLOYEES", + Operation: OpTypeUpdate, + CSN: NewCSN(12345), + IntentSeq: 2, + Data: map[string]any{ + "id": 123, + "salary": 80000.00, + }, + BeforeData: map[string]any{ + "id": 123, + "salary": 75000.00, + }, + } + + data, err := json.Marshal(event) + require.NoError(t, err) + + var decoded map[string]any + err = json.Unmarshal(data, &decoded) + require.NoError(t, err) + + // Verify before_data is present + assert.NotNil(t, decoded["before_data"]) + beforeData := decoded["before_data"].(map[string]any) + assert.Equal(t, float64(75000.00), beforeData["salary"]) + }) + + t.Run("JSON serialization without timestamp", func(t *testing.T) { + event := ChangeEvent{ + Schema: "DB2ADMIN", + Table: "EMPLOYEES", + Operation: OpTypeRead, + CSN: NewCSN(0), + Data: map[string]any{"id": 1}, + } + + str := event.String() + var decoded map[string]any + err := json.Unmarshal([]byte(str), &decoded) + require.NoError(t, err) + + // timestamp should be omitted or empty when zero + timestamp, ok := decoded["timestamp"] + if ok { + assert.Empty(t, timestamp) + } + }) +} + +// TestGetTimeStringParsing verifies that getTime correctly parses DB2 native +// TIMESTAMP strings returned by the DB2 CLI driver (SQL_C_CHAR binding). The +// CLI returns TIMESTAMP columns as "YYYY-MM-DD HH:MM:SS.ffffff" strings rather +// than native time.Time values. Both the DB2 native format and RFC3339 must be +// accepted; other string formats must return zero time. +// +// This mirrors Debezium's timestamp column handling: DB2 change table rows carry +// IBMSNAP_LOGMARKER as a TIMESTAMP column which the CLI surfaces as a string. +func TestGetTimeStringParsing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input any + wantZero bool + wantTime time.Time + }{ + { + name: "DB2 native timestamp format", + input: "2024-05-15 10:30:45.123456", + wantZero: false, + wantTime: time.Date(2024, 5, 15, 10, 30, 45, 123456000, time.UTC), + }, + { + name: "DB2 native timestamp zero microseconds", + input: "2024-01-01 00:00:00.000000", + wantZero: false, + wantTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + }, + { + name: "RFC3339 format accepted as fallback", + input: "2024-05-15T10:30:45Z", + wantZero: false, + wantTime: time.Date(2024, 5, 15, 10, 30, 45, 0, time.UTC), + }, + { + name: "invalid string returns zero time", + input: "not-a-timestamp", + wantZero: true, + }, + { + name: "nil returns zero time", + input: nil, + wantZero: true, + }, + { + name: "time.Time pass-through", + input: time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC), + wantZero: false, + wantTime: time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + v := tc.input + result := getTime(&v) + if tc.wantZero { + assert.True(t, result.IsZero(), "expected zero time for input %v", tc.input) + } else { + assert.Equal(t, tc.wantTime, result) + } + }) + } +} + +// TestPairOpcodeEventsNullColumns verifies that pairOpcodeEvents correctly merges +// D+I pairs even when column values in Data are nil. The merged OpTypeUpdate event +// must preserve nil entries (not omit them) in both Data and BeforeData maps. +// +// This corresponds to Debezium's null column handling: when a column is nullable +// and NULL in the DB2 row, it appears as nil in the Data map. Merging before and +// after images must not silently drop nil-valued keys. +func TestPairOpcodeEventsNullColumns(t *testing.T) { + t.Parallel() + + csn := NewCSN(50) + + tests := []struct { + name string + input []ChangeEvent + wantData map[string]any + wantBefore map[string]any + wantOperation OpType + }{ + { + name: "update sets column to null — nil preserved in Data", + input: []ChangeEvent{ + { + CSN: csn, IntentSeq: 1, Operation: opTypeUpdateBefore, + Data: map[string]any{"ID": int64(5), "MANAGER_ID": int64(42), "DEPT": "ENG"}, + }, + { + CSN: csn, IntentSeq: 2, Operation: opTypeUpdateAfter, + Data: map[string]any{"ID": int64(5), "MANAGER_ID": nil, "DEPT": nil}, + }, + }, + wantOperation: OpTypeUpdate, + wantData: map[string]any{"ID": int64(5), "MANAGER_ID": nil, "DEPT": nil}, + wantBefore: map[string]any{"ID": int64(5), "MANAGER_ID": int64(42), "DEPT": "ENG"}, + }, + { + name: "update clears null — nil before, value after", + input: []ChangeEvent{ + { + CSN: csn, IntentSeq: 1, Operation: opTypeUpdateBefore, + Data: map[string]any{"ID": int64(7), "SCORE": nil}, + }, + { + CSN: csn, IntentSeq: 2, Operation: opTypeUpdateAfter, + Data: map[string]any{"ID": int64(7), "SCORE": int64(100)}, + }, + }, + wantOperation: OpTypeUpdate, + wantData: map[string]any{"ID": int64(7), "SCORE": int64(100)}, + wantBefore: map[string]any{"ID": int64(7), "SCORE": nil}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result := pairOpcodeEvents(tc.input) + require.Len(t, result, 1, "D+I pair must merge to a single event") + assert.Equal(t, tc.wantOperation, result[0].Operation) + assert.Equal(t, tc.wantData, result[0].Data) + assert.Equal(t, tc.wantBefore, result[0].BeforeData) + }) + } +} + +// TestCSNBinaryRoundTrip verifies that CSN values are formatted as 20-character +// zero-padded hex literals for use in SQL WHERE clauses against CHAR(10) FOR BIT +// DATA columns (IBMSNAP_COMMITSEQ). DB2 requires the X'...' literal to be +// exactly 20 hexadecimal characters (10 bytes) — a shorter literal causes +// SQLSTATE 22001 (string data, right truncation). +func TestCSNBinaryRoundTrip(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + value uint64 + wantStr string // canonical "CSN:<16hex>" representation + wantSQL string // 20-char SQL hex literal + }{ + { + name: "zero", + value: 0, + wantStr: "CSN:0000000000000000", + wantSQL: "00000000000000000000", + }, + { + name: "one", + value: 1, + wantStr: "CSN:0000000000000001", + wantSQL: "00000000000000000001", + }, + { + name: "0xC350 = 50000", + value: 50000, + wantStr: "CSN:000000000000C350", + wantSQL: "0000000000000000C350", + }, + { + name: "0xFFFF = 65535", + value: 65535, + wantStr: "CSN:000000000000FFFF", + wantSQL: "0000000000000000FFFF", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + csn := NewCSN(tc.value) + + // Canonical string format uses 16 hex chars. + assert.Equal(t, tc.wantStr, csn.String()) + + // SQL hex literal format must be exactly 20 chars for CHAR(10) FOR BIT DATA. + sqlLiteral := fmt.Sprintf("%020X", csn.Uint64()) + assert.Len(t, sqlLiteral, 20, + "SQL hex literal for CHAR(10) FOR BIT DATA must be exactly 20 chars") + assert.Equal(t, tc.wantSQL, sqlLiteral) + }) + } +} + +// TestCSNCompareRawBytes verifies that Compare uses full rawBytes comparison +// for DB2-read CSNs. DB2 12.1 uses 16-byte LSNs; nearby transactions on the +// same log page share the same first-8-byte value (uint64) but differ only in +// bytes 8–15. A uint64-only comparison would return 0 (equal) for such pairs, +// preventing the streamer from advancing SYNCHPOINT. +func TestCSNCompareRawBytes(t *testing.T) { + t.Parallel() + + // Same first 8 bytes (0x1754) but different trailing 8 bytes. + // This matches real DB2 12.1 output: 0000000000001754 | 000000000005A227 + base := NewCSNFromBytes([]byte{0, 0, 0, 0, 0, 0, 0x17, 0x54, 0, 0, 0, 0, 0, 0x05, 0xA2, 0x27}) + next := NewCSNFromBytes([]byte{0, 0, 0, 0, 0, 0, 0x17, 0x54, 0, 0, 0, 0, 0, 0x05, 0xB0, 0x00}) + + assert.Equal(t, uint64(0x1754), base.Uint64(), "both have same uint64") + assert.Equal(t, uint64(0x1754), next.Uint64(), "both have same uint64") + + assert.True(t, next.Greater(base), "next must be greater than base via rawBytes comparison") + assert.True(t, base.Less(next), "base must be less than next via rawBytes comparison") + assert.False(t, base.Equal(next), "base and next must not be equal") + + // Equal rawBytes → Equal CSNs. + same := NewCSNFromBytes([]byte{0, 0, 0, 0, 0, 0, 0x17, 0x54, 0, 0, 0, 0, 0, 0x05, 0xA2, 0x27}) + assert.True(t, base.Equal(same), "identical rawBytes must be equal") + + // uint64-only CSNs (no rawBytes) still compare by value. + u1 := NewCSN(0x1754) + u2 := NewCSN(0x1755) + assert.True(t, u2.Greater(u1), "uint64-only comparison still works") +} + +// TestCSNFromBytesPreservesAllBytes verifies that CHAR(10) FOR BIT DATA values +// from DB2 are decoded correctly to uint64. NewCSNFromBytes reads the first 8 +// bytes in big-endian order (bytes 8 and 9 of the 10-byte CHAR(10) column are +// ignored because uint64 holds only 8 bytes). Both 8-byte and 10-byte encodings +// of the same value must decode to identical CSN values. +func TestCSNFromBytesPreservesAllBytes(t *testing.T) { + t.Parallel() + + // DB2 CHAR(10) FOR BIT DATA: the first 8 bytes hold the uint64 value in + // big-endian order. Bytes 8 and 9 are extra padding. Value 50000 = 0xC350 + // encodes to bytes: {0,0,0,0,0,0,0xC3,0x50} in the first 8 positions. + tenByteCSN := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x50, 0x00, 0x00} + csn10 := NewCSNFromBytes(tenByteCSN) + assert.False(t, csn10.IsNull()) + assert.Equal(t, uint64(50000), csn10.Uint64()) + + // The canonical 8-byte representation must decode identically. + eightByteCSN := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x50} + csn8 := NewCSNFromBytes(eightByteCSN) + assert.Equal(t, csn10.Uint64(), csn8.Uint64(), + "8-byte and 10-byte encodings of the same CSN must produce equal values") + assert.True(t, csn10.Equal(csn8), + "CSN.Equal must hold between 8-byte and 10-byte encodings of the same value") +} + +// TestCSNStringRoundTripPreservesRawBytes verifies that String() serialises all +// rawBytes so that ParseCSN(c.String()).Equal(c) holds for 16-byte DB2 12.1 CSNs. +// The previous %016X format silently dropped bytes 8–15 of a 16-byte LSN. +func TestCSNStringRoundTripPreservesRawBytes(t *testing.T) { + t.Parallel() + + // 16-byte CSN: bytes 8-15 are significant (DB2 12.1 format). + raw16 := []byte{0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0xAB, 0xCD, 0xEF, 0x01, 0x02, 0x03, 0x04, 0x05} + csn := NewCSNFromBytes(raw16) + + s := csn.String() + assert.Equal(t, "CSN:0000000100000002ABCDEF0102030405", s, "String must encode all 16 bytes") + + restored, err := ParseCSN(s) + require.NoError(t, err) + assert.Equal(t, 0, csn.Compare(restored), "round-trip must produce Equal CSN") +} + +// TestCSNSQLHexLeftAligned verifies that SQLHex left-aligns rawBytes (DB2 ≤11.x +// CHAR(10) layout: uint64 in bytes 0–7, trailing zeros in bytes 8–9). +// An 8-byte rawBytes slice in a 10-byte column must pad with 2 trailing zeros, +// not 2 leading zeros. +func TestCSNSQLHexLeftAligned(t *testing.T) { + t.Parallel() + + // 8-byte rawBytes [00 00 00 00 00 00 C3 50] in a 10-byte column. + // Left-aligned: buf = [00,00,00,00,00,00,C3,50,00,00] → "000000000000C3500000" (20 hex chars) + raw8 := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x50} + csn := NewCSNFromBytes(raw8) + hex10 := csn.SQLHex(10) + assert.Equal(t, "000000000000C3500000", hex10) + assert.Len(t, hex10, 20, "10-byte column produces 20 hex characters") + + // uint64-only CSN must produce the same hex as 8-byte rawBytes with same value. + csnUint := NewCSN(50000) + assert.Equal(t, hex10, csnUint.SQLHex(10), "rawBytes and uint64 paths must produce identical SQL literals") +} + +// TestCSNCompareWithTrailingZeros verifies that Compare left-aligns shorter +// rawBytes (trailing zeros), so that 8-byte and 10-byte CSNs with the same +// uint64 value compare as equal. +func TestCSNCompareWithTrailingZeros(t *testing.T) { + t.Parallel() + + // DB2 ≤11.x: 10-byte = [uint64_bytes 0-7, 0x00, 0x00]; 8-byte = [uint64_bytes 0-7]. + raw8 := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x50} + raw10 := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x50, 0x00, 0x00} + csn8 := NewCSNFromBytes(raw8) + csn10 := NewCSNFromBytes(raw10) + assert.Equal(t, 0, csn8.Compare(csn10), "8-byte and 10-byte encodings of the same value must compare equal") + assert.True(t, csn8.Equal(csn10)) + + // Ordering: a smaller uint64 must compare less regardless of trailing zeros. + rawSmall := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x4F, 0x00, 0x00} + csnSmall := NewCSNFromBytes(rawSmall) + assert.Equal(t, -1, csnSmall.Compare(csn10), "smaller value must compare less") + assert.Equal(t, 1, csn10.Compare(csnSmall), "larger value must compare greater") +} + +// TestCSNSQLHexFullRawBytes verifies that a 10-byte rawBytes CSN (DB2 ≤11.x +// typical output) produces the correct 20-hex-char SQL literal without truncation. +func TestCSNSQLHexFullRawBytes(t *testing.T) { + t.Parallel() + + // Value 1000 (0x3E8) as 10 bytes: [00,00,00,00,00,00,03,E8,00,00] + raw10 := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00} + csn := NewCSNFromBytes(raw10) + assert.Equal(t, "00000000000003E80000", csn.SQLHex(10)) +} + +func TestColumnKind(t *testing.T) { + t.Parallel() + + tests := []struct { + typeName string + want db2ColumnKind + }{ + // Text — single-byte character types + {"CHAR", db2ColumnKindText}, + {"VARCHAR", db2ColumnKindText}, + {"LONG VARCHAR", db2ColumnKindText}, + {"NCHAR", db2ColumnKindText}, + {"NVARCHAR", db2ColumnKindText}, + // Text — character large objects + {"CLOB", db2ColumnKindText}, + {"NCLOB", db2ColumnKindText}, + {"DBCLOB", db2ColumnKindText}, + // Text — double-byte character strings (GRAPHIC family) + {"GRAPHIC", db2ColumnKindText}, + {"VARGRAPHIC", db2ColumnKindText}, + {"LONG VARGRAPHIC", db2ColumnKindText}, + // Text — compatibility type + {"TEXT", db2ColumnKindText}, + // Binary + {"BINARY", db2ColumnKindBinary}, + {"VARBINARY", db2ColumnKindBinary}, + {"BLOB", db2ColumnKindBinary}, + // Other — numeric and date/time types + {"INTEGER", db2ColumnKindOther}, + {"BIGINT", db2ColumnKindOther}, + {"SMALLINT", db2ColumnKindOther}, + {"DECIMAL", db2ColumnKindOther}, + {"FLOAT", db2ColumnKindOther}, + {"DOUBLE", db2ColumnKindOther}, + {"DATE", db2ColumnKindOther}, + {"TIME", db2ColumnKindOther}, + {"TIMESTAMP", db2ColumnKindOther}, + {"BOOLEAN", db2ColumnKindOther}, + // Case-insensitive + {"varchar", db2ColumnKindText}, + {"Vargraphic", db2ColumnKindText}, + {"clob", db2ColumnKindText}, + {"blob", db2ColumnKindBinary}, + // Empty string — unknown type + {"", db2ColumnKindOther}, + } + + for _, tc := range tests { + t.Run(tc.typeName, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, columnKind(tc.typeName)) + }) + } +} diff --git a/internal/impl/db2/testmain_test.go b/internal/impl/db2/testmain_test.go new file mode 100644 index 0000000000..9c1ff0aae1 --- /dev/null +++ b/internal/impl/db2/testmain_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2_test + +import ( + "context" + "flag" + "fmt" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/redpanda-data/connect/v4/internal/impl/db2/db2test" +) + +func TestMain(m *testing.M) { + // Only start the DB2 container when integration tests are explicitly + // requested via -run (e.g. go test -run '^TestIntegration'). Without + // this guard a plain `go test ./internal/impl/db2/...` would attempt to + // spin up a Docker container even for unit-only runs. + runStr := flag.Lookup("test.run").Value.String() + integrationRE := regexp.MustCompile(strings.Split(runStr, "/")[0]) + const integrationPattern = "TestIntegration" + if runStr == "" || integrationRE.FindString(integrationPattern) == "" { + os.Exit(m.Run()) + } + + // DB2 first-time setup (CREATE DATABASE + LOGARCHMETH1 restart) takes + // 8–12 min on modern hardware. The context must outlive that window. + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute) + defer cancel() + + // Warm up the shared container before any test runs. If Docker is + // unavailable the individual tests skip themselves via SetupTest. + if _, _, err := db2test.AcquireSharedContainer(ctx); err != nil { + // Container unavailable — individual tests will skip themselves. + // Exit 0 so the test binary reports success (skipped, not failed). + fmt.Fprintf(os.Stderr, "db2test: AcquireSharedContainer failed: %v\n", err) + os.Exit(0) + } + + code := m.Run() + + db2test.TerminateSharedContainer(context.Background()) + os.Exit(code) +} diff --git a/internal/plugins/info.csv b/internal/plugins/info.csv index 85605d9bd1..ba59943a12 100644 --- a/internal/plugins/info.csv +++ b/internal/plugins/info.csv @@ -67,6 +67,7 @@ csv ,input ,csv ,certified ,n csv ,scanner ,csv ,certified ,n ,y ,y , cyborgdb ,output ,cyborgdb ,community ,n ,y ,y , cypher ,output ,cypher ,community ,n ,n ,n ,not yet certified for cloud +db2_cdc ,input ,db2_cdc ,enterprise ,n ,n ,n ,requires IBM libdb2.so / libdb2.dylib / db2cli.dll or equivalent on system where this connector runs decompress ,processor ,decompress ,certified ,n ,y ,y , decompress ,scanner ,decompress ,certified ,n ,y ,y , dedupe ,processor ,dedupe ,certified ,n ,y ,y , diff --git a/public/components/all/package.go b/public/components/all/package.go index f6b5ca4179..530b2817c2 100644 --- a/public/components/all/package.go +++ b/public/components/all/package.go @@ -17,6 +17,7 @@ import ( _ "github.com/redpanda-data/connect/v4/public/components/community" // Import all enterprise components. + _ "github.com/redpanda-data/connect/v4/public/components/db2" _ "github.com/redpanda-data/connect/v4/public/components/gateway" _ "github.com/redpanda-data/connect/v4/public/components/gcp/enterprise" _ "github.com/redpanda-data/connect/v4/public/components/google" diff --git a/public/components/db2/package.go b/public/components/db2/package.go new file mode 100644 index 0000000000..8593b7b987 --- /dev/null +++ b/public/components/db2/package.go @@ -0,0 +1,14 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +package db2 + +import ( + // Bring in the internal plugin definitions. + _ "github.com/redpanda-data/connect/v4/internal/impl/db2" +) diff --git a/scripts/run-db2-macos-integration-tests.sh b/scripts/run-db2-macos-integration-tests.sh new file mode 100755 index 0000000000..8f81a30abb --- /dev/null +++ b/scripts/run-db2-macos-integration-tests.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# Run DB2 CDC integration tests locally. +# +# On macOS: runs the test binary NATIVELY on the host using IBM's clidriver +# (libdb2.dylib). No docker-exec, no cross-compilation. +# +# On Linux (CI): cross-compiles for linux/amd64, copies the binary into the +# DB2 container, and runs it there with DB2_USE_LOCAL=1 (unchanged behaviour). +# +# Usage: +# ./scripts/run-db2-macos-integration-tests.sh +# ./scripts/run-db2-macos-integration-tests.sh -test.run TestIntegrationDB2CDCDriver +# +# Flags: +# -timeout-minutes N Override test timeout (default 40). +# -test.run= Override which tests to run. +# --keep Keep the DB2 container alive after the run. +# --reset Remove the named persistent container and volume, +# then start fresh. +# +# macOS — install the IBM clidriver (one of): +# pip3 install --user ibm_db # bundles libdb2.dylib in the Python package +# export DB2_DYLIB_PATH=/path/to/libdb2.dylib +# +# See: https://www.ibm.com/support/pages/db2-odbc-cli-driver-download-and-installation-information +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OS="$(uname -s)" +BIN="$(mktemp -t db2test.XXXXXX)" +DB2_CTR_NAME="${DB2_CONTAINER_NAME:-db2-integration-test-persistent}" +DB2_VOLUME="db2-testdb-data" +SETUP_LOG="$(mktemp -t db2setup.XXXXXX)" +LOG_PID="" +KEEP=false +RESET=false + +cleanup() { + [[ -n "$LOG_PID" ]] && kill "$LOG_PID" 2>/dev/null || true + rm -f "$BIN" "$SETUP_LOG" + if [[ "$KEEP" == "false" ]]; then + echo "==> Cleaning up container: $DB2_CTR_NAME" + docker rm -f "$DB2_CTR_NAME" 2>/dev/null || true + else + echo "==> Keeping container $DB2_CTR_NAME alive for reuse (--keep was set)" + fi +} +trap cleanup EXIT + +# -------------------------------------------------------------------------- +# Parse flags +# -------------------------------------------------------------------------- +TEST_TIMEOUT="40m" +TEST_RUN="^TestIntegration" +TEST_EXTRA_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + -timeout-minutes) + shift; TEST_TIMEOUT="${1}m"; shift ;; + -test.run=*) + TEST_RUN="${1#-test.run=}"; shift ;; + --keep) + KEEP=true; shift ;; + --reset) + RESET=true; shift ;; + *) + TEST_EXTRA_ARGS+=("$1"); shift ;; + esac +done + +# -------------------------------------------------------------------------- +# macOS: locate libdb2.dylib (IBM clidriver — TCP-only, no SysV IPC) +# -------------------------------------------------------------------------- +find_darwin_lib() { + # 1. Explicit env var override + if [[ -n "${DB2_DYLIB_PATH:-}" ]]; then + if [[ -f "$DB2_DYLIB_PATH" ]]; then + echo "$DB2_DYLIB_PATH" + return 0 + fi + echo "ERROR: DB2_DYLIB_PATH='$DB2_DYLIB_PATH' does not exist" >&2 + return 1 + fi + + # 2. ibm_db Python package bundles the clidriver + local ibm_db_dir + ibm_db_dir=$(python3 -c 'import ibm_db, os; print(os.path.dirname(ibm_db.__file__))' 2>/dev/null || true) + if [[ -n "$ibm_db_dir" ]]; then + local candidate="$ibm_db_dir/clidriver/lib/libdb2.dylib" + if [[ -f "$candidate" ]]; then + echo "$candidate" + return 0 + fi + fi + + # 3. Standard IBM installation paths + for p in \ + "/Library/IBM/SQLLIB/lib64/libdb2.dylib" \ + "/Library/IBM/SQLLIB/lib/libdb2.dylib" \ + "$HOME/sqllib/lib64/libdb2.dylib"; do + if [[ -f "$p" ]]; then + echo "$p" + return 0 + fi + done + + cat >&2 <<'EOF' +ERROR: IBM DB2 ODBC CLI Driver (libdb2.dylib) not found. + +Install options (choose one): + pip3 install --user ibm_db # pip bundles the clidriver; --user avoids sudo + export DB2_DYLIB_PATH=/path/to/libdb2.dylib + +IBM official download: + https://www.ibm.com/support/pages/db2-odbc-cli-driver-download-and-installation-information +EOF + return 1 +} + +# -------------------------------------------------------------------------- +# Reset: destroy container + volume so next run starts fresh. +# -------------------------------------------------------------------------- +if [[ "$RESET" == "true" ]]; then + echo "==> --reset: removing container $DB2_CTR_NAME and volume $DB2_VOLUME" + docker rm -f "$DB2_CTR_NAME" 2>/dev/null || true + docker volume rm "$DB2_VOLUME" 2>/dev/null || true + echo "==> Reset complete. Re-run without --reset to start fresh." + exit 0 +fi + +# -------------------------------------------------------------------------- +# Reuse: if the container already exists and is running, skip setup entirely. +# -------------------------------------------------------------------------- +CONTAINER_REUSED=false +if docker ps -q --filter "name=^${DB2_CTR_NAME}$" | grep -q .; then + echo "==> Reusing existing container $DB2_CTR_NAME (skipping ~8-min first-time setup)" + CONTAINER_REUSED=true +elif docker ps -aq --filter "name=^${DB2_CTR_NAME}$" | grep -q .; then + echo "==> Starting stopped container $DB2_CTR_NAME" + docker start "$DB2_CTR_NAME" + CONTAINER_REUSED=true +fi + +# -------------------------------------------------------------------------- +# First run: pull image, create volume, start container. +# -------------------------------------------------------------------------- +if [[ "$CONTAINER_REUSED" == "false" ]]; then + echo "==> Pulling DB2 image (no-op if already cached)" + docker pull --platform linux/amd64 icr.io/db2_community/db2:latest + + echo "==> Creating named volume $DB2_VOLUME (no-op if already exists)" + docker volume create "$DB2_VOLUME" >/dev/null + + echo "==> Starting DB2 container $DB2_CTR_NAME" + if [[ "$OS" == "Darwin" ]]; then + # macOS: clidriver connects via TCP/IP — no SysV IPC, so --ipc=host is not needed. + docker run --detach \ + --name "$DB2_CTR_NAME" \ + --platform linux/amd64 \ + --privileged \ + --shm-size 512m \ + --volume "${DB2_VOLUME}:/database" \ + -p 50000:50000 \ + -e LICENSE=accept \ + -e DB2INST1_PASSWORD=password \ + -e DBNAME=TESTDB \ + -e ARCHIVE_LOGS=true \ + -e AUTOCONFIG=false \ + -e SAMPLEDB=false \ + -e REPODB=false \ + -e HADR_ENABLED=NO \ + -e UPDATEAVAIL=NO \ + icr.io/db2_community/db2:latest + else + docker run --detach \ + --name "$DB2_CTR_NAME" \ + --platform linux/amd64 \ + --privileged \ + --ipc=host \ + --shm-size 512m \ + --volume "${DB2_VOLUME}:/database" \ + -p 50000:50000 \ + -e LICENSE=accept \ + -e DB2INST1_PASSWORD=password \ + -e DBNAME=TESTDB \ + -e ARCHIVE_LOGS=true \ + -e AUTOCONFIG=false \ + -e SAMPLEDB=false \ + -e REPODB=false \ + -e HADR_ENABLED=NO \ + -e UPDATEAVAIL=NO \ + icr.io/db2_community/db2:latest + fi + + docker logs -f "$DB2_CTR_NAME" 2>&1 | tee "$SETUP_LOG" | sed 's/^/[db2] /' & + LOG_PID=$! + + echo "==> Waiting for DB2 first-time setup (up to 25 min)..." + SETUP_DEADLINE=$(( $(date +%s) + 1500 )) + while true; do + if grep -q "Setup has completed" "$SETUP_LOG" 2>/dev/null; then + echo "==> DB2 setup complete" + break + fi + if [[ $(date +%s) -ge $SETUP_DEADLINE ]]; then + echo "ERROR: DB2 setup did not complete within 25 min" >&2 + exit 1 + fi + if ! docker ps -q --filter "name=^${DB2_CTR_NAME}$" | grep -q . 2>/dev/null; then + echo "ERROR: DB2 container exited before setup completed" >&2 + exit 1 + fi + sleep 5 + done + + kill "$LOG_PID" 2>/dev/null || true + LOG_PID="" +fi + +# -------------------------------------------------------------------------- +# Build and run tests. +# -------------------------------------------------------------------------- +cd "$REPO_ROOT" + +if [[ "$OS" == "Darwin" ]]; then + # Locate clidriver before building so we fail fast with a helpful message. + FOUND_DYLIB=$(find_darwin_lib) + + echo "==> Building test binary for macOS host (native, no cross-compilation)" + go test -c -o "$BIN" ./internal/impl/db2/ + + echo "==> Running integration tests natively on macOS (clidriver: $FOUND_DYLIB)" + DB2_DYLIB_PATH="$FOUND_DYLIB" \ + DB2_DARWIN_HOST=localhost \ + DB2_DARWIN_PORT=50000 \ + DB2_DARWIN_CONTAINER="$DB2_CTR_NAME" \ + "$BIN" -test.v -test.timeout "$TEST_TIMEOUT" "-test.run=${TEST_RUN}" "${TEST_EXTRA_ARGS[@]+"${TEST_EXTRA_ARGS[@]}"}" +else + echo "==> Building linux/amd64 test binary" + GOOS=linux GOARCH=amd64 go test -c -o "$BIN" ./internal/impl/db2/ + + echo "==> Copying test binary into $DB2_CTR_NAME" + docker cp "$BIN" "$DB2_CTR_NAME:/tmp/db2test" + docker exec "$DB2_CTR_NAME" chmod 755 /tmp/db2test + + echo "==> Running integration tests inside DB2 container (timeout: $TEST_TIMEOUT, run: $TEST_RUN)" + docker exec \ + -e DB2_USE_LOCAL=1 \ + -e DB2DIR=/opt/ibm/db2/V12.1 \ + -e DB2INSTANCE=db2inst1 \ + -e DB2MSGPATH=/opt/ibm/db2/V12.1/msg \ + -e LANG=en_US.iso88591 \ + "$DB2_CTR_NAME" \ + /tmp/db2test -test.v -test.timeout "$TEST_TIMEOUT" "-test.run=${TEST_RUN}" "${TEST_EXTRA_ARGS[@]+"${TEST_EXTRA_ARGS[@]}"}" +fi