Skip to content

Commit ce6e761

Browse files
bvt123claude
bvt123
and
claude
committed
refactor(tests): Unify e2e-core and e2e-optional into tests/e2e
- Merge tests/e2e-core/ and tests/e2e-optional/ into single tests/e2e/ - Delete non-aligned tests (kafka2kafka, mysql2kafka, pg2pg) - Remove duplicate kafka2ch from e2e-optional - Update Makefile: LAYER=e2e-core -> LAYER=e2e - Update CI workflows to use tests/e2e paths - Update matrix YAML files (cdc_local_suite, cdc_optional_suite, core2ch, sources) - Fix MV column alias mismatch in kafka2ch/replication_mv - Switch from external Zookeeper to ClickHouse built-in Keeper - Disable optional e2e tests by default in CI - Update AGENTS.md and tests/README.md documentation Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 01e3d92 commit ce6e761

166 files changed

Lines changed: 252 additions & 1318 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yml‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -56,17 +56,16 @@ jobs:
5656
matrix:
5757
suite: [
5858
# core e2e suites
59-
{ group: "tests/e2e-core", name: "pg2ch", path: "pg2ch" },
60-
{ group: "tests/e2e-core", name: "mysql2ch", path: "mysql2ch" },
61-
{ group: "tests/e2e-core", name: "mongo2ch", path: "mongo2ch" },
62-
{ group: "tests/e2e-core", name: "kafka2ch", path: "kafka2ch" },
59+
{ group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
60+
{ group: "tests/e2e", name: "mysql2ch", path: "mysql2ch" },
61+
{ group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
62+
{ group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
6363
# optional e2e suites
64-
{ group: "tests/e2e-optional", name: "airbyte2ch", path: "airbyte2ch" },
65-
{ group: "tests/e2e-optional", name: "ch2ch", path: "ch2ch" },
66-
{ group: "tests/e2e-optional", name: "eventhub2ch", path: "eventhub2ch" },
67-
{ group: "tests/e2e-optional", name: "kafka2ch-optional", path: "kafka2ch" },
68-
{ group: "tests/e2e-optional", name: "kinesis2ch", path: "kinesis2ch" },
69-
{ group: "tests/e2e-optional", name: "oracle2ch", path: "oracle2ch" },
64+
{ group: "tests/e2e", name: "airbyte2ch", path: "airbyte2ch" },
65+
{ group: "tests/e2e", name: "ch2ch", path: "ch2ch" },
66+
{ group: "tests/e2e", name: "eventhub2ch", path: "eventhub2ch" },
67+
{ group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
68+
{ group: "tests/e2e", name: "oracle2ch", path: "oracle2ch" },
7069
]
7170
steps:
7271
- name: Checkout

‎.github/workflows/build_and_test.yml‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -60,17 +60,16 @@ jobs:
6060
matrix:
6161
suite: [
6262
# core e2e suites
63-
{ group: "tests/e2e-core", name: "pg2ch", path: "pg2ch" },
64-
{ group: "tests/e2e-core", name: "mysql2ch", path: "mysql2ch" },
65-
{ group: "tests/e2e-core", name: "mongo2ch", path: "mongo2ch" },
66-
{ group: "tests/e2e-core", name: "kafka2ch", path: "kafka2ch" },
63+
{ group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
64+
{ group: "tests/e2e", name: "mysql2ch", path: "mysql2ch" },
65+
{ group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
66+
{ group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
6767
# optional e2e suites
68-
{ group: "tests/e2e-optional", name: "airbyte2ch", path: "airbyte2ch" },
69-
{ group: "tests/e2e-optional", name: "ch2ch", path: "ch2ch" },
70-
{ group: "tests/e2e-optional", name: "eventhub2ch", path: "eventhub2ch" },
71-
{ group: "tests/e2e-optional", name: "kafka2ch-optional", path: "kafka2ch" },
72-
{ group: "tests/e2e-optional", name: "kinesis2ch", path: "kinesis2ch" },
73-
{ group: "tests/e2e-optional", name: "oracle2ch", path: "oracle2ch" },
68+
{ group: "tests/e2e", name: "airbyte2ch", path: "airbyte2ch" },
69+
{ group: "tests/e2e", name: "ch2ch", path: "ch2ch" },
70+
{ group: "tests/e2e", name: "eventhub2ch", path: "eventhub2ch" },
71+
{ group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
72+
{ group: "tests/e2e", name: "oracle2ch", path: "oracle2ch" },
7473
]
7574
steps:
7675
- name: Checkout

‎.github/workflows/reusable-ci.yml‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ on:
1515
run_optional:
1616
description: "Run optional e2e suites"
1717
required: false
18-
default: true
18+
default: false
1919
type: boolean
2020
go_version:
2121
description: "Go version to use"
@@ -152,20 +152,20 @@ jobs:
152152
if: failure()
153153
run: exit 1
154154

155-
e2e-core:
155+
e2e:
156156
if: inputs.run_e2e && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
157157
needs: [build]
158-
name: e2e-core / ${{ inputs.stream }} / ${{ matrix.suite.name }}
158+
name: e2e / ${{ inputs.stream }} / ${{ matrix.suite.name }}
159159
runs-on: ubuntu-latest
160160
timeout-minutes: 30
161161
strategy:
162162
fail-fast: false
163163
matrix:
164164
suite: [
165-
{ group: "tests/e2e-core", name: "pg2ch", path: "pg2ch" },
166-
{ group: "tests/e2e-core", name: "mysql2ch", path: "mysql2ch" },
167-
{ group: "tests/e2e-core", name: "mongo2ch", path: "mongo2ch" },
168-
{ group: "tests/e2e-core", name: "kafka2ch", path: "kafka2ch" },
165+
{ group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
166+
{ group: "tests/e2e", name: "mysql2ch", path: "mysql2ch" },
167+
{ group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
168+
{ group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
169169
]
170170
steps:
171171
- name: Checkout
@@ -211,12 +211,11 @@ jobs:
211211
fail-fast: false
212212
matrix:
213213
suite: [
214-
{ group: "tests/e2e-optional", name: "airbyte2ch", path: "airbyte2ch" },
215-
{ group: "tests/e2e-optional", name: "ch2ch", path: "ch2ch" },
216-
{ group: "tests/e2e-optional", name: "eventhub2ch", path: "eventhub2ch" },
217-
{ group: "tests/e2e-optional", name: "kafka2ch", path: "kafka2ch" },
218-
{ group: "tests/e2e-optional", name: "kinesis2ch", path: "kinesis2ch" },
219-
{ group: "tests/e2e-optional", name: "oracle2ch", path: "oracle2ch" },
214+
{ group: "tests/e2e", name: "airbyte2ch", path: "airbyte2ch" },
215+
{ group: "tests/e2e", name: "ch2ch", path: "ch2ch" },
216+
{ group: "tests/e2e", name: "eventhub2ch", path: "eventhub2ch" },
217+
{ group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
218+
{ group: "tests/e2e", name: "oracle2ch", path: "oracle2ch" },
220219
]
221220
steps:
222221
- name: Checkout
@@ -254,7 +253,7 @@ jobs:
254253

255254
test-report:
256255
if: always() && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
257-
needs: [generic-tests, e2e-core, e2e-optional]
256+
needs: [generic-tests, e2e, e2e-optional]
258257
name: test-report / ${{ inputs.stream }}
259258
runs-on: ubuntu-latest
260259
steps:

‎AGENTS.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ transferia/
2929
│ └── connection/ # Connection management
3030
├── internal/ # Internal packages (logger, config, metrics)
3131
├── tests/
32-
│ ├── e2e-core/ # End-to-end tests (pg2ch, mysql2ch, mongo2ch)
32+
│ ├── e2e/ # End-to-end tests (pg2ch, mysql2ch, mongo2ch)
3333
│ ├── helpers/ # Test utilities and helpers
3434
│ ├── canon/ # Type/schema validation tests
3535
│ └── storage/ # Provider storage tests
@@ -146,7 +146,7 @@ Providers register via `init()` with:
146146
2. Implement required interfaces (Storage, Source, Sink as needed)
147147
3. Register in `init()` function
148148
4. Add test recipes in `recipe/`
149-
5. Create e2e tests in `tests/e2e-core/`
149+
5. Create e2e tests in `tests/e2e/`
150150

151151
### Adding a Transformer
152152

@@ -168,7 +168,7 @@ make test-cdc-full
168168
make test-cdc-wave WAVE=providers
169169

170170
# Specific layer
171-
make test-layer LAYER=e2e-core DB=pg2ch
171+
make test-layer LAYER=e2e DB=pg2ch
172172
```
173173

174174
## Build Commands

‎Makefile‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -30,21 +30,21 @@ test:
3030

3131

3232
# Define variables for the suite group, path, and name with defaults
33-
SUITE_GROUP ?= 'tests/e2e-core'
33+
SUITE_GROUP ?= 'tests/e2e'
3434
SUITE_PATH ?= 'pg2ch'
35-
SUITE_NAME ?= 'e2e-core-pg2ch'
35+
SUITE_NAME ?= 'e2e-pg2ch'
3636
GO_TEST_ARGS ?= -timeout=15m
3737
SHELL := /bin/bash
3838
GOTESTSUM_FORMAT ?= standard-quiet
3939
ifeq ($(GITHUB_ACTIONS),true)
4040
GOTESTSUM_FORMAT = github-actions
4141
endif
42-
MATRIX_CONTRACT ?= tests/e2e-core/matrix/core2ch.yaml
42+
MATRIX_CONTRACT ?= tests/e2e/matrix/core2ch.yaml
4343
MATRIX_TOOL ?= go run ./tools/testmatrix
44-
MATRIX_REPORT ?= tests/e2e-core/matrix/coverage_report.md
44+
MATRIX_REPORT ?= tests/e2e/matrix/coverage_report.md
4545
MATRIX_TEST_GO_ARGS ?= -count=1 -timeout=20m
46-
CDC_SUITE_MANIFEST ?= tests/e2e-core/matrix/cdc_local_suite.yaml
47-
CDC_OPTIONAL_SUITE_MANIFEST ?= tests/e2e-core/matrix/cdc_optional_suite.yaml
46+
CDC_SUITE_MANIFEST ?= tests/e2e/matrix/cdc_local_suite.yaml
47+
CDC_OPTIONAL_SUITE_MANIFEST ?= tests/e2e/matrix/cdc_optional_suite.yaml
4848
CDC_GO_TEST_ARGS ?= -timeout=20m
4949
TEST_STATE_DIR ?= .teststate
5050
TEST_STATE_WAVES_DIR ?= $(TEST_STATE_DIR)/waves
@@ -57,35 +57,35 @@ SUPPORTED_FLOW_DBS := pg2ch mysql2ch mongo2ch
5757
SUPPORTED_COMPONENT_DBS := postgres mysql mongo
5858
SUPPORTED_STREAM_FLOW_DBS := kafka2ch
5959
SUPPORTED_OPTIONAL_FLOW_DBS := kafka2ch eventhub2ch kinesis2ch airbyte2ch oracle2ch ch2ch
60-
SUPPORTED_LAYERS := storage canon e2e-core evolution resume large
60+
SUPPORTED_LAYERS := storage canon e2e evolution resume large
6161
SUPPORTED_SOURCE_VARIANTS := \
6262
postgres/17 postgres/18 \
6363
mysql/mysql84 mysql/mariadb118 \
6464
mongo/6 mongo/7 \
6565
kafka/confluent75 kafka/redpanda24
6666
RESUME_TEST_PATTERN ?= ResumeFromCoordinator|Resume
67-
LAYER ?= e2e-core
67+
LAYER ?= e2e
6868
DB ?= pg2ch
6969
SOURCE_VARIANT ?=
7070
MATRIX_FAMILY ?= postgres
71-
MATRIX_CORE_LAYERS ?= e2e-core evolution large
71+
MATRIX_CORE_LAYERS ?= e2e evolution large
7272
MATRIX_GO_TEST_ARGS ?= -count=1 -timeout=15m
73-
KAFKA_MATRIX_LAYERS ?= e2e-core evolution large
74-
CDC_WAVES := providers storage-canon e2e-core evolution resume large
73+
KAFKA_MATRIX_LAYERS ?= e2e evolution large
74+
CDC_WAVES := providers storage-canon e2e evolution resume large
7575
WAVE_TARGETS := $(addprefix $(TEST_STATE_WAVES_DIR)/,$(addsuffix .ok,$(CDC_WAVES)))
7676
CDC_WAVE_SHARED_PATHS := library pkg vendor_patched tools/testmatrix
7777
WAVE_PATHS_providers := tests/helpers tests/tcrecipes
7878
WAVE_PATHS_storage-canon := tests/storage tests/canon
79-
WAVE_PATHS_e2e-core := tests/e2e-core
79+
WAVE_PATHS_e2e := tests/e2e
8080
WAVE_PATHS_evolution := tests/evolution
8181
WAVE_PATHS_resume := tests/resume
8282
WAVE_PATHS_large := tests/large
8383
CDC_OPTIONAL_WAVES := optional-queues optional-connectors optional-clickhouse-source
8484
OPTIONAL_WAVE_TARGETS := $(addprefix $(TEST_STATE_OPTIONAL_WAVES_DIR)/,$(addsuffix .ok,$(CDC_OPTIONAL_WAVES)))
8585
CDC_OPTIONAL_WAVE_SHARED_PATHS := library pkg vendor_patched tools/testmatrix
86-
OPTIONAL_WAVE_PATHS_optional-queues := tests/e2e-optional/kafka2ch tests/e2e-optional/eventhub2ch tests/e2e-optional/kinesis2ch tests/tcrecipes
87-
OPTIONAL_WAVE_PATHS_optional-connectors := tests/e2e-optional/airbyte2ch tests/e2e-optional/oracle2ch tests/tcrecipes
88-
OPTIONAL_WAVE_PATHS_optional-clickhouse-source := tests/e2e-optional/ch2ch
86+
OPTIONAL_WAVE_PATHS_optional-queues := tests/e2e/kafka2ch tests/e2e/eventhub2ch tests/e2e/kinesis2ch tests/tcrecipes
87+
OPTIONAL_WAVE_PATHS_optional-connectors := tests/e2e/airbyte2ch tests/e2e/oracle2ch tests/tcrecipes
88+
OPTIONAL_WAVE_PATHS_optional-clickhouse-source := tests/e2e/ch2ch
8989

9090
define LIST_TRACKED_FILES
9191
$(strip $(shell \
@@ -101,15 +101,15 @@ endef
101101
COMMON_WAVE_DEPS := Makefile go.mod go.sum $(CDC_SUITE_MANIFEST) $(MATRIX_CONTRACT) $(call LIST_TRACKED_FILES,$(CDC_WAVE_SHARED_PATHS))
102102
WAVE_DEPS_providers := $(call LIST_TRACKED_FILES,$(WAVE_PATHS_providers))
103103
WAVE_DEPS_storage-canon := $(call LIST_TRACKED_FILES,$(WAVE_PATHS_storage-canon))
104-
WAVE_DEPS_e2e-core := $(call LIST_TRACKED_FILES,$(WAVE_PATHS_e2e-core))
104+
WAVE_DEPS_e2e := $(call LIST_TRACKED_FILES,$(WAVE_PATHS_e2e))
105105
WAVE_DEPS_evolution := $(call LIST_TRACKED_FILES,$(WAVE_PATHS_evolution))
106106
WAVE_DEPS_resume := $(call LIST_TRACKED_FILES,$(WAVE_PATHS_resume))
107107
WAVE_DEPS_large := $(call LIST_TRACKED_FILES,$(WAVE_PATHS_large))
108108
COMMON_OPTIONAL_WAVE_DEPS := Makefile go.mod go.sum $(CDC_OPTIONAL_SUITE_MANIFEST) $(call LIST_TRACKED_FILES,$(CDC_OPTIONAL_WAVE_SHARED_PATHS))
109109
OPTIONAL_WAVE_DEPS_optional-queues := $(call LIST_TRACKED_FILES,$(OPTIONAL_WAVE_PATHS_optional-queues))
110110
OPTIONAL_WAVE_DEPS_optional-connectors := $(call LIST_TRACKED_FILES,$(OPTIONAL_WAVE_PATHS_optional-connectors))
111111
OPTIONAL_WAVE_DEPS_optional-clickhouse-source := $(call LIST_TRACKED_FILES,$(OPTIONAL_WAVE_PATHS_optional-clickhouse-source))
112-
MATRIX_CACHE_SHARED_PATHS := library pkg vendor_patched tools/testmatrix tests/e2e-core tests/evolution tests/large
112+
MATRIX_CACHE_SHARED_PATHS := library pkg vendor_patched tools/testmatrix tests/e2e tests/evolution tests/large
113113
COMMON_MATRIX_DEPS := Makefile go.mod go.sum $(CDC_SUITE_MANIFEST) $(MATRIX_CONTRACT) $(call LIST_TRACKED_FILES,$(MATRIX_CACHE_SHARED_PATHS))
114114

115115
# Define the `run-tests` target
@@ -200,7 +200,7 @@ test-list:
200200
@echo "Supported stream flow DB aliases: $(SUPPORTED_STREAM_FLOW_DBS)"
201201
@echo "Supported component DB names: $(SUPPORTED_COMPONENT_DBS)"
202202
@echo "Examples:"
203-
@echo " make test-layer LAYER=e2e-core DB=pg2ch"
203+
@echo " make test-layer LAYER=e2e DB=pg2ch"
204204
@echo " make test-layer-all LAYER=resume"
205205
@echo " make test-db DB=mysql2ch"
206206
@echo " make test-core"
@@ -687,7 +687,7 @@ test-layer:
687687
*) echo "Unsupported DB alias: $$db. Use one of: $(SUPPORTED_FLOW_DBS) $(SUPPORTED_STREAM_FLOW_DBS)"; exit 1 ;; \
688688
esac; \
689689
case "$$layer" in \
690-
e2e-core) suite_group="tests/e2e-core"; suite_path="$$db" ;; \
690+
e2e) suite_group="tests/e2e"; suite_path="$$db" ;; \
691691
evolution|resume|large) suite_group="tests"; suite_path="$$layer/$$db" ;; \
692692
canon) [[ "$$db" == "kafka2ch" ]] && { echo "canon layer is not defined for $$db"; exit 1; }; suite_group="tests"; suite_path="canon/$$source_db" ;; \
693693
storage) [[ "$$db" == "kafka2ch" ]] && { echo "storage layer is not defined for $$db"; exit 1; }; suite_group="tests"; suite_path="storage/$$source_db" ;; \
@@ -719,12 +719,12 @@ test-layer-optional:
719719
kafka2ch|eventhub2ch|kinesis2ch|airbyte2ch|oracle2ch|ch2ch) ;; \
720720
*) echo "Unsupported optional DB alias: $$db. Use one of: $(SUPPORTED_OPTIONAL_FLOW_DBS)"; exit 1 ;; \
721721
esac; \
722-
$(MAKE) run-tests SUITE_GROUP="tests/e2e-optional" SUITE_PATH="$$db" SUITE_NAME="e2e-optional-$$db" GO_TEST_ARGS="$(MATRIX_GO_TEST_ARGS)"
722+
$(MAKE) run-tests SUITE_GROUP="tests/e2e" SUITE_PATH="$$db" SUITE_NAME="e2e-$$db" GO_TEST_ARGS="$(MATRIX_GO_TEST_ARGS)"
723723

724724
.PHONY: test-db
725725
test-db:
726726
@set -euo pipefail; \
727-
for layer in storage canon e2e-core evolution resume large; do \
727+
for layer in storage canon e2e evolution resume large; do \
728728
echo "=== layer=$$layer db=$(DB) ==="; \
729729
$(MAKE) test-layer LAYER="$$layer" DB="$(DB)"; \
730730
done
@@ -736,7 +736,7 @@ test-core:
736736
echo "=== core db=$$db ==="; \
737737
$(MAKE) test-layer LAYER=storage DB="$$db"; \
738738
$(MAKE) test-layer LAYER=canon DB="$$db"; \
739-
$(MAKE) test-layer LAYER=e2e-core DB="$$db"; \
739+
$(MAKE) test-layer LAYER=e2e DB="$$db"; \
740740
$(MAKE) test-layer LAYER=resume DB="$$db"; \
741741
done
742742

‎pkg/providers/clickhouse/recipe/chrecipe.go‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package chrecipe
22

33
import (
44
"context"
5-
"fmt"
65
"os"
76
"strconv"
87

@@ -225,18 +224,12 @@ func Prepare(params ContainerParams) error {
225224
}
226225
ctx, cancel := context.WithCancel(context.Background())
227226
defer cancel()
228-
// test running outside arcadia
229-
zk, err := tc_clickhouse.PrepareZK(ctx)
230-
if err != nil {
231-
return xerrors.Errorf("unable to prepare Zookeeper: %w", err)
232-
}
233-
fmt.Printf("zk: 0.0.0.0:%s \n", zk.Port().Port())
234227

235228
chcntr, err := tc_clickhouse.Prepare(
236229
ctx,
237230
tc_clickhouse.WithDatabase("default"),
238231
tc_clickhouse.WithUsername(params.user),
239-
tc_clickhouse.WithZookeeper(zk),
232+
tc_clickhouse.WithKeeper(),
240233
tc_clickhouse.WithInitScripts(params.initScripts...),
241234
)
242235
if err != nil {

0 commit comments

Comments
 (0)