Skip to content

Commit b6d43ac

Browse files
Merge pull request #31049 from mnajda-redpanda/mnajda/confluent-source-sync
[CORE-16776] cluster_link: test SR-sync against a Confluent source
2 parents 704889a + 589e7dd commit b6d43ac

12 files changed

Lines changed: 429 additions & 62 deletions

File tree

src/v/cluster_link/schema_registry_sync/mirroring_task.cc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,11 @@ ss::future<task::state_transition> mirroring_task::full_source_sync(
602602
node);
603603
if (!dest_deleted) {
604604
work.upserts.push_back(node);
605+
// Authoritative deleted-state from the listing partition: the
606+
// reconciler imports these soft-deleted regardless of the
607+
// per-version source body, which a standard source (e.g. Confluent)
608+
// does not populate with a `deleted` flag by default.
609+
work.soft_deleted.insert(node);
605610
}
606611
}
607612

src/v/cluster_link/schema_registry_sync/reconciler.cc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ ss::future<source_result<void>> reconciler::reconcile(
136136
reconcile_stats& stats,
137137
ss::abort_source& as) {
138138
_replicated = std::move(seed_replicated);
139+
_soft_deleted = std::move(work.soft_deleted);
139140
_nodes.clear();
140141
_discover_q.clear();
141142
_import_q.clear();
@@ -438,6 +439,12 @@ ss::future<bool> reconciler::import_body(
438439
if (fail_if_contains_unsupported(n, read.unsupported)) {
439440
co_return false;
440441
}
442+
// Deleted-state is authoritative from the caller's active-vs-deleted
443+
// listing partition, not the per-version source body: a standard source
444+
// (e.g. Confluent) omits the `deleted` flag from that body by default, so
445+
// the fetched flag cannot be trusted to propagate a soft-delete.
446+
read.schema.deleted = _soft_deleted.contains(n) ? ppsr::is_deleted::yes
447+
: ppsr::is_deleted::no;
441448
data(n).state = node_state::importing;
442449
// The graph key `n` stays in the source namespace; only the schema written
443450
// to the destination is remapped.

src/v/cluster_link/schema_registry_sync/reconciler.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,16 @@ struct reconcile_stats {
5151
/// version) node imported from the source with its source deleted state. The
5252
/// caller selects which nodes to upsert (create, reactivate, or propagate a
5353
/// soft-delete); the reconciler imports them in reference order.
54+
///
55+
/// `soft_deleted` names the upserts that are soft-deleted on the source, so the
56+
/// reconciler imports them soft-deleted regardless of the per-version source
57+
/// response, which cannot be relied on for deleted-state: a standard source
58+
/// (e.g. Confluent) omits the `deleted` flag from the per-version body by
59+
/// default. The caller instead derives this from its own active-vs-deleted
60+
/// listing partition, which is authoritative across SR vendors.
5461
struct work_set {
5562
chunked_vector<ppsr::subject_version> upserts;
63+
chunked_hash_set<ppsr::subject_version> soft_deleted;
5664
};
5765

5866
/// \brief Imports missing source schema versions into the destination in
@@ -222,6 +230,9 @@ class reconciler {
222230
_feature_policy;
223231

224232
chunked_hash_set<ppsr::subject_version> _replicated;
233+
/// Upserts the caller classified as soft-deleted on the source; imported
234+
/// soft-deleted, overriding the per-version source body's deleted flag.
235+
chunked_hash_set<ppsr::subject_version> _soft_deleted;
225236
chunked_hash_map<ppsr::subject_version, node_data> _nodes;
226237
chunked_vector<ppsr::subject_version> _discover_q;
227238
chunked_vector<ppsr::subject_version> _import_q;

src/v/cluster_link/schema_registry_sync/tests/reconciler_test.cc

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -365,8 +365,8 @@ TEST(reconciler, seed_replicated_upsert_is_still_imported) {
365365
}
366366

367367
// A soft-deleted source version imports preserving its deleted state: the
368-
// reconciler fetches the body (which carries the flag) and import_schema stores
369-
// it soft-deleted. Underpins the task syncing soft-deleted source versions
368+
// caller declares it in work_set.soft_deleted and import_schema stores it
369+
// soft-deleted. Underpins the task syncing soft-deleted source versions
370370
// preserving their deleted state.
371371
TEST(reconciler, imports_soft_deleted_as_deleted) {
372372
reconcile_harness h;
@@ -375,6 +375,7 @@ TEST(reconciler, imports_soft_deleted_as_deleted) {
375375

376376
srs::work_set work;
377377
work.upserts.push_back(key(a, 1));
378+
work.soft_deleted.insert(key(a, 1));
378379

379380
auto stats = h.run(std::move(work)).get();
380381
ASSERT_TRUE(stats.has_value());
@@ -388,10 +389,11 @@ TEST(reconciler, imports_soft_deleted_as_deleted) {
388389
}
389390

390391
// Soft-delete propagation onto an active destination version: re-importing the
391-
// source body (which now carries deleted=yes) transitions an existing active
392-
// version to soft-deleted. The node is in the seed (a destination version is in
393-
// `all`), so this also guards that a seeded upsert is still imported. Underpins
394-
// the task propagating a source soft-delete onto a live destination version.
392+
// version with the caller's declared deleted-state transitions an existing
393+
// active version to soft-deleted. The node is in the seed (a destination
394+
// version is in `all`), so this also guards that a seeded upsert is still
395+
// imported. Underpins the task propagating a source soft-delete onto a live
396+
// destination version.
395397
TEST(reconciler, propagates_soft_delete_over_active_destination) {
396398
reconcile_harness h;
397399
auto a = ppsr::context_subject::unqualified("a");
@@ -403,6 +405,7 @@ TEST(reconciler, propagates_soft_delete_over_active_destination) {
403405
seed.insert(key(a, 1));
404406
srs::work_set work;
405407
work.upserts.push_back(key(a, 1));
408+
work.soft_deleted.insert(key(a, 1));
406409

407410
auto stats = h.run(std::move(work), std::move(seed)).get();
408411
ASSERT_TRUE(stats.has_value());
@@ -415,6 +418,34 @@ TEST(reconciler, propagates_soft_delete_over_active_destination) {
415418
EXPECT_EQ(all[ia].deleted, ppsr::is_deleted::yes);
416419
}
417420

421+
// The caller's declared soft-delete wins over the per-version source body: a
422+
// standard (e.g. Confluent) source omits the `deleted` flag from that body by
423+
// default, so read_subject_version reports the version active, yet declaring it
424+
// in work_set.soft_deleted still lands it soft-deleted on the destination.
425+
// Guards the cross-vendor soft-delete propagation the HTTP source reader
426+
// relies on.
427+
TEST(reconciler, caller_soft_delete_wins_over_active_source_body) {
428+
reconcile_harness h;
429+
auto a = ppsr::context_subject::unqualified("a");
430+
// Source body reports the version ACTIVE, like a standard SR that does not
431+
// echo a deleted flag.
432+
h.source.add(a, 1, ppsr::is_deleted::no);
433+
434+
srs::work_set work;
435+
work.upserts.push_back(key(a, 1));
436+
work.soft_deleted.insert(key(a, 1));
437+
438+
auto stats = h.run(std::move(work)).get();
439+
ASSERT_TRUE(stats.has_value());
440+
EXPECT_EQ(stats->versions_changed, 1);
441+
EXPECT_EQ(stats->errors, 0);
442+
443+
const auto& all = h.destination.get_all();
444+
auto ia = index_of(all, "a");
445+
ASSERT_GE(ia, 0);
446+
EXPECT_EQ(all[ia].deleted, ppsr::is_deleted::yes);
447+
}
448+
418449
TEST(reconciler, cyclic_source_does_not_hang) {
419450
reconcile_harness h;
420451
auto a = ppsr::context_subject::unqualified("a");

tests/docker/Dockerfile

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,14 @@ RUN --mount=type=cache,id=dl-cache,target=/dl-cache /kafka-tools && \
100100

101101
#################################
102102

103+
FROM base AS confluent-schema-registry
104+
105+
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/confluent-schema-registry /
106+
RUN --mount=type=cache,id=dl-cache,target=/dl-cache /confluent-schema-registry && \
107+
rm /confluent-schema-registry
108+
109+
#################################
110+
103111
FROM java-base AS base-with-tools
104112

105113
COPY --chmod=0755 tests/docker/ducktape-deps/tool-pkgs /
@@ -444,6 +452,7 @@ COPY --from=java-verifiers /opt/redpanda-tests/java /opt/redpanda-tests/java
444452
COPY --from=java-verifiers /opt/verifiers /opt/verifiers
445453
COPY --from=java-verifiers /opt/kafka-serde /opt/kafka-serde
446454
COPY --from=kafka-tools /opt /opt
455+
COPY --from=confluent-schema-registry /opt/confluent /opt/confluent
447456
COPY --from=kcat /usr/local/bin/kcat /usr/local/bin/
448457
COPY --from=sarama-examples /opt/sarama /opt/sarama
449458
COPY --from=golang-test-clients /opt/redpanda-tests/go /opt/redpanda-tests/go
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
source "$(dirname "${BASH_SOURCE[0]}")/download-utils" # import download-utils
5+
6+
# Confluent Platform Community Edition includes Schema Registry, used as a
7+
# cross-vendor source in the shadow-link SR migration ducktape tests. Pin to a
8+
# version compatible with the Apache Kafka version used by the multi-cluster
9+
# shadow-link tests (kafka 3.7/3.8 -> CP 7.7.x).
10+
CP_VERSION=7.7.1
11+
CP_FILE=confluent-community-${CP_VERSION}.tar.gz
12+
13+
# Confluent's archive is the canonical source for the tarball.
14+
CP_URL=https://packages.confluent.io/archive/7.7/${CP_FILE}
15+
16+
mkdir -p /opt/confluent
17+
chmod a+rw /opt/confluent
18+
DL_STRIP=1 download_extract "${CP_URL}" /opt/confluent
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Copyright 2026 Redpanda Data, Inc.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the file licenses/BSL.md
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0
9+
10+
import os
11+
import tempfile
12+
13+
import requests
14+
from ducktape.services.service import Service
15+
from ducktape.utils.util import wait_until
16+
17+
CP_DIR = "/opt/confluent"
18+
SR_BIN = os.path.join(CP_DIR, "bin", "schema-registry-start")
19+
SR_LOG = "/var/log/schema-registry.log"
20+
SR_PROPS = "/tmp/schema-registry.properties"
21+
SR_DEFAULT_PORT = 8081
22+
23+
24+
class ConfluentSchemaRegistryService(Service):
25+
"""
26+
Single-node Confluent Schema Registry, used for migration / cross-vendor
27+
compatibility tests. The registry stores schemas in the `_schemas` topic
28+
of whichever Kafka-compatible cluster is passed as `bootstrap_provider`.
29+
30+
`bootstrap_provider` is anything that exposes a `bootstrap_servers()`
31+
method returning a comma-separated host:port list — e.g. a
32+
`KafkaServiceAdapter` wrapping `kafkatest.services.kafka.KafkaService`,
33+
or a `RedpandaService`.
34+
"""
35+
36+
logs = {
37+
"schema_registry_log": {
38+
"path": SR_LOG,
39+
"collect_default": True,
40+
},
41+
}
42+
43+
def __init__(self, context, bootstrap_provider, port: int = SR_DEFAULT_PORT):
44+
super().__init__(context, num_nodes=1)
45+
self._bootstrap_provider = bootstrap_provider
46+
self.port = port
47+
48+
def _bootstrap_servers(self) -> str:
49+
bs = self._bootstrap_provider.bootstrap_servers()
50+
# Confluent SR expects PLAINTEXT://host:port form for plaintext.
51+
return ",".join(
52+
f"PLAINTEXT://{ep}" if "://" not in ep else ep for ep in bs.split(",")
53+
)
54+
55+
def _properties(self) -> str:
56+
return (
57+
f"listeners=http://0.0.0.0:{self.port}\n"
58+
f"kafkastore.bootstrap.servers={self._bootstrap_servers()}\n"
59+
"kafkastore.topic=_schemas\n"
60+
"schema.compatibility.level=none\n"
61+
# This is a test-only registry; RF=1 keeps `_schemas` writable on a
62+
# small source cluster regardless of its broker count.
63+
"kafkastore.topic.replication.factor=1\n"
64+
)
65+
66+
def url(self, node=None) -> str:
67+
node = node or self.nodes[0]
68+
return f"http://{node.account.hostname}:{self.port}"
69+
70+
def alive(self, node) -> bool:
71+
try:
72+
r = requests.get(f"{self.url(node)}/subjects", timeout=2)
73+
return r.status_code == 200
74+
except Exception:
75+
return False
76+
77+
def pids(self, node):
78+
# The registry's JVM main class is
79+
# io.confluent.kafka.schemaregistry.rest.SchemaRegistryMain; match on
80+
# the (unambiguous) simple class name so stop_node actually finds it.
81+
return node.account.java_pids("SchemaRegistryMain")
82+
83+
def start_node(self, node, **kwargs):
84+
props = self._properties()
85+
self.logger.info(f"Starting Confluent SR on {node.account.hostname}\n{props}")
86+
with tempfile.NamedTemporaryFile(mode="w") as f:
87+
f.write(props)
88+
f.flush()
89+
node.account.copy_to(f.name, SR_PROPS)
90+
91+
cmd = f"nohup {SR_BIN} {SR_PROPS} > {SR_LOG} 2>&1 &"
92+
node.account.ssh(cmd, allow_fail=False)
93+
94+
wait_until(
95+
lambda: self.alive(node),
96+
timeout_sec=90,
97+
backoff_sec=2,
98+
err_msg=f"Confluent SR did not become reachable at {self.url(node)}",
99+
)
100+
self.logger.info(f"Confluent SR is up at {self.url(node)}")
101+
102+
def stop_node(self, node, **kwargs):
103+
for pid in self.pids(node):
104+
node.account.signal(pid, 15, allow_fail=True)
105+
wait_until(
106+
lambda: len(self.pids(node)) == 0,
107+
timeout_sec=30,
108+
backoff_sec=1,
109+
err_msg="Confluent SR did not stop",
110+
)
111+
112+
def clean_node(self, node, **kwargs):
113+
node.account.ssh(f"rm -f {SR_PROPS} {SR_LOG}", allow_fail=True)

tests/rptest/services/multi_cluster_services.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -165,12 +165,14 @@ def __str__(self):
165165

166166
class SecondaryClusterArgs:
167167
"""
168-
Container used to hold args and kwargs for the secondary cluster.
168+
Container used to hold args and kwargs for the secondary (source) cluster.
169169
170-
Will be passed to the secondary cluster's create method
170+
`num_brokers` sizes the source cluster; the remaining args/kwargs are
171+
passed to the source cluster's create method.
171172
"""
172173

173-
def __init__(self, *args, **kwargs):
174+
def __init__(self, num_brokers: int = 3, *args, **kwargs):
175+
self.num_brokers = num_brokers
174176
self.args = args
175177
self.kwargs = kwargs
176178

@@ -184,7 +186,6 @@ def __init__(
184186
secondary_spec: SecondaryClusterSpec = SecondaryClusterSpec(
185187
ServiceType.REDPANDA
186188
),
187-
num_brokers=3,
188189
secondary_args: SecondaryClusterArgs = SecondaryClusterArgs(),
189190
):
190191
self.test_ctx = test_ctx
@@ -194,7 +195,7 @@ def __init__(
194195
self._clusters.append(
195196
RedpandaCluster.create(
196197
self.test_ctx,
197-
num_brokers,
198+
secondary_args.num_brokers,
198199
*secondary_args.args,
199200
**secondary_args.kwargs,
200201
)
@@ -203,7 +204,7 @@ def __init__(
203204
self._clusters.append(
204205
KafkaCluster.create(
205206
self.test_ctx,
206-
num_brokers,
207+
secondary_args.num_brokers,
207208
secondary_spec.kafka_version
208209
if secondary_spec.kafka_version
209210
else KAFKA_VERSION,

tests/rptest/tests/cluster_linking_e2e_test.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,6 @@ def test_basic_ops(self):
157157
self.logger,
158158
self.redpanda,
159159
secondary_spec=SecondaryClusterSpec(ServiceType.REDPANDA),
160-
num_brokers=3,
161160
) as services:
162161
assert services.secondary.is_redpanda, (
163162
f"Expected Redpanda service, got {services.secondary}"
@@ -188,7 +187,6 @@ def test_basic_ops(self):
188187
secondary_spec=SecondaryClusterSpec(
189188
ServiceType.KAFKA, kafka_version="3.8.0", kafka_quorum="COMBINED_KRAFT"
190189
),
191-
num_brokers=3,
192190
) as services:
193191
assert services.secondary.is_kafka, (
194192
f"Expected Kafka service, got {services.secondary}"

0 commit comments

Comments
 (0)