Skip to content

Commit bad9af7

Browse files
tests: add replicated_partition mid-migration API-parity unit test
Exercises the replicated_partition API surface on a cloud-storage-backed partition mid tiered->cloud migration (cloud_topic_enabled + non-empty archival manifest + migration flag) and asserts it matches the same partition as plain tiered storage: offsets (start / high_watermark / last_stable / local_start / leader_epoch / offset_lag), size estimates, and timequery are unchanged when the storage mode flips to cloud, and prefix truncation works and advances the start. Complements the e2e API battery with per-method C++ coverage -- including the non-Kafka-reachable methods (estimate_size_between) -- and regresses the timequery fix (timequery must consult the imported extents, not just the local log, while migrating). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 95392bc commit bad9af7

1 file changed

Lines changed: 148 additions & 0 deletions

File tree

src/v/cloud_storage/tests/cloud_storage_e2e_test.cc

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,154 @@ TEST_F(ManualFixture, TestSizeEstimationWithCloud) {
198198
}
199199
}
200200

201+
// Exercises the replicated_partition API surface on a partition mid
202+
// tiered->cloud migration (cloud_topic_enabled + non-empty archival manifest +
203+
// migration flag) and asserts it behaves identically to the same partition as
204+
// plain tiered storage. Migration is transparent for a TS-served partition, so
205+
// offsets, size estimates, and timequery must not change when the storage mode
206+
// flips to cloud. Regression for the make_partition_proxy routing /
207+
// storage_mode-vs-manifest corner -- e.g. timequery once consulted only the
208+
// local log mid-migration and returned the local-log start instead of the true
209+
// earliest offset.
210+
TEST_F(ManualFixture, ReplicatedPartitionMidMigrationApiParity) {
211+
test_local_cfg.get("cloud_storage_disable_upload_loop_for_tests")
212+
.set_value(true);
213+
test_local_cfg.get("cloud_storage_spillover_manifest_max_segments")
214+
.set_value(std::make_optional<size_t>(5));
215+
test_local_cfg.get("cloud_storage_spillover_manifest_size")
216+
.set_value(std::optional<size_t>{});
217+
const model::topic topic_name("migrating-rp");
218+
model::ntp ntp(model::kafka_namespace, topic_name, 0);
219+
220+
cluster::topic_properties props;
221+
props.shadow_indexing = model::shadow_indexing_mode::full;
222+
props.cleanup_policy_bitflags = model::cleanup_policy_bitflags::deletion;
223+
add_topic({model::kafka_namespace, topic_name}, 1, props).get();
224+
wait_for_leader(ntp).get();
225+
226+
auto partition = app.partition_manager.local().get(ntp);
227+
auto& archiver = partition->archiver().value().get();
228+
archiver.initialize_probe();
229+
tests::remote_segment_generator gen(make_kafka_client().get(), *partition);
230+
auto deferred_g_close = ss::defer([&gen] { gen.stop().get(); });
231+
auto total_records = gen.num_segments(40)
232+
.batches_per_segment(5)
233+
.additional_local_segments(10)
234+
.produce()
235+
.get();
236+
ASSERT_GE(total_records, 200);
237+
ASSERT_TRUE(archiver.sync_for_tests().get());
238+
archiver.apply_spillover().get();
239+
240+
// Keep some segments local and the prefix only in tiered storage, so the
241+
// API spans both regions (cloud prefix + local suffix) and the parity check
242+
// is meaningful.
243+
auto log = partition->log();
244+
auto& manifest = partition->archival_meta_stm()->manifest();
245+
log->set_cloud_gc_offset(model::next_offset(manifest.get_last_offset()));
246+
// Wait for GC to trim the local prefix so it lives only in tiered storage
247+
// (the local log start advances above 0). Reads of the prefix must then come
248+
// from the cloud, which is what makes the migrating-parity check below
249+
// meaningful (and is the condition the timequery bug needed to manifest).
250+
RPTEST_REQUIRE_EVENTUALLY(
251+
20s, [&] { return log->offsets().start_offset > model::offset(0); });
252+
253+
struct snapshot {
254+
model::offset start;
255+
model::offset hwm;
256+
model::offset local_start;
257+
model::offset lso;
258+
kafka::leader_epoch epoch;
259+
model::offset offset_lag;
260+
size_t total_sz;
261+
size_t cloud_sz;
262+
std::optional<model::offset> tq_earliest;
263+
};
264+
auto capture = [](kafka::replicated_partition& rp) {
265+
auto lso = rp.last_stable_offset();
266+
EXPECT_FALSE(lso.has_error());
267+
auto last = kafka::prev_offset(model::offset_cast(lso.value()));
268+
auto local_start = model::offset_cast(rp.local_start_offset());
269+
// timequery for the epoch resolves to the earliest available offset; it
270+
// must consult the tiered/imported prefix, not just the local log.
271+
storage::timequery_config tq_cfg{
272+
rp.start_offset(),
273+
model::timestamp(0),
274+
rp.high_watermark(),
275+
{model::record_batch_type::raft_data},
276+
std::nullopt};
277+
auto tq = rp.timequery(tq_cfg).get();
278+
return snapshot{
279+
.start = rp.start_offset(),
280+
.hwm = rp.high_watermark(),
281+
.local_start = rp.local_start_offset(),
282+
.lso = lso.value(),
283+
.epoch = rp.leader_epoch(),
284+
.offset_lag = rp.offset_lag(),
285+
.total_sz = rp.estimate_size_between(kafka::offset(0), last),
286+
.cloud_sz = rp.estimate_size_between(
287+
kafka::offset(0), kafka::prev_offset(local_start)),
288+
.tq_earliest = tq.has_value()
289+
? std::optional<model::offset>(tq->offset)
290+
: std::nullopt,
291+
};
292+
};
293+
294+
kafka::replicated_partition rp_tiered(partition);
295+
auto tiered = capture(rp_tiered);
296+
// The prefix really is cloud-only (local log starts above 0), so timequery
297+
// for the epoch must reach into tiered storage to answer 0.
298+
ASSERT_GT(tiered.local_start, model::offset(0));
299+
ASSERT_EQ(
300+
tiered.tq_earliest, std::optional<model::offset>(model::offset(0)));
301+
302+
// Flip the partition into the migrating state: cloud storage mode
303+
// (cloud_topic_enabled) + the archival STM migration flag, manifest still
304+
// non-empty.
305+
ASSERT_TRUE(log->config().has_overrides());
306+
auto overrides = log->config().get_overrides();
307+
overrides.storage_mode = model::redpanda_storage_mode::cloud;
308+
log->set_overrides(overrides);
309+
ss::abort_source as;
310+
auto mec = partition->archival_meta_stm()
311+
->set_migration_state(true, ss::lowres_clock::now() + 30s, as)
312+
.get();
313+
ASSERT_FALSE(mec) << "set_migration_state failed: " << mec.message();
314+
ASSERT_TRUE(log->config().cloud_topic_enabled());
315+
ASSERT_TRUE(partition->archival_meta_stm()->is_migrating());
316+
317+
kafka::replicated_partition rp_mig(partition);
318+
auto migrating = capture(rp_mig);
319+
320+
// Transparent migration: every read/size API returns exactly what it did as
321+
// plain tiered storage.
322+
EXPECT_EQ(tiered.start, migrating.start);
323+
EXPECT_EQ(tiered.hwm, migrating.hwm);
324+
EXPECT_EQ(tiered.local_start, migrating.local_start);
325+
EXPECT_EQ(tiered.lso, migrating.lso);
326+
EXPECT_EQ(tiered.epoch, migrating.epoch);
327+
EXPECT_EQ(tiered.offset_lag, migrating.offset_lag);
328+
// total_sz spans the local region, where flipping to migrating appended a
329+
// set_migration_state control batch to the active segment; allow that small
330+
// delta. cloud_sz reads only the (unchanged) manifest, so stays exact.
331+
EXPECT_NEAR(tiered.total_sz, migrating.total_sz, 4096);
332+
EXPECT_EQ(tiered.cloud_sz, migrating.cloud_sz);
333+
// Regression: timequery still resolves to offset 0 via the imported extents
334+
// (not the local-log start) while migrating.
335+
EXPECT_EQ(tiered.tq_earliest, migrating.tq_earliest);
336+
EXPECT_EQ(
337+
migrating.tq_earliest, std::optional<model::offset>(model::offset(0)));
338+
339+
// DeleteRecords / prefix truncation works while migrating and advances the
340+
// start offset (eviction STM + archival manifest head-prune path).
341+
auto trunc = model::offset(total_records / 2);
342+
auto tec
343+
= rp_mig.prefix_truncate(trunc, ss::lowres_clock::now() + 30s).get();
344+
ASSERT_EQ(tec, kafka::error_code::none);
345+
RPTEST_REQUIRE_EVENTUALLY(
346+
10s, [&] { return rp_mig.start_offset() >= trunc; });
347+
}
348+
201349
class EndToEndFixture
202350
: public s3_imposter_fixture
203351
, public manual_metadata_upload_mixin

0 commit comments

Comments
 (0)