Skip to content

Commit 46e6290

Browse files
vishwasgarg18vishwasgarg18
committed
Add remove count and remove bytes and additional changes
Signed-off-by: Vishwas Garg <vishwasgarg14@gmail.com>
1 parent e5fe9ad commit 46e6290

11 files changed

Lines changed: 463 additions & 337 deletions

File tree

‎sandbox/plugins/block-cache-foyer/src/internalClusterTest/java/org/opensearch/blockcache/foyer/BlockCacheKeyIndexRecoveryIT.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,122 @@ public void testWrongVersionKeyIndexFileResultsInCleanStartup() throws Exception
423423
assertDocCount(indexName + "-copy", 50L);
424424
}
425425

426+
// ── Tiered-mode coverage ─────────────────────────────────────────────────
427+
428+
/**
429+
* Tiered mode (default): both the data tier and the metadata tier each persist
430+
* their own {@code key_index.json} on graceful warm-node restart.
431+
*
432+
* <p>With {@code block_cache.foyer.metadata_cache_ratio > 0} (5% by default) the
433+
* Foyer block cache splits into two independent instances under
434+
* {@code foyer-block-cache/data} and {@code foyer-block-cache/metadata}. Each
435+
* tier has its own Drop impl writing its own snapshot. This test guards against
436+
* a regression where one tier's persistence wiring is broken while the other
437+
* appears healthy — the previous helper would only see the data tier and miss
438+
* a metadata-tier failure entirely.
439+
*/
440+
public void testTieredBothTiersWriteKeyIndexOnShutdown() throws Exception {
441+
final Client client = client();
442+
final String indexName = "test-tiered-shutdown";
443+
444+
setupWarmNodeWithIndex(client, indexName);
445+
assertDocCount(indexName + "-copy", 50L);
446+
447+
logger.info("[tiered-shutdown-01] restarting warm node '{}'", getWarmNodeName());
448+
internalCluster().restartNode(getWarmNodeName());
449+
ensureGreen();
450+
451+
Path dataDir = findFoyerCacheDir();
452+
Path metaDir = findFoyerMetadataCacheDir();
453+
assertNotNull("data-tier cache directory must exist on warm node after restart", dataDir);
454+
assertNotNull(
455+
"metadata-tier cache directory must exist on warm node after restart "
456+
+ "(check block_cache.foyer.metadata_cache_ratio is > 0)",
457+
metaDir
458+
);
459+
460+
// Both tiers wrote a key_index.json.
461+
Path dataKeyIndex = dataDir.resolve(KEY_INDEX_FILENAME);
462+
Path metaKeyIndex = metaDir.resolve(KEY_INDEX_FILENAME);
463+
assertTrue("data-tier key_index.json must exist after restart", Files.exists(dataKeyIndex));
464+
assertTrue("metadata-tier key_index.json must exist after restart", Files.exists(metaKeyIndex));
465+
466+
// No leftover .tmp files on either tier.
467+
assertFalse(
468+
"data-tier .key_index.json.tmp must not exist after successful rename",
469+
Files.exists(dataDir.resolve(KEY_INDEX_TMP_FILENAME))
470+
);
471+
assertFalse(
472+
"metadata-tier .key_index.json.tmp must not exist after successful rename",
473+
Files.exists(metaDir.resolve(KEY_INDEX_TMP_FILENAME))
474+
);
475+
476+
// Both snapshots are valid JSON with version=1.
477+
String dataContent = Files.readString(dataKeyIndex);
478+
String metaContent = Files.readString(metaKeyIndex);
479+
assertFalse("data-tier key_index.json must not be empty", dataContent.isBlank());
480+
assertFalse("metadata-tier key_index.json must not be empty", metaContent.isBlank());
481+
assertTrue("data-tier key_index.json must contain version:1", dataContent.contains("\"version\":1"));
482+
assertTrue("metadata-tier key_index.json must contain version:1", metaContent.contains("\"version\":1"));
483+
484+
// Cluster healthy after restart.
485+
assertDocCount(indexName + "-copy", 50L);
486+
}
487+
488+
/**
489+
* Tiered mode: the periodic persist task fires independently on both tiers
490+
* and writes {@code key_index.json} within the configured interval.
491+
*
492+
* <p>Each tier owns its own persist task, so this test catches the case where
493+
* one tier's task is starved or never spawned — symptoms that would otherwise
494+
* only surface on shutdown.
495+
*/
496+
public void testTieredBothTiersPeriodicPersistFires() throws Exception {
497+
final Client client = client();
498+
final String indexName = "test-tiered-periodic";
499+
500+
setupWarmNodeWithIndex(client, indexName);
501+
assertDocCount(indexName + "-copy", 50L);
502+
503+
Path dataDir = findFoyerCacheDir();
504+
Path metaDir = findFoyerMetadataCacheDir();
505+
assertNotNull("data-tier cache directory must exist on warm node", dataDir);
506+
assertNotNull(
507+
"metadata-tier cache directory must exist on warm node " + "(check block_cache.foyer.metadata_cache_ratio is > 0)",
508+
metaDir
509+
);
510+
511+
logger.info("[tiered-periodic-01] waiting up to 30s for periodic persist (interval=5s) on data and metadata tiers");
512+
513+
// Persist interval is 5s (set in nodeSettings); allow up to 30s for both tiers.
514+
assertBusy(() -> {
515+
assertTrue(
516+
"data-tier key_index.json must be written by periodic persist task within interval",
517+
Files.exists(dataDir.resolve(KEY_INDEX_FILENAME))
518+
);
519+
assertTrue(
520+
"metadata-tier key_index.json must be written by periodic persist task within interval",
521+
Files.exists(metaDir.resolve(KEY_INDEX_FILENAME))
522+
);
523+
}, 30, TimeUnit.SECONDS);
524+
525+
// No leftover .tmp files after successful rename on either tier.
526+
assertFalse(
527+
"data-tier .key_index.json.tmp must not exist after successful periodic persist",
528+
Files.exists(dataDir.resolve(KEY_INDEX_TMP_FILENAME))
529+
);
530+
assertFalse(
531+
"metadata-tier .key_index.json.tmp must not exist after successful periodic persist",
532+
Files.exists(metaDir.resolve(KEY_INDEX_TMP_FILENAME))
533+
);
534+
535+
// Both tiers' content is valid JSON with version=1.
536+
String dataContent = Files.readString(dataDir.resolve(KEY_INDEX_FILENAME));
537+
String metaContent = Files.readString(metaDir.resolve(KEY_INDEX_FILENAME));
538+
assertTrue("data-tier key_index.json must contain version:1", dataContent.contains("\"version\":1"));
539+
assertTrue("metadata-tier key_index.json must contain version:1", metaContent.contains("\"version\":1"));
540+
}
541+
426542
// ── Helper methods ────────────────────────────────────────────────────────
427543

428544
/**
@@ -520,7 +636,44 @@ private long getWarmNodeUsedBytes(Client client) {
520636
* different node ordinals. Fails the test with a descriptive message if the directory is
521637
* not found — this ensures file-level assertions are never silently skipped.
522638
*/
639+
/**
640+
* Locates the Foyer block cache directory that holds {@code key_index.json}.
641+
*
642+
* <p>In tiered mode (default — {@code block_cache.foyer.metadata_cache_ratio > 0})
643+
* the layout is {@code foyer-block-cache/data/} and {@code foyer-block-cache/metadata/};
644+
* each subdir has its own {@code key_index.json}. This helper returns the data
645+
* subdir in that case. In single-cache mode the file lives directly under
646+
* {@code foyer-block-cache/}, which is what gets returned then.
647+
*
648+
* <p>Use {@link #findFoyerMetadataCacheDir()} for the metadata-tier subdir.
649+
*/
523650
private Path findFoyerCacheDir() {
651+
Path root = findFoyerCacheRoot();
652+
if (root == null) {
653+
return null;
654+
}
655+
Path dataSubdir = root.resolve("data");
656+
if (Files.isDirectory(dataSubdir)) {
657+
return dataSubdir; // tiered mode
658+
}
659+
return root; // single-cache mode
660+
}
661+
662+
/**
663+
* Returns the metadata-tier subdir ({@code foyer-block-cache/metadata}) when
664+
* the cache is in tiered mode, or {@code null} in single-cache mode.
665+
*/
666+
private Path findFoyerMetadataCacheDir() {
667+
Path root = findFoyerCacheRoot();
668+
if (root == null) {
669+
return null;
670+
}
671+
Path metaSubdir = root.resolve("metadata");
672+
return Files.isDirectory(metaSubdir) ? metaSubdir : null;
673+
}
674+
675+
/** Locates the {@code foyer-block-cache} parent directory on the warm node. */
676+
private Path findFoyerCacheRoot() {
524677
String warmName = getWarmNodeName();
525678
assertNotNull("A warm node must be present in the cluster", warmName);
526679

‎sandbox/plugins/block-cache-foyer/src/internalClusterTest/java/org/opensearch/blockcache/foyer/BlockCacheKeyIndexScaleIT.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,48 @@ public void testEvictPrefixAfterRecovery() throws Exception {
224224
assertTrue("Warm node must still be present and healthy after bulk evict_prefix", foundWarm);
225225
}
226226

227+
/**
228+
* Tiered mode: verify that BOTH the data tier and the metadata tier write
229+
* their own {@code key_index.json} on graceful warm-node shutdown.
230+
*
231+
* <p>With the default {@code block_cache.foyer.metadata_cache_ratio} (5%) the
232+
* Foyer block cache runs in tiered mode and creates two independent Foyer
233+
* instances rooted at {@code foyer-block-cache/data} and
234+
* {@code foyer-block-cache/metadata}. Each tier owns its own key_index.json,
235+
* its own Drop impl, and its own periodic-persist task. A regression in either
236+
* tier's persistence wiring would leak state across restarts.
237+
*/
238+
public void testKeyIndexJsonForBothTiersWrittenOnShutdown() throws Exception {
239+
setupScaleShards();
240+
241+
logger.info("[scale-tiered-01] restarting warm node '{}'", getWarmNodeName());
242+
internalCluster().restartNode(getWarmNodeName());
243+
ensureGreen(TimeValue.timeValueSeconds(120));
244+
245+
Path dataDir = findFoyerCacheDir();
246+
Path metaDir = findFoyerMetadataCacheDir();
247+
assertNotNull("data-tier cache directory must exist on warm node after restart", dataDir);
248+
assertNotNull(
249+
"metadata-tier cache directory must exist on warm node after restart "
250+
+ "(check block_cache.foyer.metadata_cache_ratio is > 0)",
251+
metaDir
252+
);
253+
254+
// Data tier
255+
Path dataKeyIndex = dataDir.resolve(KEY_INDEX_FILENAME);
256+
assertTrue("data-tier key_index.json must exist after restart", Files.exists(dataKeyIndex));
257+
String dataContent = Files.readString(dataKeyIndex);
258+
assertFalse("data-tier key_index.json must not be empty", dataContent.isBlank());
259+
assertTrue("data-tier key_index.json must contain version:1", dataContent.contains("\"version\":1"));
260+
261+
// Metadata tier
262+
Path metaKeyIndex = metaDir.resolve(KEY_INDEX_FILENAME);
263+
assertTrue("metadata-tier key_index.json must exist after restart", Files.exists(metaKeyIndex));
264+
String metaContent = Files.readString(metaKeyIndex);
265+
assertFalse("metadata-tier key_index.json must not be empty", metaContent.isBlank());
266+
assertTrue("metadata-tier key_index.json must contain version:1", metaContent.contains("\"version\":1"));
267+
}
268+
227269
/**
228270
* Verifies that the warm node remains healthy after restoring all indices,
229271
* cluster is GREEN, and block_cache stats are accessible.
@@ -359,7 +401,44 @@ private long getWarmNodeUsedBytes() {
359401
return 0L;
360402
}
361403

404+
/**
405+
* Locates the Foyer block cache directory that holds {@code key_index.json}.
406+
*
407+
* <p>In tiered mode (default — {@code block_cache.foyer.metadata_cache_ratio > 0})
408+
* the layout is {@code foyer-block-cache/data/} and {@code foyer-block-cache/metadata/};
409+
* each subdir has its own {@code key_index.json}. This helper returns the data
410+
* subdir in that case. In single-cache mode the file lives directly under
411+
* {@code foyer-block-cache/}, which is what gets returned then.
412+
*
413+
* <p>Use {@link #findFoyerMetadataCacheDir()} for the metadata-tier subdir.
414+
*/
362415
private Path findFoyerCacheDir() {
416+
Path root = findFoyerCacheRoot();
417+
if (root == null) {
418+
return null;
419+
}
420+
Path dataSubdir = root.resolve("data");
421+
if (Files.isDirectory(dataSubdir)) {
422+
return dataSubdir; // tiered mode
423+
}
424+
return root; // single-cache mode
425+
}
426+
427+
/**
428+
* Returns the metadata-tier subdir ({@code foyer-block-cache/metadata}) when
429+
* the cache is in tiered mode, or {@code null} in single-cache mode.
430+
*/
431+
private Path findFoyerMetadataCacheDir() {
432+
Path root = findFoyerCacheRoot();
433+
if (root == null) {
434+
return null;
435+
}
436+
Path metaSubdir = root.resolve("metadata");
437+
return Files.isDirectory(metaSubdir) ? metaSubdir : null;
438+
}
439+
440+
/** Locates the {@code foyer-block-cache} parent directory on the warm node. */
441+
private Path findFoyerCacheRoot() {
363442
String warmName = getWarmNodeName();
364443
assertNotNull("A warm node must be present in the cluster", warmName);
365444
Environment environment = internalCluster().getInstance(Environment.class, warmName);

0 commit comments

Comments
 (0)