Skip to content

Commit 6742ae8

Browse files
vishwasgarg18Vishwas GargBukhtawar
authored
Feature/warm cache spi and stats and wiring foyer with tiered object store (opensearch-project#21530)
* Unified Cache Service * Add block cache SPI and Foyer plugin for warm nodes Introduces a plugin-extensible block cache layer on warm nodes that caches variable-size byte ranges (Parquet column chunks, remote segment reads) on local SSD alongside the existing FileCache. Core changes: - BlockCacheProvider SPI: plugins declare their cache and report how much SSD budget they need. Node.java partitions the warm-node SSD budget between FileCache and all registered block-cache plugins before any createComponents() call, so capacities are consistent at startup. - BlockCacheStats (11-field record): universal stats contract across the plugin boundary, including hit/miss byte counts and eviction bytes. - NodeCacheOrchestrator: owns SSD budget validation, FileCache lifecycle, and block cache registration. Replaces UnifiedCacheService. - _nodes/stats aggregate_file_cache now folds block cache counters (hits, misses, eviction bytes, used bytes, total capacity) into the existing warm-node stats sections via mergeStats(). - WarmFsService virtual capacity formula updated: total virtual bytes now equals (fileCacheRatio × fileCacheCapacity) + Σ(plugin virtual bytes). Foyer plugin (sandbox): - BlockCacheFoyerPlugin: implements BlockCacheProvider, owns all four Foyer settings (block_cache.size, block_cache.block_size, block_cache.io_engine, block_cache.data_to_cache_ratio). - FoyerAggregatedStats: parses two-section native FFM buffer into BlockCacheStats records without wire serialization. - FoyerStatsCounter: atomic counters updated on every hot-path operation. Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * Simplify and optimise warm stats Signed-off-by: Bukhtawar Khan <bukhtawa@amazon.com> * Simplify and optimise warm stats Signed-off-by: Bukhtawar Khan <bukhtawa@amazon.com> * _nodes/stats to respond with aggregated stats Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * Add tests Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * rename settings Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * spotless fix Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * spotless fix Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * Integerate foyer with tiered object store Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * address PR comments, add removed, removed, handle tiered_object_store on remove cache clear Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * address PR comments, add removed, removed, handle tiered_object_store on remove cache clear Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * Fix tiered directory tests Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * Fix DF test Signed-off-by: Vishwas Garg <glvishwa@amazon.com> * Additional tests and fixes Signed-off-by: Vishwas Garg <glvishwa@amazon.com> --------- Signed-off-by: Vishwas Garg <glvishwa@amazon.com> Signed-off-by: Bukhtawar Khan <bukhtawa@amazon.com> Co-authored-by: Vishwas Garg <glvishwa@amazon.com> Co-authored-by: Bukhtawar Khan <bukhtawa@amazon.com>
1 parent a8c9190 commit 6742ae8

55 files changed

Lines changed: 4078 additions & 530 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

sandbox/libs/tiered-storage/src/main/rust/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ thiserror = { workspace = true }
1818
async-trait = { workspace = true }
1919
bytes = { workspace = true }
2020
native-bridge-common = { workspace = true }
21+
opensearch-block-cache = { path = "../../../../../plugins/block-cache-foyer/src/main/rust" }
2122

2223
[dev-dependencies]
2324
tempfile = { workspace = true }

sandbox/libs/tiered-storage/src/main/rust/src/ffm.rs

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ use std::sync::Arc;
1818
use native_bridge_common::ffm_safe;
1919
use object_store::ObjectStore;
2020

21+
use opensearch_block_cache::traits::BlockCache;
22+
2123
use crate::registry::TieredStorageRegistry;
2224
use crate::registry::FileRegistry;
2325
use crate::tiered_object_store::TieredObjectStore;
@@ -72,11 +74,15 @@ unsafe fn arc_from_ptr(ptr: i64) -> Result<Arc<TieredObjectStore>, String> {
7274
/// - `remote_store_box_ptr`: if non-zero, a `Box<Arc<dyn ObjectStore>>` pointer from a repository
7375
/// plugin. The Arc is cloned (ownership is NOT taken — the pointer remains valid for other
7476
/// shards). If 0, no remote store.
77+
/// - `cache_box_ptr`: if non-zero, a `Box<Arc<dyn BlockCache>>` pointer from a block cache
78+
/// plugin. The Arc is cloned (ownership is NOT taken — the pointer remains valid for the
79+
/// node lifetime). If 0, no block cache is attached.
7580
#[ffm_safe]
7681
#[no_mangle]
7782
pub extern "C" fn ts_create_tiered_object_store(
7883
local_store_box_ptr: i64,
7984
remote_store_box_ptr: i64,
85+
cache_box_ptr: i64,
8086
) -> i64 {
8187
let file_registry = Arc::new(TieredStorageRegistry::new());
8288

@@ -86,7 +92,7 @@ pub extern "C" fn ts_create_tiered_object_store(
8692
Arc::new(object_store::local::LocalFileSystem::new())
8793
};
8894

89-
let store = Arc::new(TieredObjectStore::new(file_registry, local));
95+
let mut store = TieredObjectStore::new(file_registry, local);
9096

9197
if remote_store_box_ptr != NULL_PTR {
9298
// IMPORTANT: Do NOT consume the Box — the pointer is node-level and shared
@@ -96,8 +102,17 @@ pub extern "C" fn ts_create_tiered_object_store(
96102
store.set_remote(remote_arc);
97103
}
98104

99-
let ptr = Arc::into_raw(store) as i64;
100-
native_bridge_common::log_info!("ffm: ts_create_tiered_object_store ptr={}", ptr);
105+
if cache_box_ptr != NULL_PTR {
106+
let cache_box = unsafe { &*(cache_box_ptr as *const Arc<dyn BlockCache>) };
107+
store = store.with_cache(Arc::clone(cache_box));
108+
native_bridge_common::log_info!("ffm: ts_create_tiered_object_store: block cache wired");
109+
}
110+
111+
let ptr = Arc::into_raw(Arc::new(store)) as i64;
112+
native_bridge_common::log_info!(
113+
"ffm: ts_create_tiered_object_store cache_wired={}",
114+
cache_box_ptr != NULL_PTR
115+
);
101116
Ok(ptr)
102117
}
103118

@@ -117,7 +132,7 @@ pub extern "C" fn ts_get_object_store_box_ptr(tiered_store_ptr: i64) -> i64 {
117132
// Coerce to trait object and box it
118133
let boxed: Box<Arc<dyn ObjectStore>> = Box::new(arc as Arc<dyn ObjectStore>);
119134
let ptr = Box::into_raw(boxed) as i64;
120-
native_bridge_common::log_info!("ffm: ts_get_object_store_box_ptr input={}, output={}", tiered_store_ptr, ptr);
135+
native_bridge_common::log_info!("ffm: ts_get_object_store_box_ptr: ok");
121136
Ok(ptr)
122137
}
123138

@@ -130,7 +145,7 @@ pub extern "C" fn ts_destroy_object_store_box_ptr(ptr: i64) -> i64 {
130145
return Err("ts_destroy_object_store_box_ptr: null pointer (0)".to_string());
131146
}
132147
let _boxed = unsafe { Box::from_raw(ptr as *mut Arc<dyn ObjectStore>) };
133-
native_bridge_common::log_info!("ffm: ts_destroy_object_store_box_ptr ptr={}", ptr);
148+
native_bridge_common::log_info!("ffm: ts_destroy_object_store_box_ptr: ok");
134149
Ok(0)
135150
}
136151

@@ -144,7 +159,7 @@ pub extern "C" fn ts_destroy_tiered_object_store(ptr: i64) -> i64 {
144159
return Err("ts_destroy_tiered_object_store: null pointer (0)".to_string());
145160
}
146161
let _store = unsafe { Arc::from_raw(ptr as *const TieredObjectStore) };
147-
native_bridge_common::log_info!("ffm: ts_destroy_tiered_object_store ptr={}", ptr);
162+
native_bridge_common::log_info!("ffm: ts_destroy_tiered_object_store: ok");
148163
Ok(0)
149164
}
150165

@@ -223,6 +238,7 @@ pub extern "C" fn ts_remove_file(
223238
let path = path.strip_prefix('/').unwrap_or(path);
224239

225240
store.registry().remove(path, false);
241+
store.evict_path(path);
226242

227243
native_bridge_common::log_debug!("ffm: ts_remove_file path='{}'", path);
228244
Ok(0)

sandbox/libs/tiered-storage/src/main/rust/src/ffm_tests.rs

Lines changed: 66 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ fn test_destroy_null_returns_error() {
77

88
#[test]
99
fn test_create_and_destroy_no_leak() {
10-
let store_ptr = ts_create_tiered_object_store(0, 0);
10+
let store_ptr = ts_create_tiered_object_store(0, 0, 0);
1111
assert!(store_ptr > 0);
1212
assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0);
1313
}
@@ -27,7 +27,7 @@ fn test_remove_file_null_store_returns_error() {
2727

2828
#[test]
2929
fn test_register_files_and_remove_round_trip() {
30-
let store_ptr = ts_create_tiered_object_store(0, 0);
30+
let store_ptr = ts_create_tiered_object_store(0, 0, 0);
3131
assert!(store_ptr > 0);
3232

3333
// Batch register: two files as Remote (triplets: path\nremotePath\nsize\n...)
@@ -44,7 +44,7 @@ fn test_register_files_and_remove_round_trip() {
4444

4545
#[test]
4646
fn test_register_files_invalid_location_returns_error() {
47-
let store_ptr = ts_create_tiered_object_store(0, 0);
47+
let store_ptr = ts_create_tiered_object_store(0, 0, 0);
4848
assert!(store_ptr > 0);
4949

5050
let entries = b"test.parquet\nremote/test.parquet\n2048";
@@ -66,7 +66,7 @@ fn test_destroy_object_store_box_ptr_null_returns_error() {
6666

6767
#[test]
6868
fn test_get_and_destroy_object_store_box_ptr_round_trip() {
69-
let store_ptr = ts_create_tiered_object_store(0, 0);
69+
let store_ptr = ts_create_tiered_object_store(0, 0, 0);
7070
assert!(store_ptr > 0);
7171

7272
// Get a boxed pointer — this increments the Arc refcount
@@ -83,7 +83,7 @@ fn test_get_and_destroy_object_store_box_ptr_round_trip() {
8383

8484
#[test]
8585
fn test_get_object_store_box_ptr_multiple_calls() {
86-
let store_ptr = ts_create_tiered_object_store(0, 0);
86+
let store_ptr = ts_create_tiered_object_store(0, 0, 0);
8787
assert!(store_ptr > 0);
8888

8989
// Multiple box pointers can coexist (simulates multiple reader managers)
@@ -109,8 +109,8 @@ fn test_create_with_remote_does_not_consume_pointer() {
109109
let remote_ptr = Box::into_raw(remote_box) as i64;
110110

111111
// Create two TieredObjectStores sharing the same remote pointer
112-
let store1 = ts_create_tiered_object_store(0, remote_ptr);
113-
let store2 = ts_create_tiered_object_store(0, remote_ptr);
112+
let store1 = ts_create_tiered_object_store(0, remote_ptr, 0);
113+
let store2 = ts_create_tiered_object_store(0, remote_ptr, 0);
114114
assert!(store1 > 0);
115115
assert!(store2 > 0);
116116

@@ -121,3 +121,62 @@ fn test_create_with_remote_does_not_consume_pointer() {
121121
// Clean up the remote Box (simulates repository.doClose())
122122
let _remote_box = unsafe { Box::from_raw(remote_ptr as *mut Arc<dyn ObjectStore>) };
123123
}
124+
125+
// -- cache_box_ptr tests ------------------------------------------------
126+
127+
/// ts_create_tiered_object_store with 0 cache_box_ptr creates a store without
128+
/// a cache. The store must still be functional for all registry and I/O operations.
129+
#[test]
130+
fn test_create_with_zero_cache_ptr_creates_uncached_store() {
131+
let store_ptr = ts_create_tiered_object_store(0, 0, 0);
132+
assert!(store_ptr > 0);
133+
134+
// Store is functional — register a file without error.
135+
let entries = b"a.parquet\nremote/a.parquet\n512";
136+
let result = ts_register_files(store_ptr, entries.as_ptr(), entries.len() as i64, 1, 1);
137+
assert_eq!(result, 0);
138+
139+
assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0);
140+
}
141+
142+
/// ts_create_tiered_object_store with a valid cache_box_ptr wires the cache
143+
/// without consuming the Box — two stores can share the same cache pointer.
144+
#[test]
145+
fn test_create_with_cache_does_not_consume_pointer() {
146+
use opensearch_block_cache::range_cache::CacheKey;
147+
use opensearch_block_cache::traits::BlockCache;
148+
use bytes::Bytes;
149+
150+
// Minimal no-op cache used only to construct a valid Box<Arc<dyn BlockCache>> pointer.
151+
struct NoopCache;
152+
impl BlockCache for NoopCache {
153+
fn as_any(&self) -> &dyn std::any::Any { self }
154+
fn get<'a>(&'a self, _key: &'a CacheKey)
155+
-> std::pin::Pin<Box<dyn std::future::Future<Output = Option<Bytes>> + Send + 'a>>
156+
{
157+
Box::pin(std::future::ready(None))
158+
}
159+
fn put(&self, _key: &CacheKey, _data: Bytes) {}
160+
fn evict_prefix(&self, _prefix: &str) {}
161+
fn clear(&self) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
162+
Box::pin(std::future::ready(()))
163+
}
164+
}
165+
166+
let cache: Arc<dyn BlockCache> = Arc::new(NoopCache);
167+
let cache_box = Box::new(Arc::clone(&cache));
168+
let cache_ptr = Box::into_raw(cache_box) as i64;
169+
170+
// Create two stores sharing the same cache pointer.
171+
let store1 = ts_create_tiered_object_store(0, 0, cache_ptr);
172+
let store2 = ts_create_tiered_object_store(0, 0, cache_ptr);
173+
assert!(store1 > 0);
174+
assert!(store2 > 0);
175+
176+
// Both stores created — cache pointer not consumed (Arc strong count still valid).
177+
assert_eq!(ts_destroy_tiered_object_store(store1), 0);
178+
assert_eq!(ts_destroy_tiered_object_store(store2), 0);
179+
180+
// Clean up the cache Box (simulates FoyerBlockCache.close()).
181+
let _cache_box = unsafe { Box::from_raw(cache_ptr as *mut Arc<dyn BlockCache>) };
182+
}

0 commit comments

Comments
 (0)