Skip to content

Commit f90365a

Browse files
kmbandyclaude
kmbandy
and
claude
committed
fix: resolve build errors blocking llama-server compilation
- ops.cpp: remove redundant extern in extern-C decl (turbo-quant merge issue) - src/CMakeLists.txt: add llama-kv-cache-tiered.cpp to llama lib + HIP includes - tools/server/CMakeLists.txt: add /opt/rocm/include + __HIP_PLATFORM_AMD__ - server-tiered-cache.h: add default ctor; fix global_stats/stats_ name collision - server-tiered-cache.cpp: update stats_ references after rename - server-context.cpp: fix unique_ptr copy assignment; fix tiered_cache scope in metrics lambda; fix n_discard ordering; fix SLT_INF variadic macro; fix json::object push_back for ordered_json; add friend server_routes Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent c34bd25 commit f90365a

6 files changed

Lines changed: 49 additions & 52 deletions

File tree

‎ggml/src/ggml-cpu/ops.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#include <cfloat>
1313
#include <cmath>
1414

15-
extern "C" GGML_API int turbo3_cpu_wht_group_size;
15+
extern "C" int turbo3_cpu_wht_group_size;
1616

1717
// ggml_compute_forward_dup
1818

‎src/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ add_library(llama
2424
llama-io.cpp
2525
llama-kv-cache.cpp
2626
llama-kv-cache-iswa.cpp
27+
llama-kv-cache-tiered.cpp
2728
llama-memory.cpp
2829
llama-memory-hybrid.cpp
2930
llama-memory-hybrid-iswa.cpp
@@ -50,6 +51,10 @@ set_target_properties(llama PROPERTIES
5051

5152
target_include_directories(llama PRIVATE .)
5253
target_include_directories(llama PUBLIC ../include)
54+
if(GGML_HIP)
55+
target_include_directories(llama PRIVATE /opt/rocm/include)
56+
target_compile_definitions(llama PRIVATE __HIP_PLATFORM_AMD__)
57+
endif()
5358
target_compile_features (llama PRIVATE cxx_std_17) # don't bump
5459

5560
target_link_libraries(llama PUBLIC ggml)

‎tools/server/CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ endif()
2727

2828
target_include_directories(${TARGET} PRIVATE ../mtmd)
2929
target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR})
30+
target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src)
31+
if(GGML_HIP)
32+
target_include_directories(${TARGET} PRIVATE /opt/rocm/include)
33+
target_compile_definitions(${TARGET} PRIVATE __HIP_PLATFORM_AMD__)
34+
endif()
3035
target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT})
3136

3237

@@ -72,6 +77,11 @@ install(TARGETS ${TARGET} RUNTIME)
7277

7378
target_include_directories(${TARGET} PRIVATE ../mtmd)
7479
target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR})
80+
target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src)
81+
if(GGML_HIP)
82+
target_include_directories(${TARGET} PRIVATE /opt/rocm/include)
83+
endif()
84+
target_compile_definitions(${TARGET} PRIVATE __HIP_PLATFORM_AMD__)
7585
target_link_libraries(${TARGET} PRIVATE server-context PUBLIC llama-common cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
7686

7787
target_compile_features(${TARGET} PRIVATE cxx_std_17)

‎tools/server/server-context.cpp‎

Lines changed: 26 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,7 @@ struct server_metrics {
634634

635635
struct server_context_impl {
636636
friend struct server_context;
637+
friend struct server_routes;
637638

638639
public:
639640
// only use these pointers outside of this class:
@@ -683,7 +684,7 @@ struct server_context_impl {
683684
int n_empty_consecutive = 0;
684685

685686
std::unique_ptr<server_prompt_cache> prompt_cache;
686-
server_tiered_cache tiered_cache;
687+
std::unique_ptr<server_tiered_cache> tiered_cache;
687688

688689
server_metrics metrics;
689690

@@ -858,7 +859,7 @@ struct server_context_impl {
858859

859860
// Initialize tiered cache if enabled
860861
if (params_base.kv_tiered_enabled) {
861-
tiered_cache = server_tiered_cache(params_base);
862+
tiered_cache = std::make_unique<server_tiered_cache>(params_base);
862863
SRV_INF("tiered cache initialized, hot=%f%%, warm=%f%%, cold=%f%%\n",
863864
params_base.kv_tier_hot_pct, params_base.kv_tier_warm_pct, params_base.kv_tier_cold_pct);
864865
}
@@ -916,10 +917,10 @@ struct server_context_impl {
916917

917918
// initialize tier manager for slot if enabled
918919
if (params_base.kv_tiered_enabled) {
919-
if (!tiered_cache.init_slot(i, *model)) {
920+
if (!tiered_cache->init_slot(i, *model)) {
920921
SRV_WRN("failed to initialize tier manager for slot %d\n", i);
921922
} else {
922-
SLT_INF(slot, "tier manager initialized for slot\n");
923+
SLT_INF(slot, "tier manager initialized for slot %s\n", "");
923924
}
924925
}
925926

@@ -2184,20 +2185,6 @@ struct server_context_impl {
21842185
continue;
21852186
}
21862187

2187-
// Try tiered cache eviction before context shift
2188-
if (params_base.kv_tiered_enabled) {
2189-
auto* slot_tier = tiered_cache.get_slot_manager(slot.id);
2190-
if (slot_tier && slot_tier->initialized) {
2191-
// Evict tokens to tiered cache before context shift
2192-
int n_evict = std::min(n_discard, int(slot_tier->tiered_cache->get_config().cold_capacity()));
2193-
if (n_evict > 0) {
2194-
if (tiered_cache.evict_from_slot(slot.id, n_evict)) {
2195-
SLT_INF(slot, "tiered cache eviction: %d tokens\n", n_evict);
2196-
}
2197-
}
2198-
}
2199-
}
2200-
22012188
// Shift context
22022189
int n_keep = slot.task->params.n_keep < 0 ? slot.task->n_tokens() : slot.task->params.n_keep;
22032190

@@ -2210,6 +2197,20 @@ struct server_context_impl {
22102197
const int n_left = slot.prompt.n_tokens() - n_keep;
22112198
const int n_discard = slot.task->params.n_discard ? slot.task->params.n_discard : (n_left / 2);
22122199

2200+
// Try tiered cache eviction before context shift
2201+
if (params_base.kv_tiered_enabled) {
2202+
auto* slot_tier = tiered_cache->get_slot_manager(slot.id);
2203+
if (slot_tier && slot_tier->initialized) {
2204+
// Evict tokens to tiered cache before context shift
2205+
int n_evict = std::min(n_discard, int(slot_tier->tiered_cache->get_config().cold_capacity()));
2206+
if (n_evict > 0) {
2207+
if (tiered_cache->evict_from_slot(slot.id, n_evict)) {
2208+
SLT_INF(slot, "tiered cache eviction: %d tokens\n", n_evict);
2209+
}
2210+
}
2211+
}
2212+
}
2213+
22132214
SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard);
22142215

22152216
llama_memory_seq_rm (llama_get_memory(ctx), slot.id, n_keep , n_keep + n_discard);
@@ -3532,33 +3533,13 @@ void server_routes::init_routes() {
35323533
};
35333534

35343535
// Add tiered cache metrics if enabled
3535-
if (params.kv_tiered_enabled) {
3536-
auto global_stats = tiered_cache.get_global_stats();
3537-
all_metrics_def["counter"].push({
3538-
{"name", "tier_evictions_total"},
3539-
{"help", "Total number of tier evictions"},
3540-
{"value", global_stats.total_evictions}
3541-
});
3542-
all_metrics_def["counter"].push({
3543-
{"name", "tier_migrations_total"},
3544-
{"help", "Total number of tier migrations"},
3545-
{"value", global_stats.total_migrations}
3546-
});
3547-
all_metrics_def["counter"].push({
3548-
{"name", "tier_cache_hits_total"},
3549-
{"help", "Total cache hits"},
3550-
{"value", global_stats.total_cache_hits}
3551-
});
3552-
all_metrics_def["counter"].push({
3553-
{"name", "tier_cache_misses_total"},
3554-
{"help", "Total cache misses"},
3555-
{"value", global_stats.total_cache_misses}
3556-
});
3557-
all_metrics_def["gauge"].push({
3558-
{"name", "tier_migration_latency_seconds"},
3559-
{"help", "Total migration latency in seconds"},
3560-
{"value", global_stats.total_migration_latency_us / 1e6}
3561-
});
3536+
if (params.kv_tiered_enabled && ctx_server.tiered_cache) {
3537+
auto global_stats = ctx_server.tiered_cache->get_global_stats();
3538+
{ json m; m["name"] = "tier_evictions_total"; m["help"] = "Total number of tier evictions"; m["value"] = global_stats.total_evictions; all_metrics_def["counter"].push_back(m); }
3539+
{ json m; m["name"] = "tier_migrations_total"; m["help"] = "Total number of tier migrations"; m["value"] = global_stats.total_migrations; all_metrics_def["counter"].push_back(m); }
3540+
{ json m; m["name"] = "tier_cache_hits_total"; m["help"] = "Total cache hits"; m["value"] = global_stats.total_cache_hits; all_metrics_def["counter"].push_back(m); }
3541+
{ json m; m["name"] = "tier_cache_misses_total"; m["help"] = "Total cache misses"; m["value"] = global_stats.total_cache_misses; all_metrics_def["counter"].push_back(m); }
3542+
{ json m; m["name"] = "tier_migration_latency_seconds"; m["help"] = "Total migration latency in seconds"; m["value"] = global_stats.total_migration_latency_us / 1e6; all_metrics_def["gauge"].push_back(m); }
35623543
}
35633544

35643545
std::stringstream prometheus;

‎tools/server/server-tiered-cache.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ server_tiered_cache::server_tiered_cache(const common_params& params)
2828
// Extract attention threshold
2929
attention_threshold = params.kv_tier_attention_threshold;
3030

31-
global_stats.reset();
31+
stats_.reset();
3232
}
3333
}
3434

@@ -120,7 +120,7 @@ bool server_tiered_cache::evict_from_slot(int slot_id, uint32_t n_tokens) {
120120
if (result) {
121121
// Update global stats
122122
std::lock_guard<std::mutex> lock(mutex);
123-
global_stats.total_evictions += n_tokens;
123+
stats_.total_evictions += n_tokens;
124124
}
125125

126126
return result;
@@ -145,7 +145,7 @@ bool server_tiered_cache::migrate_in_slot(int slot_id,
145145
if (result) {
146146
// Update global stats
147147
std::lock_guard<std::mutex> lock(mutex);
148-
global_stats.total_migrations += positions.size();
148+
stats_.total_migrations += positions.size();
149149
}
150150

151151
return result;
@@ -166,10 +166,10 @@ llama_tier_stats server_tiered_cache::get_slot_stats(int slot_id) {
166166

167167
server_tiered_cache::global_stats server_tiered_cache::get_global_stats() {
168168
std::lock_guard<std::mutex> lock(mutex);
169-
return global_stats;
169+
return stats_;
170170
}
171171

172172
void server_tiered_cache::reset_stats() {
173173
std::lock_guard<std::mutex> lock(mutex);
174-
global_stats.reset();
174+
stats_.reset();
175175
}

‎tools/server/server-tiered-cache.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ struct server_tiered_cache {
2222
}
2323
};
2424

25+
server_tiered_cache() : enabled(false) {}
2526
server_tiered_cache(const common_params& params);
2627
~server_tiered_cache();
2728

@@ -69,7 +70,7 @@ struct server_tiered_cache {
6970
private:
7071
bool enabled = false;
7172
std::unordered_map<int, slot_tier_manager> slot_managers;
72-
global_stats global_stats;
73+
global_stats stats_;
7374
mutable std::mutex mutex;
7475

7576
common_params params;

0 commit comments

Comments
 (0)