Skip to content

Commit 09c3416

Browse files
adamdebreceniadamdebreceni
committed
MINIFICPP-2719 - Fix template use
1 parent 9dc1f90 commit 09c3416

5 files changed

Lines changed: 37 additions & 29 deletions

File tree

‎cmake/LlamaCpp.cmake‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ set(PC ${Bash_EXECUTABLE} -c "set -x &&\
3838

3939

4040
FetchContent_Declare(llamacpp
41-
URL https://github.com/ggml-org/llama.cpp/archive/refs/tags/b7836.tar.gz
42-
URL_HASH SHA256=3d384e7e8b3bc3cd31abddedf684a6e201405c1d932cafb3c4a5277d872b0614
41+
URL https://github.com/ggml-org/llama.cpp/archive/refs/tags/b8944.tar.gz
42+
URL_HASH SHA256=ca231c8aca086f56bad3ed371f6dc5b01e971e812a8ddf67564f087390c0e781
4343
PATCH_COMMAND "${PC}"
4444
SYSTEM
4545
)
@@ -50,5 +50,7 @@ set(LLAMACPP_INCLUDE_DIRS
5050
"${llamacpp_SOURCE_DIR}/include"
5151
"${llamacpp_SOURCE_DIR}/ggml/include"
5252
"${llamacpp_SOURCE_DIR}/tools"
53+
"${llamacpp_SOURCE_DIR}/common"
54+
"${llamacpp_SOURCE_DIR}/vendor"
5355
CACHE STRING "" FORCE
5456
)

‎extensions/llamacpp/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ add_minifi_library(minifi-llamacpp SHARED ${SOURCES})
3131
target_include_directories(minifi-llamacpp PUBLIC "${CMAKE_SOURCE_DIR}/extensions/llamacpp")
3232
target_include_directories(minifi-llamacpp PUBLIC "${LLAMACPP_INCLUDE_DIRS}")
3333

34-
target_link_libraries(minifi-llamacpp minifi-cpp-extension-lib llama mtmd)
34+
target_link_libraries(minifi-llamacpp minifi-cpp-extension-lib llama mtmd llama-common)
3535

3636
register_c_api_extension(minifi-llamacpp "LLAMACPP EXTENSION" LLAMACPP-EXTENSION "Provides llama.cpp support" "extensions/llamacpp/tests")
3737

‎extensions/llamacpp/processors/DefaultLlamaContext.cpp‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ DefaultLlamaContext::DefaultLlamaContext(const std::filesystem::path& model_path
5252
throw Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, fmt::format("Failed to load model from '{}'", model_path.string()));
5353
}
5454

55+
chat_template_ = common_chat_templates_init(llama_model_, "");
56+
5557
llama_context_params ctx_params = llama_context_default_params();
5658
ctx_params.n_ctx = llama_ctx_params.n_ctx;
5759
ctx_params.n_batch = llama_ctx_params.n_batch;
@@ -108,27 +110,16 @@ DefaultLlamaContext::~DefaultLlamaContext() {
108110
}
109111

110112
std::optional<std::string> DefaultLlamaContext::applyTemplate(const std::vector<LlamaChatMessage>& messages) {
111-
std::vector<llama_chat_message> llama_messages;
112-
llama_messages.reserve(messages.size());
113-
std::transform(messages.begin(), messages.end(), std::back_inserter(llama_messages),
114-
[](const LlamaChatMessage& msg) { return llama_chat_message{.role = msg.role.c_str(), .content = msg.content.c_str()}; });
115-
std::string text;
116-
text.resize(DEFAULT_BUFFER_SIZE);
117-
const char * chat_template = llama_model_chat_template(llama_model_, nullptr);
118-
int32_t res_size = llama_chat_apply_template(chat_template, llama_messages.data(), llama_messages.size(), true, text.data(), gsl::narrow<int32_t>(text.size()));
119-
if (res_size < 0) {
113+
if (!chat_template_) {
120114
return std::nullopt;
121115
}
122-
if (res_size > gsl::narrow<int32_t>(text.size())) {
123-
text.resize(res_size);
124-
res_size = llama_chat_apply_template(chat_template, llama_messages.data(), llama_messages.size(), true, text.data(), gsl::narrow<int32_t>(text.size()));
125-
if (res_size < 0) {
126-
return std::nullopt;
127-
}
116+
common_chat_templates_inputs inputs;
117+
for (auto& msg : messages) {
118+
inputs.messages.push_back(common_chat_msg{.role = msg.role, .content = msg.content});
128119
}
129-
text.resize(res_size);
120+
inputs.enable_thinking = false; // TODO(adebreceni): MINIFICPP-2800 common_chat_templates_support_enable_thinking(chat_template_.get());
130121

131-
return text;
122+
return common_chat_templates_apply(chat_template_.get(), inputs).prompt;
132123
}
133124

134125
namespace {

‎extensions/llamacpp/processors/DefaultLlamaContext.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "LlamaContext.h"
2020
#include "llama.h"
2121
#include "LlamaBackendInitializer.h"
22+
#include "chat.h"
2223
#include "mtmd/mtmd.h"
2324
#include "minifi-cpp/core/logging/Logger.h"
2425

@@ -40,6 +41,7 @@ class DefaultLlamaContext : public LlamaContext {
4041
private:
4142
const LlamaBackendInitializer& llama_context_initializer_ = LlamaBackendInitializer::get();
4243
llama_model* llama_model_{};
44+
common_chat_templates_ptr chat_template_;
4345
llama_context* llama_ctx_{};
4446
mtmd_context* multimodal_ctx_{};
4547
llama_sampler* llama_sampler_{};

‎thirdparty/llamacpp/mtmd-fix.patch‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,27 @@
1-
diff --color=auto -rupN llamacpp-src-original/CMakeLists.txt llamacpp-src-patched/CMakeLists.txt
2-
--- llamacpp-src-original/CMakeLists.txt 2026-01-25 21:19:47
3-
+++ llamacpp-src-patched/CMakeLists.txt 2026-02-18 13:15:46
4-
@@ -212,6 +212,7 @@ add_subdirectory(src)
1+
diff --color=auto -rupN llama.cpp-b8944/CMakeLists.txt llama.cpp-b8944-patched/CMakeLists.txt
2+
--- llama.cpp-b8944/CMakeLists.txt 2026-04-27 08:30:55
3+
+++ llama.cpp-b8944-patched/CMakeLists.txt 2026-04-27 13:49:25
4+
@@ -191,6 +191,7 @@ add_subdirectory(src)
55
#
66

77
add_subdirectory(src)
88
+add_subdirectory(tools/mtmd)
99

1010
#
1111
# utils, programs, examples and tests
12-
diff --color=auto -rupN llamacpp-src-original/tools/mtmd/CMakeLists.txt llamacpp-src-patched/tools/mtmd/CMakeLists.txt
13-
--- llamacpp-src-original/tools/mtmd/CMakeLists.txt 2026-01-25 21:19:47
14-
+++ llamacpp-src-patched/tools/mtmd/CMakeLists.txt 2026-02-18 13:13:40
15-
@@ -80,16 +80,3 @@ endif()
12+
diff --color=auto -rupN llama.cpp-b8944/common/ngram-mod.cpp llama.cpp-b8944-patched/common/ngram-mod.cpp
13+
--- llama.cpp-b8944/common/ngram-mod.cpp 2026-04-27 08:30:55
14+
+++ llama.cpp-b8944-patched/common/ngram-mod.cpp 2026-04-30 08:28:08
15+
@@ -1,4 +1,5 @@
16+
#include "ngram-mod.h"
17+
+#include <algorithm>
18+
19+
//
20+
// common_ngram_mod
21+
diff --color=auto -rupN llama.cpp-b8944/tools/mtmd/CMakeLists.txt llama.cpp-b8944-patched/tools/mtmd/CMakeLists.txt
22+
--- llama.cpp-b8944/tools/mtmd/CMakeLists.txt 2026-04-27 08:30:55
23+
+++ llama.cpp-b8944-patched/tools/mtmd/CMakeLists.txt 2026-04-27 13:50:45
24+
@@ -101,20 +101,6 @@ endif()
1625
endif()
1726
endif()
1827

@@ -27,5 +36,9 @@ diff --color=auto -rupN llamacpp-src-original/tools/mtmd/CMakeLists.txt llamacpp
2736
-if(LLAMA_TOOLS_INSTALL)
2837
- install(TARGETS ${TARGET} RUNTIME)
2938
-endif()
30-
-target_link_libraries (${TARGET} PRIVATE common mtmd Threads::Threads)
39+
-target_link_libraries (${TARGET} PRIVATE llama-common mtmd Threads::Threads)
3140
-target_compile_features(${TARGET} PRIVATE cxx_std_17)
41+
-
42+
# mtmd-debug tool
43+
add_executable(llama-mtmd-debug debug/mtmd-debug.cpp)
44+
set_target_properties(llama-mtmd-debug PROPERTIES OUTPUT_NAME llama-mtmd-debug)

0 commit comments

Comments
 (0)