diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 84c9d525e269..8f01c5591bf3 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1820,6 +1820,7 @@ class vk_perf_logger { return; } print_count = 0; + last_report_json = build_report_json(); GGML_LOG_DEBUG("================\nVulkan Profiling Results:\n================\n\n"); print_legacy_timings(); print_triplet_timings(); @@ -1828,6 +1829,19 @@ class vk_perf_logger { triplet_timings.clear(); } + // Copies the most recently completed graph's compact timing report. The + // report is retained after print_timings() clears the live accumulators so + // callers can retrieve it through the Vulkan registry proc-address API. + size_t copy_last_report_json(char * dst, size_t dst_size) const { + const size_t size = last_report_json.size(); + if (dst != nullptr && dst_size > 0) { + const size_t n = std::min(size, dst_size - 1); + memcpy(dst, last_report_json.data(), n); + dst[n] = '\0'; + } + return size; + } + std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) { *n_flops = 0; std::string fusion_str; @@ -1949,8 +1963,57 @@ class vk_perf_logger { std::map> timings; std::map> flops; std::map triplet_timings; + std::string last_report_json; uint32_t print_count {}; + static std::string json_escape(const std::string & value) { + std::string escaped; + escaped.reserve(value.size()); + for (const char c : value) { + switch (c) { + case '"': escaped += "\\\""; break; + case '\\': escaped += "\\\\"; break; + case '\b': escaped += "\\b"; break; + case '\f': escaped += "\\f"; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: + if (static_cast(c) >= 0x20) { + escaped += c; + } + break; + } + } + return escaped; + } + + std::string build_report_json() const { + std::stringstream ss; + ss << "{\"ops\":["; + bool first = true; + for (const auto & entry : timings) { + uint64_t total_ns = 0; + for (const uint64_t time : entry.second) { + total_ns += time; + } + const size_t count = entry.second.size(); + if (!first) { + ss << ","; + } + first = false; + ss << "{\"name\":\"" << json_escape(entry.first) + << "\",\"count\":" << count + << ",\"total_us\":" << std::fixed << std::setprecision(3) + << (static_cast(total_ns) / 1000.0) + << ",\"avg_us\":" + << (count == 0 ? 0.0 : static_cast(total_ns) / count / 1000.0) + << "}"; + } + ss << "]}"; + return ss.str(); + } + void log_triplet_timing(const ggml_tensor * node, uint64_t time, const std::string& operation_type = "generic_matmul") { if (!node || !node->src[0] || !node->src[1] || time == 0) return; @@ -18076,11 +18139,31 @@ static bool ggml_backend_vk_supports_efficient_fa(ggml_backend_t backend) { return ctx->device->coopmat2 || ctx->device->coopmat1_fa_support; } +// Vulkan-only extension retrieved through ggml_backend_reg_get_proc_address(). +// Returns the JSON size excluding the terminating NUL. Passing nullptr/0 queries +// the required size. Returns 0 when profiling is disabled or no graph has completed. +static size_t ggml_backend_vk_get_perf_report_json( + ggml_backend_t backend, + char * buffer, + size_t buffer_size) { + if (backend == nullptr || backend->context == nullptr) { + return 0; + } + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *) backend->context; + if (!ctx->perf_logger) { + return 0; + } + return ctx->perf_logger->copy_last_report_json(buffer, buffer_size); +} + static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { GGML_UNUSED(reg); if (strcmp(name, "ggml_backend_supports_efficient_fa") == 0) { return (void *)ggml_backend_vk_supports_efficient_fa; } + if (strcmp(name, "ggml_backend_vk_get_perf_report_json") == 0) { + return (void *)ggml_backend_vk_get_perf_report_json; + } return NULL; } diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index db97f95ffe05..b666bc1edb7e 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -179,6 +179,9 @@ struct clip_ctx { bool debug_output_embeddings = false; clip_image_tile_mode tile_mode = CLIP_IMAGE_TILE_MODE_SEQUENTIAL; + // Most recent Vulkan graph profile, populated after a successful image encode + // when the backend exposes ggml_backend_vk_get_perf_report_json. + std::string last_backend_profile_json; // When the GPU backend lacks bf16 support but the GGUF has bf16 weights, // we declare the in-context tensors as f16 and convert on disk-load. @@ -3431,6 +3434,38 @@ bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f3 return clip_image_batch_encode(ctx, n_threads, &imgs, vec); } +static void clip_capture_backend_profile(clip_ctx * ctx) { + ctx->last_backend_profile_json.clear(); + if (ctx->backend == nullptr || ctx->backend == ctx->backend_cpu) { + return; + } + + ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg == nullptr) { + return; + } + + // Vulkan exposes this as an optional registry extension. Keeping the lookup + // dynamic makes mtmd work unchanged for CPU, Metal, OpenCL, CUDA, and any + // backend which does not implement the profiler snapshot API. + using get_perf_report_t = size_t (*)(ggml_backend_t, char *, size_t); + auto get_perf_report = reinterpret_cast( + ggml_backend_reg_get_proc_address(reg, "ggml_backend_vk_get_perf_report_json")); + if (get_perf_report == nullptr) { + return; + } + + const size_t size = get_perf_report(ctx->backend, nullptr, 0); + if (size == 0) { + return; + } + std::vector report(size + 1, '\0'); + if (get_perf_report(ctx->backend, report.data(), report.size()) == size) { + ctx->last_backend_profile_json.assign(report.data(), size); + } +} + bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs_c_ptr, float * vec) { const clip_image_f32_batch & imgs = *imgs_c_ptr; int batch_size = imgs.entries.size(); @@ -4091,6 +4126,11 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima return false; } + // Snapshot the completed vision graph before a subsequent graph overwrites + // the backend profiler state. No-op on non-Vulkan backends or when profiling + // is not compiled/enabled. + clip_capture_backend_profile(ctx); + // the last node is the embedding tensor ggml_tensor * embeddings = ggml_graph_node(gf, -1); @@ -4286,6 +4326,13 @@ std::map clip_get_mem_usage(const struct clip_ctx * return result; } +const char * clip_get_backend_profile_json(const struct clip_ctx * ctx) { + if (ctx == nullptr || ctx->last_backend_profile_json.empty()) { + return nullptr; + } + return ctx->last_backend_profile_json.c_str(); +} + bool clip_has_whisper_encoder(const struct clip_ctx * ctx) { switch (ctx->proj_type()) { case PROJECTOR_TYPE_ULTRAVOX: diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 65c5a95b6f47..926a240a1147 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -142,4 +142,9 @@ bool clip_has_whisper_encoder(const struct clip_ctx * ctx); // summed with the scheduler's compute reservations. Used by mtmd_get_memory_usage // (and ultimately by common/fit.cpp's heuristic). Restored from upstream b9341. std::map clip_get_mem_usage(const struct clip_ctx * ctx); + +// JSON report for the most recently completed vision graph when the active +// backend exposes structured profiling. The returned pointer is owned by ctx +// and remains valid until the next image encode or clip_free(). +const char * clip_get_backend_profile_json(const struct clip_ctx * ctx); #endif diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 11b760ab8218..1a3c77209f67 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -134,6 +134,7 @@ struct mtmd_context { struct clip_ctx * ctx_a; // audio const struct llama_model * text_model; std::vector image_embd_v; // image embedding vector + std::string last_vision_profile_json; bool print_timings; int n_threads; @@ -1136,6 +1137,13 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) ctx->image_embd_v.data()); } + if (ok) { + const char * profile = clip_get_backend_profile_json(ctx_clip); + ctx->last_vision_profile_json = profile ? profile : ""; + } else { + ctx->last_vision_profile_json.clear(); + } + return ok ? 0 : 1; } @@ -1143,6 +1151,13 @@ float * mtmd_get_output_embd(mtmd_context * ctx) { return ctx->image_embd_v.data(); } +const char * mtmd_get_vision_profile_json(const mtmd_context * ctx) { + if (ctx == nullptr || ctx->last_vision_profile_json.empty()) { + return nullptr; + } + return ctx->last_vision_profile_json.c_str(); +} + // qvac: const-qualifiers updated to match the declarations in mtmd.h. The // previous non-const signatures caused link failures because the public API // shape (declared via extern "C" in the header) didn't match what we emitted. diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index cd6b50c130e2..52cb257c33fa 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -264,6 +264,11 @@ MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx, // llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float) MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx); +// JSON profile for the most recently completed vision encode. Returns nullptr +// when the active backend does not expose structured profiling. The returned +// pointer is owned by ctx and remains valid until the next vision encode. +MTMD_API const char * mtmd_get_vision_profile_json(const mtmd_context * ctx); + // Set callback for all future logging events. // If this is not called, or NULL is supplied, everything is output on stderr. MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);