QVAC-21901 perf(mtmd/gemma4v): Mali Vulkan encoder profiling and measured sweep

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -1820,6 +1820,7 @@ class vk_perf_logger {
             return;
         }
         print_count = 0;
+        last_report_json = build_report_json();
         GGML_LOG_DEBUG("================\nVulkan Profiling Results:\n================\n\n");
         print_legacy_timings();
         print_triplet_timings();
@@ -1828,6 +1829,19 @@ class vk_perf_logger {
         triplet_timings.clear();
     }
 
+    // Copies the most recently completed graph's compact timing report. The
+    // report is retained after print_timings() clears the live accumulators so
+    // callers can retrieve it through the Vulkan registry proc-address API.
+    size_t copy_last_report_json(char * dst, size_t dst_size) const {
+        const size_t size = last_report_json.size();
+        if (dst != nullptr && dst_size > 0) {
+            const size_t n = std::min(size, dst_size - 1);
+            memcpy(dst, last_report_json.data(), n);
+            dst[n] = '\0';
+        }
+        return size;
+    }
+
     std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) {
         *n_flops = 0;
         std::string fusion_str;
@@ -1949,8 +1963,57 @@ class vk_perf_logger {
     std::map<std::string, std::vector<uint64_t>> timings;
     std::map<std::string, std::vector<uint64_t>> flops;
     std::map<std::string, triplet_timing_info> triplet_timings;
+    std::string last_report_json;
     uint32_t print_count {};
 
+    static std::string json_escape(const std::string & value) {
+        std::string escaped;
+        escaped.reserve(value.size());
+        for (const char c : value) {
+            switch (c) {
+                case '"':  escaped += "\\\""; break;
+                case '\\': escaped += "\\\\"; break;
+                case '\b': escaped += "\\b";  break;
+                case '\f': escaped += "\\f";  break;
+                case '\n': escaped += "\\n";  break;
+                case '\r': escaped += "\\r";  break;
+                case '\t': escaped += "\\t";  break;
+                default:
+                    if (static_cast<unsigned char>(c) >= 0x20) {
+                        escaped += c;
+                    }
+                    break;
+            }
+        }
+        return escaped;
+    }
+
+    std::string build_report_json() const {
+        std::stringstream ss;
+        ss << "{\"ops\":[";
+        bool first = true;
+        for (const auto & entry : timings) {
+            uint64_t total_ns = 0;
+            for (const uint64_t time : entry.second) {
+                total_ns += time;
+            }
+            const size_t count = entry.second.size();
+            if (!first) {
+                ss << ",";
+            }
+            first = false;
+            ss << "{\"name\":\"" << json_escape(entry.first)
+               << "\",\"count\":" << count
+               << ",\"total_us\":" << std::fixed << std::setprecision(3)
+               << (static_cast<double>(total_ns) / 1000.0)
+               << ",\"avg_us\":"
+               << (count == 0 ? 0.0 : static_cast<double>(total_ns) / count / 1000.0)
+               << "}";
+        }
+        ss << "]}";
+        return ss.str();
+    }
+
     void log_triplet_timing(const ggml_tensor * node, uint64_t time, const std::string& operation_type = "generic_matmul") {
         if (!node || !node->src[0] || !node->src[1] || time == 0) return;
 
@@ -18076,11 +18139,31 @@ static bool ggml_backend_vk_supports_efficient_fa(ggml_backend_t backend) {
     return ctx->device->coopmat2 || ctx->device->coopmat1_fa_support;
 }
 
+// Vulkan-only extension retrieved through ggml_backend_reg_get_proc_address().
+// Returns the JSON size excluding the terminating NUL. Passing nullptr/0 queries
+// the required size. Returns 0 when profiling is disabled or no graph has completed.
+static size_t ggml_backend_vk_get_perf_report_json(
+        ggml_backend_t backend,
+        char * buffer,
+        size_t buffer_size) {
+    if (backend == nullptr || backend->context == nullptr) {
+        return 0;
+    }
+    ggml_backend_vk_context * ctx = (ggml_backend_vk_context *) backend->context;
+    if (!ctx->perf_logger) {
+        return 0;
+    }
+    return ctx->perf_logger->copy_last_report_json(buffer, buffer_size);
+}
+
 static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {
     GGML_UNUSED(reg);
     if (strcmp(name, "ggml_backend_supports_efficient_fa") == 0) {
         return (void *)ggml_backend_vk_supports_efficient_fa;
     }
+    if (strcmp(name, "ggml_backend_vk_get_perf_report_json") == 0) {
+        return (void *)ggml_backend_vk_get_perf_report_json;
+    }
     return NULL;
 }
 

tools/mtmd/clip.cpp

@@ -179,6 +179,9 @@ struct clip_ctx {
 
     bool debug_output_embeddings = false;
     clip_image_tile_mode tile_mode = CLIP_IMAGE_TILE_MODE_SEQUENTIAL;
+    // Most recent Vulkan graph profile, populated after a successful image encode
+    // when the backend exposes ggml_backend_vk_get_perf_report_json.
+    std::string last_backend_profile_json;
 
     // When the GPU backend lacks bf16 support but the GGUF has bf16 weights,
     // we declare the in-context tensors as f16 and convert on disk-load.
@@ -3431,6 +3434,38 @@ bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f3
     return clip_image_batch_encode(ctx, n_threads, &imgs, vec);
 }
 
+static void clip_capture_backend_profile(clip_ctx * ctx) {
+    ctx->last_backend_profile_json.clear();
+    if (ctx->backend == nullptr || ctx->backend == ctx->backend_cpu) {
+        return;
+    }
+
+    ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend);
+    ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
+    if (reg == nullptr) {
+        return;
+    }
+
+    // Vulkan exposes this as an optional registry extension. Keeping the lookup
+    // dynamic makes mtmd work unchanged for CPU, Metal, OpenCL, CUDA, and any
+    // backend which does not implement the profiler snapshot API.
+    using get_perf_report_t = size_t (*)(ggml_backend_t, char *, size_t);
+    auto get_perf_report = reinterpret_cast<get_perf_report_t>(
+        ggml_backend_reg_get_proc_address(reg, "ggml_backend_vk_get_perf_report_json"));
+    if (get_perf_report == nullptr) {
+        return;
+    }
+
+    const size_t size = get_perf_report(ctx->backend, nullptr, 0);
+    if (size == 0) {
+        return;
+    }
+    std::vector<char> report(size + 1, '\0');
+    if (get_perf_report(ctx->backend, report.data(), report.size()) == size) {
+        ctx->last_backend_profile_json.assign(report.data(), size);
+    }
+}
+
 bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs_c_ptr, float * vec) {
     const clip_image_f32_batch & imgs = *imgs_c_ptr;
     int batch_size = imgs.entries.size();
@@ -4091,6 +4126,11 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
         return false;
     }
 
+    // Snapshot the completed vision graph before a subsequent graph overwrites
+    // the backend profiler state. No-op on non-Vulkan backends or when profiling
+    // is not compiled/enabled.
+    clip_capture_backend_profile(ctx);
+
     // the last node is the embedding tensor
     ggml_tensor * embeddings = ggml_graph_node(gf, -1);
 
@@ -4286,6 +4326,13 @@ std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx *
     return result;
 }
 
+const char * clip_get_backend_profile_json(const struct clip_ctx * ctx) {
+    if (ctx == nullptr || ctx->last_backend_profile_json.empty()) {
+        return nullptr;
+    }
+    return ctx->last_backend_profile_json.c_str();
+}
+
 bool clip_has_whisper_encoder(const struct clip_ctx * ctx) {
     switch (ctx->proj_type()) {
         case PROJECTOR_TYPE_ULTRAVOX:

tools/mtmd/clip.h

@@ -142,4 +142,9 @@ bool clip_has_whisper_encoder(const struct clip_ctx * ctx);
 // summed with the scheduler's compute reservations. Used by mtmd_get_memory_usage
 // (and ultimately by common/fit.cpp's heuristic). Restored from upstream b9341.
 std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
+
+// JSON report for the most recently completed vision graph when the active
+// backend exposes structured profiling. The returned pointer is owned by ctx
+// and remains valid until the next image encode or clip_free().
+const char * clip_get_backend_profile_json(const struct clip_ctx * ctx);
 #endif

tools/mtmd/mtmd.cpp

@@ -134,6 +134,7 @@ struct mtmd_context {
     struct clip_ctx * ctx_a; // audio
     const struct llama_model * text_model;
     std::vector<float> image_embd_v; // image embedding vector
+    std::string last_vision_profile_json;
 
     bool print_timings;
     int n_threads;
@@ -1136,13 +1137,27 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens)
             ctx->image_embd_v.data());
     }
 
+    if (ok) {
+        const char * profile = clip_get_backend_profile_json(ctx_clip);
+        ctx->last_vision_profile_json = profile ? profile : "";
+    } else {
+        ctx->last_vision_profile_json.clear();
+    }
+
     return ok ? 0 : 1;
 }
 
 float * mtmd_get_output_embd(mtmd_context * ctx) {
     return ctx->image_embd_v.data();
 }
 
+const char * mtmd_get_vision_profile_json(const mtmd_context * ctx) {
+    if (ctx == nullptr || ctx->last_vision_profile_json.empty()) {
+        return nullptr;
+    }
+    return ctx->last_vision_profile_json.c_str();
+}
+
 // qvac: const-qualifiers updated to match the declarations in mtmd.h. The
 // previous non-const signatures caused link failures because the public API
 // shape (declared via extern "C" in the header) didn't match what we emitted.

tools/mtmd/mtmd.h

@@ -264,6 +264,11 @@ MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx,
 // llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float)
 MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx);
 
+// JSON profile for the most recently completed vision encode. Returns nullptr
+// when the active backend does not expose structured profiling. The returned
+// pointer is owned by ctx and remains valid until the next vision encode.
+MTMD_API const char * mtmd_get_vision_profile_json(const mtmd_context * ctx);
+
 // Set callback for all future logging events.
 // If this is not called, or NULL is supplied, everything is output on stderr.
 MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);