test(paged): cover MoE swiglu down chain

Assisted-by: Claude:opus-4.8 [Claude Code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

tests/test-backend-ops.cpp

@@ -4447,6 +4447,91 @@ struct test_mul_mat_id_fusion : public test_case {
     }
 };
 
+// Merged MoE gate_up -> SWIGLU -> down MUL_MAT_ID chain.
+struct test_moe_swiglu_down : public test_case {
+    const ggml_type type_a;
+    const int n_mats;
+    const int n_used;
+    const int64_t n_ff;
+    const int64_t n_tokens;
+    const int64_t n_embd;
+
+    std::string vars() override {
+        return VARS_TO_STR6(type_a, n_mats, n_used, n_ff, n_tokens, n_embd);
+    }
+
+    double max_nmse_err() override {
+        return 5e-4;
+    }
+
+    double max_nmse_err(ggml_backend_t backend) override {
+        if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) {
+            // This whole-graph gate compounds two native-FP4 MUL_MAT_ID ops with
+            // SWIGLU between them, so it needs slightly more room than the
+            // single-op FP4 MUL_MAT_ID gate.
+            return 2.5e-2;
+        }
+        return max_nmse_err();
+    }
+
+    uint64_t op_flops(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return 2 * n_ff * n_embd * n_tokens * n_used * 3;
+    }
+
+    test_moe_swiglu_down(ggml_type type_a = GGML_TYPE_F32, int n_mats = 128, int n_used = 8,
+            int64_t n_ff = 768, int64_t n_tokens = 128, int64_t n_embd = 2048)
+        : type_a(type_a), n_mats(n_mats), n_used(n_used), n_ff(n_ff), n_tokens(n_tokens), n_embd(n_embd) {
+            GGML_ASSERT(n_used <= n_mats);
+        }
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        ggml_tensor * gate_up = ggml_new_tensor_3d(ctx, type_a, n_embd, 2 * n_ff, n_mats);
+        ggml_set_name(gate_up, "gate_up");
+
+        ggml_tensor * down = ggml_new_tensor_3d(ctx, type_a, n_ff, n_embd, n_mats);
+        ggml_set_name(down, "down");
+
+        ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n_tokens);
+        ggml_set_name(ids, "ids");
+        if (n_used != n_mats) {
+            ids = ggml_view_2d(ctx, ids, n_used, n_tokens, ids->nb[1], 0);
+            ggml_set_name(ids, "view_of_ids");
+        }
+
+        ggml_tensor * cur = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, n_used, n_tokens);
+        ggml_set_name(cur, "cur");
+
+        ggml_tensor * gate_up_out = ggml_mul_mat_id(ctx, gate_up, cur, ids);
+        ggml_set_name(gate_up_out, "gate_up_out");
+
+        ggml_tensor * gate = ggml_view_3d(ctx, gate_up_out, n_ff, n_used, n_tokens, gate_up_out->nb[1], gate_up_out->nb[2], 0);
+        ggml_set_name(gate, "gate");
+
+        ggml_tensor * up = ggml_view_3d(ctx, gate_up_out, n_ff, n_used, n_tokens, gate_up_out->nb[1], gate_up_out->nb[2], n_ff * gate_up_out->nb[0]);
+        ggml_set_name(up, "up");
+
+        ggml_tensor * act = ggml_swiglu_split(ctx, gate, up);
+        ggml_set_name(act, "swiglu");
+
+        ggml_tensor * out = ggml_mul_mat_id(ctx, down, act, ids);
+        ggml_set_name(out, "out");
+
+        return out;
+    }
+
+    void initialize_tensors(ggml_context * ctx) override {
+        init_mul_mat_id_tensors(ctx, n_mats);
+    }
+
+    bool run_whole_graph() override { return true; }
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "MOE_SWIGLU_DOWN";
+    }
+};
+
 // GGML_OP_OUT_PROD
 struct test_out_prod : public test_case {
     const ggml_type type_a;
@@ -8759,6 +8844,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }
 
+    // [paged Phase 7] Merged MoE gate_up -> SWIGLU -> down projection gate for the
+    // serving candidate that fuses SWIGLU into NVFP4 down-input quantization.
+    test_cases.emplace_back(new test_moe_swiglu_down(GGML_TYPE_F32, 8, 2, 32, 8, 64));
+    for (int n : {16, 33, 64, 128, 130, 200}) {
+        test_cases.emplace_back(new test_moe_swiglu_down(GGML_TYPE_NVFP4, 128, 8, 768, n, 2048));
+    }
+
     // [paged P0 / track B] NVFP4/MXFP4 dense decode-shape mmq_y-down bit-exact gate.
     // The dense FP4 weight GEMM is the track-B target; P1 lowers mmq_y (the weight-row tile) on the
     // NVFP4 decode path to raise resident-CTA occupancy. mmq_y is a pure N-row tiling knob, so a