[NPUW] Model builder: add LoRA adapter support

src/plugins/intel_npu/tests/functional/behavior/npuw/test_engine/models/model_builder.cpp

@@ -6,6 +6,7 @@
 
 #include <algorithm>
 #include <limits>
+#include <memory>
 #include <unordered_map>
 
 #include "model_builder_internal.hpp"
@@ -35,6 +36,67 @@ void annotate_constants_with_weightless_cache(const std::shared_ptr<ov::Model>&
         offset += c->get_byte_size();
     }
 }
+
+// Create one LoRA tensor as either a Parameter (stateless) or a
+// ReadValue/Assign state pair (stateful) per LoRAInjector::stateful.
+ov::Output<ov::Node> make_lora_value(const LoRAInjector& lora,
+                                     ov::element::Type type,
+                                     const ov::Shape& shape,
+                                     const std::string& name) {
+    if (!lora.stateful) {
+        auto param = std::make_shared<ov::op::v0::Parameter>(type, shape);
+        param->set_friendly_name(name);
+        param->output(0).set_names({name});
+        return param->output(0);
+    }
+
+    OPENVINO_ASSERT(lora.sinks, "Stateful LoRA requires a sink collection");
+    auto variable = std::make_shared<ov::op::util::Variable>(ov::op::util::VariableInfo{shape, type, name});
+    auto read_value = std::make_shared<ov::op::v6::ReadValue>(variable);
+    read_value->set_friendly_name(name);
+    auto assign = std::make_shared<ov::op::v6::Assign>(read_value, variable);
+    assign->set_friendly_name(name + ".assign");
+    lora.sinks->push_back(assign);
+    return read_value->output(0);
+}
+
+// LoRA branch: input -> MatMul(A^T) -> Multiply(alpha) -> MatMul(B^T) -> Add(base).
+// State names match the NPUW lora_state_* convention.
+ov::Output<ov::Node> inject_lora(const ov::Output<ov::Node>& input,
+                                 const ov::Output<ov::Node>& base,
+                                 size_t in_features,
+                                 size_t out_features,
+                                 const std::string& name,
+                                 const LoRAInjector& lora) {
+    const size_t rank = lora.max_rank;
+    const std::string state_prefix = "lora_state_" + name;
+
+    auto lora_a = make_lora_value(lora, lora.precision, ov::Shape{rank, in_features}, state_prefix + ".MatMul.A");
+    auto lora_b = make_lora_value(lora, lora.precision, ov::Shape{out_features, rank}, state_prefix + ".MatMul.B");
+    auto lora_alpha = make_lora_value(lora, ov::element::f32, ov::Shape{1, rank}, state_prefix + ".MatMul.alpha");
+
+    // input @ A^T -> [batch, seq, rank]
+    auto mm_a = std::make_shared<ov::opset11::MatMul>(input, lora_a, false, true);
+    mm_a->set_friendly_name(name + "_lora_a");
+
+    // Multiply by alpha [1, rank], broadcast over batch/seq; alpha is always f32 (set from host)
+    ov::Output<ov::Node> alpha = lora_alpha;
+    if (alpha.get_element_type() != mm_a->get_output_element_type(0)) {
+        auto alpha_convert = std::make_shared<ov::opset11::Convert>(alpha, mm_a->get_output_element_type(0));
+        alpha_convert->set_friendly_name(name + "_lora_alpha_cvt");
+        alpha = alpha_convert->output(0);
+    }
+    auto scaled = std::make_shared<ov::opset11::Multiply>(mm_a, alpha);
+    scaled->set_friendly_name(name + "_lora_scale");
+
+    // scaled @ B^T -> [batch, seq, out_features]
+    auto mm_b = std::make_shared<ov::opset11::MatMul>(scaled, lora_b, false, true);
+    mm_b->set_friendly_name(name + "_lora_b");
+
+    auto lora_add = std::make_shared<ov::opset11::Add>(base, mm_b);
+    lora_add->set_friendly_name(name + "_lora_add");
+    return lora_add->output(0);
+}
 }  // namespace
 
 ov::Output<ov::Node> make_linear(const ov::Output<ov::Node>& input,
@@ -43,22 +105,29 @@ ov::Output<ov::Node> make_linear(const ov::Output<ov::Node>& input,
                                  const std::string& name,
                                  ov::element::Type precision,
                                  const WeightFn& weight_fn,
-                                 const WeightFn& bias_fn) {
+                                 const WeightFn& bias_fn,
+                                 const LoRAInjector* lora) {
     auto weight_output = weight_fn(name + ".weight", ov::Shape{out_features, in_features}, precision);
 
     auto matmul = std::make_shared<ov::opset11::MatMul>(input, weight_output, false, true);
-    matmul->set_friendly_name(name);
+    // optimum-style name so GenAI dynamic-LoRA state ids carry "MatMul" (NPUW pins rank static on it).
+    matmul->set_friendly_name(name + "/MatMul");
+
+    ov::Output<ov::Node> result = matmul->output(0);
 
     if (bias_fn) {
         auto bias = bias_fn(name + ".bias", ov::Shape{out_features}, precision);
-        auto add = std::make_shared<ov::opset11::Add>(matmul, bias);
+        auto add = std::make_shared<ov::opset11::Add>(result, bias);
         add->set_friendly_name(name + "_bias_add");
-        return add->output(0);
+        result = add->output(0);
     }
 
-    return matmul->output(0);
-}
+    if (lora && lora->should_adapt(name)) {
+        result = inject_lora(input, result, in_features, out_features, name, *lora);
+    }
 
+    return result;
+}
 
 ov::Output<ov::Node> make_embedding(const ov::Output<ov::Node>& input_ids,
                                     size_t vocab_size,
@@ -127,7 +196,6 @@ ov::Output<ov::Node> make_conv1d(const ov::Output<ov::Node>& input,
     return add->output(0);
 }
 
-
 ov::Output<ov::Node> make_transformer_layers(const ov::Output<ov::Node>& initial,
                                              size_t num_layers,
                                              const std::string& prefix_base,
@@ -140,8 +208,6 @@ ov::Output<ov::Node> make_transformer_layers(const ov::Output<ov::Node>& initial
     return current;
 }
 
-
-
 std::shared_ptr<ov::Model> ModelBuilder::get_model_with_one_op() {
     auto param = std::make_shared<ov::opset11::Parameter>(ov::element::i64, ov::PartialShape{1, 3, 2, 2});
     param->set_friendly_name("input");
@@ -683,6 +749,7 @@ std::shared_ptr<ov::Model> ModelBuilder::build_llm(const LLMConfig& config_in) {
                     "Hybrid models require use_kv_cache — SSM/conv states are inherently stateful");
     if (!config.norm)
         config.norm = LayerNorm(config.hidden_size, config.precision);
+    const bool has_custom_ffn = static_cast<bool>(config.ffn);
     if (!config.ffn) {
         if (config.num_experts > 0) {
             // Build the FFN from the config's MoE fields; moe_factory selects the topology (empty = GPT-OSS).
@@ -773,6 +840,23 @@ std::shared_ptr<ov::Model> ModelBuilder::build_llm(const LLMConfig& config_in) {
     const auto hs = config.hidden_size;
     const auto kv_heads = config.get_kv_heads();
 
+    LoRAInjector lora_injector;
+    const bool lora_enabled = config.lora_rank > 0;
+    if (lora_enabled) {
+        lora_injector.max_rank = config.lora_rank;
+        lora_injector.targets = config.lora_targets;
+        lora_injector.precision = prec;
+        lora_injector.stateful = config.lora_stateful;
+        lora_injector.sinks = &m_sinks;
+        // FFN is a functor (FFNFn); LoRA reaches it only through a struct that
+        // forwards the injector. Recreate the default dense FFN as a LoRA-aware
+        // SwiGLU; custom and MoE FFN graphs are left untouched.
+        if (!has_custom_ffn && config.num_experts == 0) {
+            config.ffn = SwiGLU(config.hidden_size, config.intermediate_size, prec, config.weight, &lora_injector);
+        }
+    }
+    const LoRAInjector* lora = lora_enabled ? &lora_injector : nullptr;
+
     Attention attn{};
     attn.hidden_size = hs;
     attn.num_heads = config.num_heads;
@@ -785,6 +869,7 @@ std::shared_ptr<ov::Model> ModelBuilder::build_llm(const LLMConfig& config_in) {
     attn.rope_fn = config.rope;
     attn.sdpa_mask = default_mask;
     attn.shared_broadcast_shape = shared_broadcast;
+    attn.lora = lora;
     attn.output_gate = config.attn_output_gate;
 
     // Non-hybrid: standard past_key_values naming. Hybrid full-attention layers get
@@ -890,6 +975,36 @@ std::shared_ptr<ov::Model> ModelBuilder::build_llm(const LLMConfig& config_in) {
     return make_model(final_norm, "last_hidden_state", model_name);
 }
 
+std::shared_ptr<ov::Model> ModelBuilder::build_lora_adapter(const LoRAConfig& config) {
+    clear();
+    const auto prec = config.precision;
+    const auto hs = config.hidden_size;
+
+    LoRAInjector lora;
+    lora.max_rank = config.lora_rank;
+    lora.targets = config.lora_targets;
+    lora.precision = prec;
+    lora.stateful = config.lora_stateful;
+    lora.sinks = &m_sinks;
+
+    const auto& targets = config.lora_targets.empty() ? LoRAInjector::default_targets() : config.lora_targets;
+
+    auto input_ids = parameter(ov::element::i64, ov::PartialShape{-1, -1}, "input_ids");
+    ov::Output<ov::Node> hidden =
+        make_embedding(input_ids->output(0), config.vocab_size, hs, "model.embed_tokens", prec);
+
+    for (size_t layer = 0; layer < config.num_layers; ++layer) {
+        const std::string prefix = "model.layers." + std::to_string(layer) + ".";
+        for (const auto& tgt : targets) {
+            const bool is_attn = tgt == "q_proj" || tgt == "k_proj" || tgt == "v_proj" || tgt == "o_proj";
+            const std::string name = prefix + (is_attn ? "self_attn." : "mlp.") + tgt;
+            hidden = make_linear(hidden, hs, hs, name, prec, config.weight, WeightFn{}, &lora);
+        }
+    }
+
+    return make_model(hidden, "output", "lora_adapter");
+}
+
 std::shared_ptr<ov::Model> ModelBuilder::build_whisper_encoder(const WhisperConfig& config_in) {
     clear();
     WhisperConfig config = config_in;
@@ -972,7 +1087,6 @@ std::shared_ptr<ov::Model> ModelBuilder::build_whisper_encoder(const WhisperConf
     return make_model(encoder_output, "last_hidden_state", "synthetic_whisper_encoder");
 }
 
-
 std::shared_ptr<ov::Model> ModelBuilder::build_whisper_decoder(const WhisperConfig& config_in) {
     clear();
     WhisperConfig config = config_in;

src/plugins/intel_npu/tests/functional/behavior/npuw/test_engine/models/model_builder.hpp

@@ -30,7 +30,8 @@ ov::Output<ov::Node> make_linear(const ov::Output<ov::Node>& input,
                                  const std::string& name,
                                  ov::element::Type precision = ov::element::f32,
                                  const WeightFn& weight_fn = FP32Weight{},
-                                 const WeightFn& bias_fn = {});
+                                 const WeightFn& bias_fn = {},
+                                 const LoRAInjector* lora = nullptr);
 
 ov::Output<ov::Node> make_embedding(const ov::Output<ov::Node>& input_ids,
                                     size_t vocab_size,
@@ -169,6 +170,13 @@ using SlidingMaskFn = std::function<ov::Output<ov::Node>(const ov::Output<ov::No
                                                          ov::element::Type,
                                                          size_t)>;
 
+/// Minimal standalone LoRA adapter model config (no attention/KV/RoPE).
+struct LoRAConfig : public BaseModelConfig {
+    size_t lora_rank = 8;
+    std::vector<std::string> lora_targets;  ///< Empty = q,k,v,o,gate,up,down.
+    bool lora_stateful = false;             ///< true = ReadValue/Assign states.
+};
+
 struct LLMConfig : public BaseModelConfig {
     bool use_kv_cache = true;
     bool use_inputs_embeds = false;
@@ -197,6 +205,16 @@ struct LLMConfig : public BaseModelConfig {
     bool use_token_type_ids = false;     ///< Gemma 3 VLM: token_type_ids param (0=text/causal, 1=image/bidir)
     SlidingMaskFn sliding_mask_fn;       ///< Empty = default float SWA (matches no NPUW pass; set make_sliding_window_mask_phi3 for Phi-3/Gemma-2/Gemma-3).
 
+    /// LoRA adapter support: inject low-rank A/B/alpha tensors per adapted linear layer.
+    /// 0 = no LoRA. >0 = inject LoRA with this max rank into adapted projections.
+    size_t lora_rank = 0;
+    /// Which projection names get LoRA adapters ("q_proj", "v_proj", etc).
+    /// Empty = default set (q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj).
+    std::vector<std::string> lora_targets;
+    /// false = expose LoRA tensors as Parameters (NPUW stateless form).
+    /// true = expose LoRA tensors as ReadValue/Assign states (GenAI dynamic form before NPUW conversion).
+    bool lora_stateful = false;
+
     /// Hybrid scheduling. Empty predicate (or null linear_mixer) = pure attention. Layers where it
     /// returns true use linear_mixer, the rest use full SDPA. Hybrid models require use_kv_cache = true.
     std::function<bool(size_t /*layer_idx*/)> is_linear_layer;
@@ -264,6 +282,7 @@ class ModelBuilder {
                                                      const std::string& name);
 
     std::shared_ptr<ov::Model> build_llm(const LLMConfig& config);
+    std::shared_ptr<ov::Model> build_lora_adapter(const LoRAConfig& config = {});
     std::shared_ptr<ov::Model> build_whisper_encoder(const WhisperConfig& config);
     std::shared_ptr<ov::Model> build_whisper_decoder(const WhisperConfig& config);
     std::shared_ptr<ov::Model> build_embedding_encoder(const BertConfig& config);

src/plugins/intel_npu/tests/functional/behavior/npuw/test_engine/models/model_builder_attention.cpp

@@ -239,7 +239,7 @@ ov::Output<ov::Node> make_sdpa(const ov::Output<ov::Node>& q,
     if (head_dim_for_scale > 0 && attention_mask.get_node()) {
         // 5-input SDPA: Q, K, V, mask, scale (required for embedding model pattern matching)
         auto scale_val = 1.0f / std::sqrt(static_cast<float>(head_dim_for_scale));
-        auto scale = ov::opset11::Constant::create(ov::element::f32, ov::Shape{}, {scale_val});
+        auto scale = ov::opset11::Constant::create(q.get_element_type(), ov::Shape{}, {scale_val});
         scale->set_friendly_name(name + ".scale");
         sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, attention_mask, scale, false);
     } else if (attention_mask.get_node()) {
@@ -259,7 +259,8 @@ ov::Output<ov::Node> make_attention_output(const ov::Output<ov::Node>& sdpa_outp
                                            ov::element::Type precision,
                                            const WeightFn& weight_fn,
                                            const WeightFn& bias_fn,
-                                           const ov::Output<ov::Node>& output_gate) {
+                                           const ov::Output<ov::Node>& output_gate,
+                                           const LoRAInjector* lora) {
     auto attn_trans = make_attention_transpose(sdpa_output, name + "_transpose");
 
     auto reshape_shape = ov::opset11::Constant::create(ov::element::i64,
@@ -277,7 +278,7 @@ ov::Output<ov::Node> make_attention_output(const ov::Output<ov::Node>& sdpa_outp
         o_proj_in = gated->output(0);
     }
 
-    return make_linear(o_proj_in, attn_dim, hidden_size, name, precision, weight_fn, bias_fn);
+    return make_linear(o_proj_in, attn_dim, hidden_size, name, precision, weight_fn, bias_fn, lora);
 }
 
 ov::Output<ov::Node> Attention::operator()(const ov::Output<ov::Node>& q,
@@ -356,7 +357,8 @@ ov::Output<ov::Node> Attention::operator()(const ov::Output<ov::Node>& q,
                                  precision,
                                  weight_fn,
                                  bias_fn,
-                                 gate_flat);
+                                 gate_flat,
+                                 lora);
 }
 
 ov::Output<ov::Node> Attention::operator()(const ov::Output<ov::Node>& input,
@@ -366,9 +368,12 @@ ov::Output<ov::Node> Attention::operator()(const ov::Output<ov::Node>& input,
     auto kv_src = kv_input.get_node() ? kv_input : input;
     size_t q_dim = (output_gate ? 2 : 1) * num_heads * head_dim;
     size_t kv_dim = num_kv_heads * head_dim;
-    auto q = make_linear(input, hidden_size, q_dim, prefix + attn_prefix + "q_proj", precision, weight_fn, bias_fn);
-    auto k = make_linear(kv_src, hidden_size, kv_dim, prefix + attn_prefix + "k_proj", precision, weight_fn, bias_fn);
-    auto v = make_linear(kv_src, hidden_size, kv_dim, prefix + attn_prefix + "v_proj", precision, weight_fn, bias_fn);
+    auto q =
+        make_linear(input, hidden_size, q_dim, prefix + attn_prefix + "q_proj", precision, weight_fn, bias_fn, lora);
+    auto k =
+        make_linear(kv_src, hidden_size, kv_dim, prefix + attn_prefix + "k_proj", precision, weight_fn, bias_fn, lora);
+    auto v =
+        make_linear(kv_src, hidden_size, kv_dim, prefix + attn_prefix + "v_proj", precision, weight_fn, bias_fn, lora);
     return (*this)(q, k, v, prefix, layer_idx);
 }
 

src/plugins/intel_npu/tests/functional/behavior/npuw/test_engine/models/model_builder_attention.hpp

@@ -80,7 +80,8 @@ ov::Output<ov::Node> make_attention_output(const ov::Output<ov::Node>& sdpa_outp
                                            ov::element::Type precision,
                                            const WeightFn& weight_fn,
                                            const WeightFn& bias_fn = {},
-                                           const ov::Output<ov::Node>& output_gate = {});
+                                           const ov::Output<ov::Node>& output_gate = {},
+                                           const LoRAInjector* lora = nullptr);
 
 /// Takes pre-projected Q, K, V. Handles reshape, QK-norm, RoPE, KV cache, GQA, SDPA, O proj.
 struct Attention {
@@ -94,6 +95,7 @@ struct Attention {
 
     ov::Output<ov::Node> sdpa_mask;
     ov::Output<ov::Node> shared_broadcast_shape;
+    const LoRAInjector* lora = nullptr;
 
     /// Qwen3.5-style gated attention: q_proj carries [q | gate] per head (2x width),
     /// split after the per-head reshape; Sigmoid(gate) scales the flattened SDPA output.

src/plugins/intel_npu/tests/functional/behavior/npuw/test_engine/models/model_builder_ffn.cpp

@@ -15,8 +15,11 @@ namespace test {
 namespace npuw {
 
 ov::Output<ov::Node> SwiGLU::operator()(const ov::Output<ov::Node>& input, const std::string& name) const {
-    auto gate = make_linear(input, hidden_size, intermediate_size, name + ".gate_proj", precision, weight_fn);
-    auto up = make_linear(input, hidden_size, intermediate_size, name + ".up_proj", precision, weight_fn);
+    WeightFn no_bias;
+    auto gate =
+        make_linear(input, hidden_size, intermediate_size, name + ".gate_proj", precision, weight_fn, no_bias, lora);
+    auto up =
+        make_linear(input, hidden_size, intermediate_size, name + ".up_proj", precision, weight_fn, no_bias, lora);
 
     auto sigmoid = std::make_shared<ov::opset11::Sigmoid>(gate);
 
@@ -26,18 +29,21 @@ ov::Output<ov::Node> SwiGLU::operator()(const ov::Output<ov::Node>& input, const
     auto gate_up = std::make_shared<ov::opset11::Multiply>(silu, up);
     gate_up->set_friendly_name(name + "_gate_up");
 
-    auto down = make_linear(gate_up, intermediate_size, hidden_size, name + ".down_proj", precision, weight_fn);
+    auto down =
+        make_linear(gate_up, intermediate_size, hidden_size, name + ".down_proj", precision, weight_fn, no_bias, lora);
 
     return down;
 }
 
 ov::Output<ov::Node> GELU::operator()(const ov::Output<ov::Node>& input, const std::string& name) const {
-    auto up = make_linear(input, hidden_size, intermediate_size, name + ".up_proj", precision, weight_fn, bias_fn);
+    auto up =
+        make_linear(input, hidden_size, intermediate_size, name + ".up_proj", precision, weight_fn, bias_fn, lora);
 
     auto gelu = std::make_shared<ov::opset11::Gelu>(up);
     gelu->set_friendly_name(name + "_gelu");
 
-    auto down = make_linear(gelu, intermediate_size, hidden_size, name + ".down_proj", precision, weight_fn, bias_fn);
+    auto down =
+        make_linear(gelu, intermediate_size, hidden_size, name + ".down_proj", precision, weight_fn, bias_fn, lora);
 
     return down;
 }