feat(agentic): Phase 3 — incremental (KV-cache) decoding seam + engine fast path (#1544)

- IIncrementalCausalLanguageModel<T> : ICausalLanguageModel<T> adds ResetCache / StartSequence(prompt) / AppendToken(id) — the contract a KV-cached model implements to advance one token at a time instead of re-feeding the full O(n^2) context.

- LocalEngineChatClient auto-detects the interface and takes the incremental fast path (prime once, then feed single tokens), falling back to full re-feed otherwise. Constraint + stop-sequence + sampler logic shared via a single PickToken helper across both paths.

- 2 tests (green net10.0 + net471): the engine drives the incremental path correctly (ResetCache once, StartSequence with the prompt, AppendToken per generated token) and produces output IDENTICAL to the full-refeed path.

The remaining half — a real K/V cache inside Mamba/GLA/attention forward — is a model-layer change (the network Predict API has no incremental entry point yet); this lands the engine-side support + verification it plugs into. Part of epic #1544 (Phase 3).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: franklinic

src/Agentic/Models/Local/IIncrementalCausalLanguageModel.cs

@@ -0,0 +1,46 @@
+using AiDotNet.LinearAlgebra;
+
+namespace AiDotNet.Agentic.Models.Local;
+
+/// <summary>
+/// An <see cref="ICausalLanguageModel{T}"/> that supports incremental (KV-cached) decoding: it processes the
+/// prompt once, caches the per-position state, and then advances one token at a time without recomputing the
+/// whole sequence. This is the fast path for autoregressive generation.
+/// </summary>
+/// <typeparam name="T">The tensor element type.</typeparam>
+/// <remarks>
+/// <para>
+/// The base <see cref="ICausalLanguageModel{T}.NextTokenLogits"/> re-feeds the full context each step, which
+/// is correct but O(n²) over a generation. A model that maintains a key/value cache implements this interface
+/// so <see cref="LocalEngineChatClient{T}"/> can drive it incrementally: <see cref="StartSequence"/> primes
+/// the cache with the prompt and returns the first next-token logits, then each <see cref="AppendToken"/>
+/// feeds a single new token and returns the following logits. <see cref="ResetCache"/> clears state between
+/// independent generations.
+/// </para>
+/// <para><b>For Beginners:</b> Without caching, predicting each new word re-reads the entire conversation —
+/// slower and slower as it grows. With caching, the model remembers the work it already did and only looks at
+/// the one new word each time. This interface is how a model advertises "I can do the fast, remember-as-you-go
+/// version"; the engine uses it automatically when available and falls back to the simple way otherwise.
+/// </para>
+/// </remarks>
+public interface IIncrementalCausalLanguageModel<T> : ICausalLanguageModel<T>
+{
+    /// <summary>
+    /// Clears any cached decoding state, so the next <see cref="StartSequence"/> begins fresh.
+    /// </summary>
+    void ResetCache();
+
+    /// <summary>
+    /// Processes the prompt, populating the cache, and returns the logits for the first generated token.
+    /// </summary>
+    /// <param name="promptTokenIds">The prompt token ids. Must be non-empty.</param>
+    /// <returns>The next-token logits following the prompt (length = <see cref="ICausalLanguageModel{T}.VocabularySize"/>).</returns>
+    Vector<T> StartSequence(IReadOnlyList<int> promptTokenIds);
+
+    /// <summary>
+    /// Appends a single token to the cached context and returns the logits for the token after it.
+    /// </summary>
+    /// <param name="tokenId">The token id to append (typically the one just generated).</param>
+    /// <returns>The next-token logits (length = <see cref="ICausalLanguageModel{T}.VocabularySize"/>).</returns>
+    Vector<T> AppendToken(int tokenId);
+}

src/Agentic/Models/Local/LocalEngineChatClient.cs

@@ -86,7 +86,9 @@ public Task<ChatResponse> GetResponseAsync(
             var sampling = ResolveSampling(options);
             var sampler = new TokenSampler<T>(sampling.Seed);
             generated = new List<int>();
-            finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken);
+            finishReason = _model is IIncrementalCausalLanguageModel<T> incremental
+                ? RunGenerationIncremental(incremental, promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken)
+                : RunGeneration(promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken);
         }
 
         var text = generated.Count > 0 ? _tokenizer.Decode(generated) : string.Empty;
@@ -182,11 +184,51 @@ private ChatFinishReason RunGeneration(
         return ChatFinishReason.Length;
     }
 
-    // Computes the next token id honoring the configured constraint, or null when the constraint signals a
-    // terminal state (no valid continuation exists).
-    private int? NextToken(
-        List<int> context,
+    // Incremental (KV-cached) generation: prime with the prompt, then advance one token at a time.
+    private ChatFinishReason RunGenerationIncremental(
+        IIncrementalCausalLanguageModel<T> model,
+        IReadOnlyList<int> promptIds,
+        int maxTokens,
+        TokenSampler<T> sampler,
+        LocalSamplingOptions sampling,
+        IReadOnlyList<string>? stopSequences,
         List<int> generated,
+        CancellationToken cancellationToken)
+    {
+        model.ResetCache();
+        var logits = model.StartSequence(promptIds);
+
+        for (var step = 0; step < maxTokens; step++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var next = PickToken(generated, logits, sampler, sampling);
+            if (next is null || next.Value == _tokenizer.EosTokenId)
+            {
+                return ChatFinishReason.Stop;
+            }
+
+            generated.Add(next.Value);
+
+            if (stopSequences is not null && ContainsStopSequence(_tokenizer.Decode(generated), stopSequences))
+            {
+                return ChatFinishReason.Stop;
+            }
+
+            if (step < maxTokens - 1)
+            {
+                logits = model.AppendToken(next.Value);
+            }
+        }
+
+        return ChatFinishReason.Length;
+    }
+
+    // Computes the next token id from precomputed logits, honoring the configured constraint, or null when
+    // the constraint signals a terminal state (no valid continuation exists).
+    private int? PickToken(
+        List<int> generated,
+        Vector<T> logits,
         TokenSampler<T> sampler,
         LocalSamplingOptions sampling)
     {
@@ -200,10 +242,17 @@ private ChatFinishReason RunGeneration(
             }
         }
 
-        var logits = _model.NextTokenLogits(context);
         return sampler.Sample(logits, sampling, allowed);
     }
 
+    // Computes the next token id by re-feeding the full context (fallback when the model has no KV-cache).
+    private int? NextToken(
+        List<int> context,
+        List<int> generated,
+        TokenSampler<T> sampler,
+        LocalSamplingOptions sampling) =>
+        PickToken(generated, _model.NextTokenLogits(context), sampler, sampling);
+
     // Beam search: explore `beamWidth` hypotheses in parallel, expanding each by its top tokens (honoring
     // the constraint), and keep the highest-scoring (length-normalized log-probability) beams. Returns the
     // best completion. Deterministic.

tests/AiDotNet.Tests/UnitTests/Agentic/Local/IncrementalDecodingTests.cs

@@ -0,0 +1,159 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using AiDotNet.Agentic.Models;
+using AiDotNet.Agentic.Models.Local;
+using AiDotNet.LinearAlgebra;
+using Xunit;
+
+namespace AiDotNetTests.UnitTests.Agentic.Local
+{
+    public class IncrementalDecodingTests
+    {
+        // Deterministic next-token rule keyed on the last token: 0(EOS)->1, 1->2, 2->3, 3->0(EOS).
+        private static readonly Dictionary<int, int> Transition = new()
+        {
+            [0] = 1,
+            [1] = 2,
+            [2] = 3,
+            [3] = 0,
+        };
+
+        private const int Vocab = 4;
+
+        private static MapTokenizer BuildTokenizer() => new(eos: 0, ("A", 1), ("B", 2), ("C", 3));
+
+        private static double[] LogitsFor(int lastToken)
+        {
+            var next = Transition.TryGetValue(lastToken, out var n) ? n : 0;
+            var logits = new double[Vocab];
+            logits[next] = 10.0;
+            return logits;
+        }
+
+        [Fact(Timeout = 60000)]
+        public async Task Engine_UsesIncrementalPath_FeedingOneTokenAtATime()
+        {
+            var model = new IncrementalProbe();
+            var engine = new LocalEngineChatClient<double>(model, BuildTokenizer(), options: new LocalEngineOptions
+            {
+                Sampling = new LocalSamplingOptions { Temperature = 0.0 },
+            });
+
+            var response = await engine.GetResponseAsync(new[] { ChatMessage.User("go") });
+
+            Assert.Equal("ABC", response.Text);
+            Assert.Equal(ChatFinishReason.Stop, response.FinishReason);
+
+            // The engine took the incremental fast path: reset once, primed once with the prompt, then fed
+            // each generated token singly via AppendToken.
+            Assert.Equal(1, model.ResetCalls);
+            Assert.Equal(1, model.StartCalls);
+            Assert.Equal(new[] { 1, 2, 3 }, model.AppendedTokens);
+        }
+
+        [Fact(Timeout = 60000)]
+        public async Task IncrementalAndFullRefeed_ProduceIdenticalOutput()
+        {
+            var tokenizer = BuildTokenizer();
+            var sampling = new LocalSamplingOptions { Temperature = 0.0 };
+
+            var incremental = new LocalEngineChatClient<double>(new IncrementalProbe(), tokenizer,
+                options: new LocalEngineOptions { Sampling = sampling });
+            var fullRefeed = new LocalEngineChatClient<double>(new FullRefeedModel(), tokenizer,
+                options: new LocalEngineOptions { Sampling = sampling });
+
+            var a = await incremental.GetResponseAsync(new[] { ChatMessage.User("go") });
+            var b = await fullRefeed.GetResponseAsync(new[] { ChatMessage.User("go") });
+
+            Assert.Equal(b.Text, a.Text);
+            Assert.Equal("ABC", a.Text);
+        }
+
+        // ---- Test doubles ----
+
+        // A KV-cached model: tracks how the engine drives it and advances state by the last token only.
+        private sealed class IncrementalProbe : IIncrementalCausalLanguageModel<double>
+        {
+            private int _lastToken;
+
+            public int VocabularySize => Vocab;
+
+            public int ResetCalls { get; private set; }
+
+            public int StartCalls { get; private set; }
+
+            public List<int> AppendedTokens { get; } = new();
+
+            public void ResetCache()
+            {
+                ResetCalls++;
+                _lastToken = 0;
+            }
+
+            public Vector<double> StartSequence(IReadOnlyList<int> promptTokenIds)
+            {
+                StartCalls++;
+                _lastToken = promptTokenIds[promptTokenIds.Count - 1];
+                return new Vector<double>(LogitsFor(_lastToken));
+            }
+
+            public Vector<double> AppendToken(int tokenId)
+            {
+                AppendedTokens.Add(tokenId);
+                _lastToken = tokenId;
+                return new Vector<double>(LogitsFor(_lastToken));
+            }
+
+            // Full-refeed fallback (unused when the engine takes the incremental path).
+            public Vector<double> NextTokenLogits(IReadOnlyList<int> tokenIds) =>
+                new(LogitsFor(tokenIds[tokenIds.Count - 1]));
+        }
+
+        // Same next-token rule, but no KV-cache: forces the engine's full-context path.
+        private sealed class FullRefeedModel : ICausalLanguageModel<double>
+        {
+            public int VocabularySize => Vocab;
+
+            public Vector<double> NextTokenLogits(IReadOnlyList<int> tokenIds) =>
+                new(LogitsFor(tokenIds[tokenIds.Count - 1]));
+        }
+
+        private sealed class MapTokenizer : IGenerationTokenizer
+        {
+            private readonly Dictionary<string, int> _wordToId = new(StringComparer.Ordinal);
+            private readonly Dictionary<int, string> _idToWord = new();
+
+            public MapTokenizer(int eos, params (string Word, int Id)[] entries)
+            {
+                EosTokenId = eos;
+                _idToWord[eos] = string.Empty;
+                foreach (var (word, id) in entries)
+                {
+                    _wordToId[word] = id;
+                    _idToWord[id] = word;
+                }
+            }
+
+            public int EosTokenId { get; }
+
+            public IReadOnlyList<int> Encode(string text)
+            {
+                var ids = text
+                    .Split(new[] { ' ', '\n', '\r', '\t' }, StringSplitOptions.RemoveEmptyEntries)
+                    .Select(w => _wordToId.TryGetValue(w, out var id) ? id : EosTokenId)
+                    .ToList();
+                if (ids.Count == 0)
+                {
+                    ids.Add(EosTokenId);
+                }
+
+                return ids;
+            }
+
+            public string Decode(IReadOnlyList<int> tokenIds) =>
+                string.Concat(tokenIds.Select(id => _idToWord.TryGetValue(id, out var w) ? w : string.Empty));
+        }
+    }
+}