feat(agentic): Phase 3 — local in-process inference engine (#1544)

The flagship local-first differentiator: an IChatClient<T> that runs entirely in-process over AiDotNet's own model — no network, no API key. Same agent code drives it as drives OpenAI/Anthropic.

- ICausalLanguageModel<T>: minimal next-token-logits seam the real Transformer (or any model) plugs into; implementations may keep an internal KV-cache.

- IGenerationTokenizer (+TokenizerGenerationAdapter over the repo's ITokenizer): encode/decode/EOS seam.

- TokenSampler<T>: greedy (temp 0 = argmax) + temperature + top-k + top-p (nucleus), seedable for reproducibility; reads logits via Convert.ToDouble so float/double share one path.

- IChatPromptTemplate + ChatMlPromptTemplate (role-tagged, opens the assistant turn).

- LocalEngineChatClient<T>: renders prompt -> encodes -> autoregressively samples until EOS or token limit -> decodes; non-streaming + streaming (incremental decode deltas); usage = prompt/generated token counts. Drop-in for agents/supervisor/swarm/memory. (Native tool-calling + structured-output constraints via constrained decoding are a follow-up; tools are ignored this slice.)

- 9 tests (green net10.0 + net471) on a deterministic ScriptedCausalModel + WordTokenizer: greedy decode + EOS stop, token-limit -> Length, streaming deltas reconstruct full text, AgentExecutor drop-in, sampler argmax/seed-reproducibility/top-k/top-p, ChatML template. No null-forgiving.

Stacked on Phase 2 (#1551) — the integration test shows the local engine driving an AgentExecutor. Part of epic #1544 (Phase 3).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: franklinic

src/Agentic/Models/Local/ChatMlPromptTemplate.cs

@@ -0,0 +1,52 @@
+using System.Text;
+using AiDotNet.Agentic.Models;
+
+// Disambiguate from the legacy AiDotNet.PromptEngineering.Templates.ChatMessage (global using).
+using ChatMessage = AiDotNet.Agentic.Models.ChatMessage;
+
+namespace AiDotNet.Agentic.Models.Local;
+
+/// <summary>
+/// A ChatML-style <see cref="IChatPromptTemplate"/> that renders each message as
+/// <c>&lt;|role|&gt;\n{text}\n</c> and ends with an open <c>&lt;|assistant|&gt;</c> turn for the model to
+/// complete. This is a widely-used, model-agnostic default.
+/// </summary>
+/// <remarks>
+/// <para><b>For Beginners:</b> This writes the conversation like a script: each line is tagged with who is
+/// speaking (<c>&lt;|user|&gt;</c>, <c>&lt;|assistant|&gt;</c>, ...), and the script stops right where the
+/// assistant is about to speak — so the model fills in the assistant's reply.
+/// </para>
+/// </remarks>
+public sealed class ChatMlPromptTemplate : IChatPromptTemplate
+{
+    /// <inheritdoc/>
+    /// <exception cref="ArgumentNullException">Thrown when <paramref name="messages"/> is <c>null</c>.</exception>
+    /// <exception cref="ArgumentException">Thrown when <paramref name="messages"/> is empty.</exception>
+    public string Render(IReadOnlyList<ChatMessage> messages)
+    {
+        Guard.NotNull(messages);
+        if (messages.Count == 0)
+        {
+            throw new ArgumentException("The conversation must contain at least one message.", nameof(messages));
+        }
+
+        var builder = new StringBuilder();
+        foreach (var message in messages)
+        {
+            builder.Append("<|").Append(RoleTag(message.Role)).Append("|>\n");
+            builder.Append(message.Text).Append('\n');
+        }
+
+        builder.Append("<|assistant|>\n");
+        return builder.ToString();
+    }
+
+    private static string RoleTag(ChatRole role) => role switch
+    {
+        ChatRole.System => "system",
+        ChatRole.User => "user",
+        ChatRole.Assistant => "assistant",
+        ChatRole.Tool => "tool",
+        _ => "user"
+    };
+}

src/Agentic/Models/Local/ICausalLanguageModel.cs

@@ -0,0 +1,38 @@
+using AiDotNet.LinearAlgebra;
+
+namespace AiDotNet.Agentic.Models.Local;
+
+/// <summary>
+/// The minimal contract an in-process language model exposes to the local generation engine: given the
+/// tokens seen so far, produce the logits for the next token. This is the seam between AiDotNet's own
+/// Transformer (or any other model) and <see cref="LocalEngineChatClient{T}"/>.
+/// </summary>
+/// <typeparam name="T">The tensor element type (e.g., <see cref="float"/> or <see cref="double"/>).</typeparam>
+/// <remarks>
+/// <para>
+/// Keeping the contract this small means the generation loop, sampling, and chat templating are written
+/// once and tested independently of any particular model, and the real network is wired in behind this
+/// interface. An implementation is free to maintain an internal KV-cache keyed on the growing context so
+/// repeated calls stay efficient; callers only ever ask for "the next-token logits given this context".
+/// </para>
+/// <para><b>For Beginners:</b> A language model, at its heart, answers one question over and over: "given
+/// everything so far, how likely is each possible next word-piece?" Those likelihoods (before turning them
+/// into probabilities) are called <em>logits</em>. This interface is exactly that one question, so the rest
+/// of the engine can focus on <em>choosing</em> the next token and stitching the words back together.
+/// </para>
+/// </remarks>
+public interface ICausalLanguageModel<T>
+{
+    /// <summary>
+    /// Gets the size of the model's vocabulary (the length of the logits vector returned by
+    /// <see cref="NextTokenLogits"/>).
+    /// </summary>
+    int VocabularySize { get; }
+
+    /// <summary>
+    /// Computes the next-token logits for the given context.
+    /// </summary>
+    /// <param name="tokenIds">The token ids of the context so far (prompt plus any tokens already generated). Must be non-empty.</param>
+    /// <returns>A vector of length <see cref="VocabularySize"/> giving the unnormalized score for each candidate next token.</returns>
+    Vector<T> NextTokenLogits(IReadOnlyList<int> tokenIds);
+}

src/Agentic/Models/Local/IChatPromptTemplate.cs

@@ -0,0 +1,27 @@
+using AiDotNet.Agentic.Models;
+
+// Disambiguate from the legacy AiDotNet.PromptEngineering.Templates.ChatMessage (global using).
+using ChatMessage = AiDotNet.Agentic.Models.ChatMessage;
+
+namespace AiDotNet.Agentic.Models.Local;
+
+/// <summary>
+/// Renders a chat conversation (a list of <see cref="ChatMessage"/>) into the single prompt string a local
+/// language model is fed. Different model families expect different role markers, so this is pluggable.
+/// </summary>
+/// <remarks>
+/// <para><b>For Beginners:</b> Cloud chat APIs accept a list of messages directly, but a raw local model
+/// just continues one block of text. This converts the conversation into that block, tagging who said what
+/// (system/user/assistant) in the format the model was trained on, and ends with the assistant's turn so the
+/// model knows it should reply next.
+/// </para>
+/// </remarks>
+public interface IChatPromptTemplate
+{
+    /// <summary>
+    /// Renders the conversation into a prompt string.
+    /// </summary>
+    /// <param name="messages">The conversation so far. Must be non-empty.</param>
+    /// <returns>The prompt text to encode and feed to the model.</returns>
+    string Render(IReadOnlyList<ChatMessage> messages);
+}

src/Agentic/Models/Local/IGenerationTokenizer.cs

@@ -0,0 +1,39 @@
+namespace AiDotNet.Agentic.Models.Local;
+
+/// <summary>
+/// The minimal tokenizer contract the local generation engine needs: turn text into token ids, turn token
+/// ids back into text, and know which token marks end-of-sequence.
+/// </summary>
+/// <remarks>
+/// <para>
+/// This is intentionally narrower than the full <see cref="AiDotNet.Tokenization.Interfaces.ITokenizer"/> so
+/// the engine stays decoupled and trivially testable. <see cref="TokenizerGenerationAdapter"/> bridges a
+/// real repo tokenizer to this seam.
+/// </para>
+/// <para><b>For Beginners:</b> Models don't read text directly — they read numbers (token ids). This turns
+/// your prompt into those numbers, turns the model's numbers back into readable text, and tells the engine
+/// the special "stop here" token so it knows when the model is done.
+/// </para>
+/// </remarks>
+public interface IGenerationTokenizer
+{
+    /// <summary>
+    /// Gets the id of the end-of-sequence token. Generation stops when the model produces it. A negative
+    /// value means "no EOS token" (generation then stops only at the token limit).
+    /// </summary>
+    int EosTokenId { get; }
+
+    /// <summary>
+    /// Encodes text into token ids.
+    /// </summary>
+    /// <param name="text">The text to encode.</param>
+    /// <returns>The token ids.</returns>
+    IReadOnlyList<int> Encode(string text);
+
+    /// <summary>
+    /// Decodes token ids back into text.
+    /// </summary>
+    /// <param name="tokenIds">The token ids to decode.</param>
+    /// <returns>The decoded text.</returns>
+    string Decode(IReadOnlyList<int> tokenIds);
+}

src/Agentic/Models/Local/LocalEngineChatClient.cs

@@ -0,0 +1,195 @@
+using System.Runtime.CompilerServices;
+using AiDotNet.Agentic.Models;
+
+// Disambiguate from the legacy AiDotNet.PromptEngineering.Templates.ChatMessage (global using).
+using ChatMessage = AiDotNet.Agentic.Models.ChatMessage;
+
+namespace AiDotNet.Agentic.Models.Local;
+
+/// <summary>
+/// An <see cref="IChatClient{T}"/> that runs entirely in-process over an <see cref="ICausalLanguageModel{T}"/>
+/// — no network, no API key, no external service. It renders the conversation to a prompt, encodes it,
+/// autoregressively samples tokens until the end-of-sequence token or a length limit, and decodes the
+/// result. This is the flagship "local-first" capability: the same agent code that drives OpenAI or
+/// Anthropic drives AiDotNet's own model.
+/// </summary>
+/// <typeparam name="T">The tensor element type shared with the model.</typeparam>
+/// <remarks>
+/// <para>
+/// Because it implements <see cref="IChatClient{T}"/>, the local engine is a drop-in for every higher layer
+/// (agents, supervisor/swarm, memory). Both non-streaming and streaming generation are supported; streaming
+/// decodes incrementally and yields the new text on each step. Native tool-calling and structured-output
+/// constraints are not applied by this slice (they require constrained decoding, a follow-up) — supplied
+/// tools are ignored and the engine produces plain text.
+/// </para>
+/// <para><b>For Beginners:</b> This is your own chatbot brain running on your machine. You hand it the
+/// conversation; it writes the reply one word-piece at a time until it decides it's done or hits the length
+/// cap. Everything else in this library that talks to a "chat model" can talk to this one instead — so you
+/// can build agents with no cloud dependency at all.
+/// </para>
+/// </remarks>
+public sealed class LocalEngineChatClient<T> : IChatClient<T>
+{
+    private readonly ICausalLanguageModel<T> _model;
+    private readonly IGenerationTokenizer _tokenizer;
+    private readonly IChatPromptTemplate _template;
+    private readonly LocalEngineOptions _options;
+
+    /// <summary>
+    /// Initializes a new local engine.
+    /// </summary>
+    /// <param name="model">The in-process language model that produces next-token logits.</param>
+    /// <param name="tokenizer">The tokenizer used to encode prompts and decode generated tokens.</param>
+    /// <param name="template">The chat prompt template. <c>null</c> uses <see cref="ChatMlPromptTemplate"/>.</param>
+    /// <param name="options">Engine settings. <c>null</c> uses defaults.</param>
+    /// <exception cref="ArgumentNullException">Thrown when <paramref name="model"/> or <paramref name="tokenizer"/> is <c>null</c>.</exception>
+    public LocalEngineChatClient(
+        ICausalLanguageModel<T> model,
+        IGenerationTokenizer tokenizer,
+        IChatPromptTemplate? template = null,
+        LocalEngineOptions? options = null)
+    {
+        Guard.NotNull(model);
+        Guard.NotNull(tokenizer);
+        _model = model;
+        _tokenizer = tokenizer;
+        _template = template ?? new ChatMlPromptTemplate();
+        _options = options ?? new LocalEngineOptions();
+    }
+
+    /// <inheritdoc/>
+    public string ModelId =>
+        _options.ModelId is { } id && id.Trim().Length > 0 ? id : "local";
+
+    /// <inheritdoc/>
+    /// <exception cref="ArgumentNullException">Thrown when <paramref name="messages"/> is <c>null</c>.</exception>
+    /// <exception cref="ArgumentException">Thrown when <paramref name="messages"/> is empty.</exception>
+    public Task<ChatResponse> GetResponseAsync(
+        IReadOnlyList<ChatMessage> messages,
+        ChatOptions? options = null,
+        CancellationToken cancellationToken = default)
+    {
+        var promptIds = BuildPromptIds(messages);
+        var sampling = ResolveSampling(options);
+        var sampler = new TokenSampler<T>(sampling.Seed);
+        var maxTokens = ResolveMaxTokens(options);
+
+        var generated = new List<int>();
+        var finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, generated, cancellationToken);
+
+        var text = generated.Count > 0 ? _tokenizer.Decode(generated) : string.Empty;
+        var usage = new ChatUsage(promptIds.Count, generated.Count);
+        var response = new ChatResponse(ChatMessage.Assistant(text), finishReason, usage, ModelId);
+        return Task.FromResult(response);
+    }
+
+    /// <inheritdoc/>
+    public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
+        IReadOnlyList<ChatMessage> messages,
+        ChatOptions? options = null,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        await Task.CompletedTask.ConfigureAwait(false);
+
+        var promptIds = BuildPromptIds(messages);
+        var sampling = ResolveSampling(options);
+        var sampler = new TokenSampler<T>(sampling.Seed);
+        var maxTokens = ResolveMaxTokens(options);
+
+        yield return new ChatResponseUpdate(role: ChatRole.Assistant);
+
+        var context = new List<int>(promptIds);
+        var generated = new List<int>();
+        var previousText = string.Empty;
+        var finishReason = ChatFinishReason.Length;
+
+        for (var step = 0; step < maxTokens; step++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var logits = _model.NextTokenLogits(context);
+            var next = sampler.Sample(logits, sampling);
+            if (next == _tokenizer.EosTokenId)
+            {
+                finishReason = ChatFinishReason.Stop;
+                break;
+            }
+
+            generated.Add(next);
+            context.Add(next);
+
+            var fullText = _tokenizer.Decode(generated);
+            if (fullText.Length > previousText.Length)
+            {
+                var delta = fullText.Substring(previousText.Length);
+                previousText = fullText;
+                yield return ChatResponseUpdate.ForText(delta);
+            }
+        }
+
+        yield return ChatResponseUpdate.ForFinish(finishReason, new ChatUsage(promptIds.Count, generated.Count));
+    }
+
+    private ChatFinishReason RunGeneration(
+        IReadOnlyList<int> promptIds,
+        int maxTokens,
+        TokenSampler<T> sampler,
+        LocalSamplingOptions sampling,
+        List<int> generated,
+        CancellationToken cancellationToken)
+    {
+        var context = new List<int>(promptIds);
+        for (var step = 0; step < maxTokens; step++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var logits = _model.NextTokenLogits(context);
+            var next = sampler.Sample(logits, sampling);
+            if (next == _tokenizer.EosTokenId)
+            {
+                return ChatFinishReason.Stop;
+            }
+
+            generated.Add(next);
+            context.Add(next);
+        }
+
+        return ChatFinishReason.Length;
+    }
+
+    private List<int> BuildPromptIds(IReadOnlyList<ChatMessage> messages)
+    {
+        Guard.NotNull(messages);
+        if (messages.Count == 0)
+        {
+            throw new ArgumentException("The conversation must contain at least one message.", nameof(messages));
+        }
+
+        var prompt = _template.Render(messages);
+        return new List<int>(_tokenizer.Encode(prompt));
+    }
+
+    private int ResolveMaxTokens(ChatOptions? options)
+    {
+        if (options?.MaxOutputTokens is { } requested && requested > 0)
+        {
+            return requested;
+        }
+
+        return _options.MaxOutputTokens is { } configured && configured > 0
+            ? configured
+            : LocalEngineOptions.DefaultMaxOutputTokens;
+    }
+
+    private LocalSamplingOptions ResolveSampling(ChatOptions? options)
+    {
+        var defaults = _options.Sampling ?? new LocalSamplingOptions();
+        return new LocalSamplingOptions
+        {
+            Temperature = options?.Temperature ?? defaults.Temperature,
+            TopK = options?.TopK ?? defaults.TopK,
+            TopP = options?.TopP ?? defaults.TopP,
+            Seed = options?.Seed ?? defaults.Seed,
+        };
+    }
+}

src/Agentic/Models/Local/LocalEngineOptions.cs

@@ -0,0 +1,36 @@
+namespace AiDotNet.Agentic.Models.Local;
+
+/// <summary>
+/// Settings for <see cref="LocalEngineChatClient{T}"/>: the reported model id, the default generation length,
+/// and the default sampling behavior (overridable per request via <see cref="ChatOptions"/>).
+/// </summary>
+/// <remarks>
+/// <para><b>For Beginners:</b> These are the local model's defaults. The most useful is
+/// <see cref="MaxOutputTokens"/> (how long a reply may get before the engine stops). Leave
+/// <see cref="Sampling"/> unset for safe, near-greedy behavior, or set it to make replies more creative.
+/// </para>
+/// </remarks>
+public sealed class LocalEngineOptions
+{
+    /// <summary>The default maximum number of tokens generated per reply when none is specified.</summary>
+    public const int DefaultMaxOutputTokens = 256;
+
+    /// <summary>
+    /// Gets or sets the model id reported by <see cref="LocalEngineChatClient{T}.ModelId"/>. <c>null</c> or
+    /// empty falls back to <c>"local"</c>.
+    /// </summary>
+    public string? ModelId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the default maximum number of tokens to generate per reply. <c>null</c> or a non-positive
+    /// value uses <see cref="DefaultMaxOutputTokens"/>. A request's <see cref="ChatOptions.MaxOutputTokens"/>
+    /// overrides this.
+    /// </summary>
+    public int? MaxOutputTokens { get; set; }
+
+    /// <summary>
+    /// Gets or sets the default sampling settings. <c>null</c> uses near-greedy defaults. Per-request
+    /// <see cref="ChatOptions"/> values (temperature, top-k, top-p, seed) override the matching fields.
+    /// </summary>
+    public LocalSamplingOptions? Sampling { get; set; }
+}