From de0d3772d1d74dd88a2ae12afb1e796668f065cc Mon Sep 17 00:00:00 2001 From: franklinic Date: Tue, 9 Jun 2026 13:02:49 -0400 Subject: [PATCH 1/6] =?UTF-8?q?feat(agentic):=20Phase=203=20=E2=80=94=20lo?= =?UTF-8?q?cal=20in-process=20inference=20engine=20(#1544)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flagship local-first differentiator: an IChatClient that runs entirely in-process over AiDotNet's own model — no network, no API key. Same agent code drives it as drives OpenAI/Anthropic. - ICausalLanguageModel: minimal next-token-logits seam the real Transformer (or any model) plugs into; implementations may keep an internal KV-cache. - IGenerationTokenizer (+TokenizerGenerationAdapter over the repo's ITokenizer): encode/decode/EOS seam. - TokenSampler: greedy (temp 0 = argmax) + temperature + top-k + top-p (nucleus), seedable for reproducibility; reads logits via Convert.ToDouble so float/double share one path. - IChatPromptTemplate + ChatMlPromptTemplate (role-tagged, opens the assistant turn). - LocalEngineChatClient: renders prompt -> encodes -> autoregressively samples until EOS or token limit -> decodes; non-streaming + streaming (incremental decode deltas); usage = prompt/generated token counts. Drop-in for agents/supervisor/swarm/memory. (Native tool-calling + structured-output constraints via constrained decoding are a follow-up; tools are ignored this slice.) - 9 tests (green net10.0 + net471) on a deterministic ScriptedCausalModel + WordTokenizer: greedy decode + EOS stop, token-limit -> Length, streaming deltas reconstruct full text, AgentExecutor drop-in, sampler argmax/seed-reproducibility/top-k/top-p, ChatML template. No null-forgiving. Stacked on Phase 2 (#1551) — the integration test shows the local engine driving an AgentExecutor. Part of epic #1544 (Phase 3). Co-Authored-By: Claude Opus 4.8 --- .../Models/Local/ChatMlPromptTemplate.cs | 52 ++++ .../Models/Local/ICausalLanguageModel.cs | 38 +++ .../Models/Local/IChatPromptTemplate.cs | 27 ++ .../Models/Local/IGenerationTokenizer.cs | 39 +++ .../Models/Local/LocalEngineChatClient.cs | 195 +++++++++++++ .../Models/Local/LocalEngineOptions.cs | 36 +++ .../Models/Local/LocalSamplingOptions.cs | 46 ++++ src/Agentic/Models/Local/TokenSampler.cs | 163 +++++++++++ .../Local/TokenizerGenerationAdapter.cs | 57 ++++ .../Agentic/Local/LocalInferenceTests.cs | 257 ++++++++++++++++++ 10 files changed, 910 insertions(+) create mode 100644 src/Agentic/Models/Local/ChatMlPromptTemplate.cs create mode 100644 src/Agentic/Models/Local/ICausalLanguageModel.cs create mode 100644 src/Agentic/Models/Local/IChatPromptTemplate.cs create mode 100644 src/Agentic/Models/Local/IGenerationTokenizer.cs create mode 100644 src/Agentic/Models/Local/LocalEngineChatClient.cs create mode 100644 src/Agentic/Models/Local/LocalEngineOptions.cs create mode 100644 src/Agentic/Models/Local/LocalSamplingOptions.cs create mode 100644 src/Agentic/Models/Local/TokenSampler.cs create mode 100644 src/Agentic/Models/Local/TokenizerGenerationAdapter.cs create mode 100644 tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs diff --git a/src/Agentic/Models/Local/ChatMlPromptTemplate.cs b/src/Agentic/Models/Local/ChatMlPromptTemplate.cs new file mode 100644 index 0000000000..469a190789 --- /dev/null +++ b/src/Agentic/Models/Local/ChatMlPromptTemplate.cs @@ -0,0 +1,52 @@ +using System.Text; +using AiDotNet.Agentic.Models; + +// Disambiguate from the legacy AiDotNet.PromptEngineering.Templates.ChatMessage (global using). +using ChatMessage = AiDotNet.Agentic.Models.ChatMessage; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// A ChatML-style that renders each message as +/// <|role|>\n{text}\n and ends with an open <|assistant|> turn for the model to +/// complete. This is a widely-used, model-agnostic default. +/// +/// +/// For Beginners: This writes the conversation like a script: each line is tagged with who is +/// speaking (<|user|>, <|assistant|>, ...), and the script stops right where the +/// assistant is about to speak — so the model fills in the assistant's reply. +/// +/// +public sealed class ChatMlPromptTemplate : IChatPromptTemplate +{ + /// + /// Thrown when is null. + /// Thrown when is empty. + public string Render(IReadOnlyList messages) + { + Guard.NotNull(messages); + if (messages.Count == 0) + { + throw new ArgumentException("The conversation must contain at least one message.", nameof(messages)); + } + + var builder = new StringBuilder(); + foreach (var message in messages) + { + builder.Append("<|").Append(RoleTag(message.Role)).Append("|>\n"); + builder.Append(message.Text).Append('\n'); + } + + builder.Append("<|assistant|>\n"); + return builder.ToString(); + } + + private static string RoleTag(ChatRole role) => role switch + { + ChatRole.System => "system", + ChatRole.User => "user", + ChatRole.Assistant => "assistant", + ChatRole.Tool => "tool", + _ => "user" + }; +} diff --git a/src/Agentic/Models/Local/ICausalLanguageModel.cs b/src/Agentic/Models/Local/ICausalLanguageModel.cs new file mode 100644 index 0000000000..5c6469a552 --- /dev/null +++ b/src/Agentic/Models/Local/ICausalLanguageModel.cs @@ -0,0 +1,38 @@ +using AiDotNet.LinearAlgebra; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// The minimal contract an in-process language model exposes to the local generation engine: given the +/// tokens seen so far, produce the logits for the next token. This is the seam between AiDotNet's own +/// Transformer (or any other model) and . +/// +/// The tensor element type (e.g., or ). +/// +/// +/// Keeping the contract this small means the generation loop, sampling, and chat templating are written +/// once and tested independently of any particular model, and the real network is wired in behind this +/// interface. An implementation is free to maintain an internal KV-cache keyed on the growing context so +/// repeated calls stay efficient; callers only ever ask for "the next-token logits given this context". +/// +/// For Beginners: A language model, at its heart, answers one question over and over: "given +/// everything so far, how likely is each possible next word-piece?" Those likelihoods (before turning them +/// into probabilities) are called logits. This interface is exactly that one question, so the rest +/// of the engine can focus on choosing the next token and stitching the words back together. +/// +/// +public interface ICausalLanguageModel +{ + /// + /// Gets the size of the model's vocabulary (the length of the logits vector returned by + /// ). + /// + int VocabularySize { get; } + + /// + /// Computes the next-token logits for the given context. + /// + /// The token ids of the context so far (prompt plus any tokens already generated). Must be non-empty. + /// A vector of length giving the unnormalized score for each candidate next token. + Vector NextTokenLogits(IReadOnlyList tokenIds); +} diff --git a/src/Agentic/Models/Local/IChatPromptTemplate.cs b/src/Agentic/Models/Local/IChatPromptTemplate.cs new file mode 100644 index 0000000000..019cbd9119 --- /dev/null +++ b/src/Agentic/Models/Local/IChatPromptTemplate.cs @@ -0,0 +1,27 @@ +using AiDotNet.Agentic.Models; + +// Disambiguate from the legacy AiDotNet.PromptEngineering.Templates.ChatMessage (global using). +using ChatMessage = AiDotNet.Agentic.Models.ChatMessage; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// Renders a chat conversation (a list of ) into the single prompt string a local +/// language model is fed. Different model families expect different role markers, so this is pluggable. +/// +/// +/// For Beginners: Cloud chat APIs accept a list of messages directly, but a raw local model +/// just continues one block of text. This converts the conversation into that block, tagging who said what +/// (system/user/assistant) in the format the model was trained on, and ends with the assistant's turn so the +/// model knows it should reply next. +/// +/// +public interface IChatPromptTemplate +{ + /// + /// Renders the conversation into a prompt string. + /// + /// The conversation so far. Must be non-empty. + /// The prompt text to encode and feed to the model. + string Render(IReadOnlyList messages); +} diff --git a/src/Agentic/Models/Local/IGenerationTokenizer.cs b/src/Agentic/Models/Local/IGenerationTokenizer.cs new file mode 100644 index 0000000000..85c7994ee5 --- /dev/null +++ b/src/Agentic/Models/Local/IGenerationTokenizer.cs @@ -0,0 +1,39 @@ +namespace AiDotNet.Agentic.Models.Local; + +/// +/// The minimal tokenizer contract the local generation engine needs: turn text into token ids, turn token +/// ids back into text, and know which token marks end-of-sequence. +/// +/// +/// +/// This is intentionally narrower than the full so +/// the engine stays decoupled and trivially testable. bridges a +/// real repo tokenizer to this seam. +/// +/// For Beginners: Models don't read text directly — they read numbers (token ids). This turns +/// your prompt into those numbers, turns the model's numbers back into readable text, and tells the engine +/// the special "stop here" token so it knows when the model is done. +/// +/// +public interface IGenerationTokenizer +{ + /// + /// Gets the id of the end-of-sequence token. Generation stops when the model produces it. A negative + /// value means "no EOS token" (generation then stops only at the token limit). + /// + int EosTokenId { get; } + + /// + /// Encodes text into token ids. + /// + /// The text to encode. + /// The token ids. + IReadOnlyList Encode(string text); + + /// + /// Decodes token ids back into text. + /// + /// The token ids to decode. + /// The decoded text. + string Decode(IReadOnlyList tokenIds); +} diff --git a/src/Agentic/Models/Local/LocalEngineChatClient.cs b/src/Agentic/Models/Local/LocalEngineChatClient.cs new file mode 100644 index 0000000000..cd11c24c2e --- /dev/null +++ b/src/Agentic/Models/Local/LocalEngineChatClient.cs @@ -0,0 +1,195 @@ +using System.Runtime.CompilerServices; +using AiDotNet.Agentic.Models; + +// Disambiguate from the legacy AiDotNet.PromptEngineering.Templates.ChatMessage (global using). +using ChatMessage = AiDotNet.Agentic.Models.ChatMessage; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// An that runs entirely in-process over an +/// — no network, no API key, no external service. It renders the conversation to a prompt, encodes it, +/// autoregressively samples tokens until the end-of-sequence token or a length limit, and decodes the +/// result. This is the flagship "local-first" capability: the same agent code that drives OpenAI or +/// Anthropic drives AiDotNet's own model. +/// +/// The tensor element type shared with the model. +/// +/// +/// Because it implements , the local engine is a drop-in for every higher layer +/// (agents, supervisor/swarm, memory). Both non-streaming and streaming generation are supported; streaming +/// decodes incrementally and yields the new text on each step. Native tool-calling and structured-output +/// constraints are not applied by this slice (they require constrained decoding, a follow-up) — supplied +/// tools are ignored and the engine produces plain text. +/// +/// For Beginners: This is your own chatbot brain running on your machine. You hand it the +/// conversation; it writes the reply one word-piece at a time until it decides it's done or hits the length +/// cap. Everything else in this library that talks to a "chat model" can talk to this one instead — so you +/// can build agents with no cloud dependency at all. +/// +/// +public sealed class LocalEngineChatClient : IChatClient +{ + private readonly ICausalLanguageModel _model; + private readonly IGenerationTokenizer _tokenizer; + private readonly IChatPromptTemplate _template; + private readonly LocalEngineOptions _options; + + /// + /// Initializes a new local engine. + /// + /// The in-process language model that produces next-token logits. + /// The tokenizer used to encode prompts and decode generated tokens. + /// The chat prompt template. null uses . + /// Engine settings. null uses defaults. + /// Thrown when or is null. + public LocalEngineChatClient( + ICausalLanguageModel model, + IGenerationTokenizer tokenizer, + IChatPromptTemplate? template = null, + LocalEngineOptions? options = null) + { + Guard.NotNull(model); + Guard.NotNull(tokenizer); + _model = model; + _tokenizer = tokenizer; + _template = template ?? new ChatMlPromptTemplate(); + _options = options ?? new LocalEngineOptions(); + } + + /// + public string ModelId => + _options.ModelId is { } id && id.Trim().Length > 0 ? id : "local"; + + /// + /// Thrown when is null. + /// Thrown when is empty. + public Task GetResponseAsync( + IReadOnlyList messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + var promptIds = BuildPromptIds(messages); + var sampling = ResolveSampling(options); + var sampler = new TokenSampler(sampling.Seed); + var maxTokens = ResolveMaxTokens(options); + + var generated = new List(); + var finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, generated, cancellationToken); + + var text = generated.Count > 0 ? _tokenizer.Decode(generated) : string.Empty; + var usage = new ChatUsage(promptIds.Count, generated.Count); + var response = new ChatResponse(ChatMessage.Assistant(text), finishReason, usage, ModelId); + return Task.FromResult(response); + } + + /// + public async IAsyncEnumerable GetStreamingResponseAsync( + IReadOnlyList messages, + ChatOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await Task.CompletedTask.ConfigureAwait(false); + + var promptIds = BuildPromptIds(messages); + var sampling = ResolveSampling(options); + var sampler = new TokenSampler(sampling.Seed); + var maxTokens = ResolveMaxTokens(options); + + yield return new ChatResponseUpdate(role: ChatRole.Assistant); + + var context = new List(promptIds); + var generated = new List(); + var previousText = string.Empty; + var finishReason = ChatFinishReason.Length; + + for (var step = 0; step < maxTokens; step++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var logits = _model.NextTokenLogits(context); + var next = sampler.Sample(logits, sampling); + if (next == _tokenizer.EosTokenId) + { + finishReason = ChatFinishReason.Stop; + break; + } + + generated.Add(next); + context.Add(next); + + var fullText = _tokenizer.Decode(generated); + if (fullText.Length > previousText.Length) + { + var delta = fullText.Substring(previousText.Length); + previousText = fullText; + yield return ChatResponseUpdate.ForText(delta); + } + } + + yield return ChatResponseUpdate.ForFinish(finishReason, new ChatUsage(promptIds.Count, generated.Count)); + } + + private ChatFinishReason RunGeneration( + IReadOnlyList promptIds, + int maxTokens, + TokenSampler sampler, + LocalSamplingOptions sampling, + List generated, + CancellationToken cancellationToken) + { + var context = new List(promptIds); + for (var step = 0; step < maxTokens; step++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var logits = _model.NextTokenLogits(context); + var next = sampler.Sample(logits, sampling); + if (next == _tokenizer.EosTokenId) + { + return ChatFinishReason.Stop; + } + + generated.Add(next); + context.Add(next); + } + + return ChatFinishReason.Length; + } + + private List BuildPromptIds(IReadOnlyList messages) + { + Guard.NotNull(messages); + if (messages.Count == 0) + { + throw new ArgumentException("The conversation must contain at least one message.", nameof(messages)); + } + + var prompt = _template.Render(messages); + return new List(_tokenizer.Encode(prompt)); + } + + private int ResolveMaxTokens(ChatOptions? options) + { + if (options?.MaxOutputTokens is { } requested && requested > 0) + { + return requested; + } + + return _options.MaxOutputTokens is { } configured && configured > 0 + ? configured + : LocalEngineOptions.DefaultMaxOutputTokens; + } + + private LocalSamplingOptions ResolveSampling(ChatOptions? options) + { + var defaults = _options.Sampling ?? new LocalSamplingOptions(); + return new LocalSamplingOptions + { + Temperature = options?.Temperature ?? defaults.Temperature, + TopK = options?.TopK ?? defaults.TopK, + TopP = options?.TopP ?? defaults.TopP, + Seed = options?.Seed ?? defaults.Seed, + }; + } +} diff --git a/src/Agentic/Models/Local/LocalEngineOptions.cs b/src/Agentic/Models/Local/LocalEngineOptions.cs new file mode 100644 index 0000000000..f9e2e13fe1 --- /dev/null +++ b/src/Agentic/Models/Local/LocalEngineOptions.cs @@ -0,0 +1,36 @@ +namespace AiDotNet.Agentic.Models.Local; + +/// +/// Settings for : the reported model id, the default generation length, +/// and the default sampling behavior (overridable per request via ). +/// +/// +/// For Beginners: These are the local model's defaults. The most useful is +/// (how long a reply may get before the engine stops). Leave +/// unset for safe, near-greedy behavior, or set it to make replies more creative. +/// +/// +public sealed class LocalEngineOptions +{ + /// The default maximum number of tokens generated per reply when none is specified. + public const int DefaultMaxOutputTokens = 256; + + /// + /// Gets or sets the model id reported by . null or + /// empty falls back to "local". + /// + public string? ModelId { get; set; } + + /// + /// Gets or sets the default maximum number of tokens to generate per reply. null or a non-positive + /// value uses . A request's + /// overrides this. + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Gets or sets the default sampling settings. null uses near-greedy defaults. Per-request + /// values (temperature, top-k, top-p, seed) override the matching fields. + /// + public LocalSamplingOptions? Sampling { get; set; } +} diff --git a/src/Agentic/Models/Local/LocalSamplingOptions.cs b/src/Agentic/Models/Local/LocalSamplingOptions.cs new file mode 100644 index 0000000000..bf7898e82a --- /dev/null +++ b/src/Agentic/Models/Local/LocalSamplingOptions.cs @@ -0,0 +1,46 @@ +namespace AiDotNet.Agentic.Models.Local; + +/// +/// Controls how the next token is chosen from the model's logits: temperature, top-k, top-p (nucleus), and +/// an optional seed for reproducibility. +/// +/// +/// +/// All values are nullable with sensible behavior when unset. A temperature of 0 (or less) selects +/// greedy decoding (always the highest-scoring token); higher temperatures increase randomness. Top-k and +/// top-p restrict sampling to the most likely tokens. These mirror the knobs exposed on +/// , which override these per request. +/// +/// For Beginners: After the model says how likely each next word-piece is, these settings decide +/// how to pick one. Low temperature = safe and repetitive; higher = more creative. Top-k ("only consider the +/// best k options") and top-p ("only consider the most likely options that together cover p% of the +/// probability") keep the choice from wandering into unlikely tokens. +/// +/// +public sealed class LocalSamplingOptions +{ + /// + /// Gets or sets the sampling temperature. null uses 1.0; 0 or less selects greedy + /// (deterministic argmax) decoding. + /// + public double? Temperature { get; set; } + + /// + /// Gets or sets the number of highest-probability tokens to consider. null or a non-positive + /// value disables the top-k restriction. + /// + public int? TopK { get; set; } + + /// + /// Gets or sets the nucleus-sampling probability mass (0–1): only the most likely tokens whose + /// cumulative probability reaches this value are considered. null or a value outside (0,1) + /// disables the top-p restriction. + /// + public double? TopP { get; set; } + + /// + /// Gets or sets the random seed for reproducible sampling. null uses a non-deterministic seed. + /// Ignored under greedy decoding (which is already deterministic). + /// + public int? Seed { get; set; } +} diff --git a/src/Agentic/Models/Local/TokenSampler.cs b/src/Agentic/Models/Local/TokenSampler.cs new file mode 100644 index 0000000000..85b818b4aa --- /dev/null +++ b/src/Agentic/Models/Local/TokenSampler.cs @@ -0,0 +1,163 @@ +using AiDotNet.LinearAlgebra; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// Selects the next token id from a logits vector according to : +/// greedy (argmax) when temperature is zero, otherwise temperature-scaled softmax sampling restricted by +/// optional top-k and top-p filters. +/// +/// The logits element type. +/// +/// +/// The sampler owns a single (seeded from when +/// provided), so a fixed seed yields a reproducible token stream. Logits are read through +/// , so the same code path serves and +/// models. +/// +/// For Beginners: This is the "dice roll" step. With temperature 0 it isn't a roll at all — it +/// just takes the single most likely token. Otherwise it turns the scores into probabilities, optionally +/// throws away the unlikely options (top-k / top-p), and then picks one at random in proportion to how +/// likely each is. +/// +/// +public sealed class TokenSampler +{ + private readonly Random _random; + + /// + /// Initializes a new sampler. + /// + /// Optional RNG seed for reproducibility. null uses a non-deterministic seed. + public TokenSampler(int? seed = null) + { + _random = seed is { } value ? new Random(value) : new Random(); + } + + /// + /// Chooses the next token id from the supplied logits. + /// + /// The next-token logits (length = vocabulary size). Must be non-empty. + /// The sampling settings. null uses defaults (temperature 1.0, no filters). + /// The chosen token id (an index into ). + /// Thrown when is null. + /// Thrown when is empty. + public int Sample(Vector logits, LocalSamplingOptions? options = null) + { + Guard.NotNull(logits); + var count = logits.Length; + if (count == 0) + { + throw new ArgumentException("Logits must be non-empty.", nameof(logits)); + } + + var settings = options ?? new LocalSamplingOptions(); + + var scores = new double[count]; + for (var i = 0; i < count; i++) + { + scores[i] = Convert.ToDouble(logits[i]); + } + + var temperature = settings.Temperature ?? 1.0; + if (temperature <= 0) + { + return ArgMax(scores); + } + + for (var i = 0; i < count; i++) + { + scores[i] /= temperature; + } + + // Numerically stable softmax. + var max = double.NegativeInfinity; + for (var i = 0; i < count; i++) + { + if (scores[i] > max) + { + max = scores[i]; + } + } + + var probabilities = new double[count]; + var sum = 0.0; + for (var i = 0; i < count; i++) + { + var p = Math.Exp(scores[i] - max); + probabilities[i] = p; + sum += p; + } + + for (var i = 0; i < count; i++) + { + probabilities[i] /= sum; + } + + // Candidate token ids, ordered by descending probability (so top-k / top-p slice from the front). + var candidates = new List(count); + for (var i = 0; i < count; i++) + { + candidates.Add(i); + } + + candidates.Sort((a, b) => probabilities[b].CompareTo(probabilities[a])); + + if (settings.TopK is { } topK && topK > 0 && topK < candidates.Count) + { + candidates = candidates.GetRange(0, topK); + } + + if (settings.TopP is { } topP && topP > 0 && topP < 1) + { + var kept = new List(candidates.Count); + var cumulative = 0.0; + foreach (var id in candidates) + { + kept.Add(id); + cumulative += probabilities[id]; + if (cumulative >= topP) + { + break; + } + } + + candidates = kept; + } + + var total = 0.0; + foreach (var id in candidates) + { + total += probabilities[id]; + } + + var threshold = _random.NextDouble() * total; + var accumulated = 0.0; + foreach (var id in candidates) + { + accumulated += probabilities[id]; + if (threshold <= accumulated) + { + return id; + } + } + + return candidates[candidates.Count - 1]; + } + + private static int ArgMax(double[] scores) + { + var bestIndex = 0; + var best = scores[0]; + for (var i = 1; i < scores.Length; i++) + { + if (scores[i] > best) + { + best = scores[i]; + bestIndex = i; + } + } + + return bestIndex; + } +} diff --git a/src/Agentic/Models/Local/TokenizerGenerationAdapter.cs b/src/Agentic/Models/Local/TokenizerGenerationAdapter.cs new file mode 100644 index 0000000000..31f764a6a5 --- /dev/null +++ b/src/Agentic/Models/Local/TokenizerGenerationAdapter.cs @@ -0,0 +1,57 @@ +using AiDotNet.Tokenization.Interfaces; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// Bridges a full repo to the engine's minimal +/// seam, so any AiDotNet tokenizer can drive . +/// +/// +/// For Beginners: The library's tokenizers do a lot more than generation needs. This adapter +/// exposes just the three things the generation loop uses (encode, decode, end-of-sequence id), so you can +/// plug a real tokenizer into the local engine without it depending on the larger tokenizer surface. +/// +/// +public sealed class TokenizerGenerationAdapter : IGenerationTokenizer +{ + private readonly ITokenizer _tokenizer; + + /// + /// Initializes a new adapter over the given tokenizer. The EOS id is resolved from the tokenizer's + /// . + /// + /// The tokenizer to wrap. + /// Thrown when is null. + public TokenizerGenerationAdapter(ITokenizer tokenizer) + { + Guard.NotNull(tokenizer); + _tokenizer = tokenizer; + + var eosToken = tokenizer.SpecialTokens.EosToken; + if (eosToken is null || eosToken.Trim().Length == 0) + { + EosTokenId = -1; + return; + } + + var ids = tokenizer.ConvertTokensToIds(new List { eosToken }); + EosTokenId = ids.Count > 0 ? ids[0] : -1; + } + + /// + public int EosTokenId { get; } + + /// + public IReadOnlyList Encode(string text) + { + Guard.NotNull(text); + return _tokenizer.Encode(text).TokenIds; + } + + /// + public string Decode(IReadOnlyList tokenIds) + { + Guard.NotNull(tokenIds); + return _tokenizer.Decode(new List(tokenIds), skipSpecialTokens: true); + } +} diff --git a/tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs new file mode 100644 index 0000000000..2be45338ef --- /dev/null +++ b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AiDotNet.Agentic.Agents; +using AiDotNet.Agentic.Models; +using AiDotNet.Agentic.Models.Local; +using AiDotNet.LinearAlgebra; +using Xunit; + +namespace AiDotNetTests.UnitTests.Agentic.Local +{ + public class LocalInferenceTests + { + // ---- TokenSampler ---- + + [Fact(Timeout = 60000)] + public async Task Sampler_Greedy_PicksArgMax() + { + var sampler = new TokenSampler(); + var logits = new Vector(new[] { 0.1, 0.9, 0.3, 0.2 }); + + var id = sampler.Sample(logits, new LocalSamplingOptions { Temperature = 0.0 }); + + Assert.Equal(1, id); + await Task.CompletedTask; + } + + [Fact(Timeout = 60000)] + public async Task Sampler_SameSeed_IsReproducible() + { + var logits = new Vector(new[] { 1.0, 1.0, 1.0, 1.0 }); + var options = new LocalSamplingOptions { Temperature = 1.0 }; + + var a = new TokenSampler(seed: 123); + var b = new TokenSampler(seed: 123); + + var seqA = Enumerable.Range(0, 20).Select(_ => a.Sample(logits, options)).ToList(); + var seqB = Enumerable.Range(0, 20).Select(_ => b.Sample(logits, options)).ToList(); + + Assert.Equal(seqA, seqB); + } + + [Fact(Timeout = 60000)] + public async Task Sampler_TopK1_AlwaysSelectsTopToken() + { + var sampler = new TokenSampler(seed: 7); + var logits = new Vector(new[] { 0.2, 5.0, 0.1 }); // index 1 dominates + var options = new LocalSamplingOptions { Temperature = 1.0, TopK = 1 }; + + for (var i = 0; i < 25; i++) + { + Assert.Equal(1, sampler.Sample(logits, options)); + } + + await Task.CompletedTask; + } + + [Fact(Timeout = 60000)] + public async Task Sampler_TinyTopP_RestrictsToMostLikely() + { + var sampler = new TokenSampler(seed: 7); + var logits = new Vector(new[] { 0.1, 5.0, 0.2 }); + // A tiny nucleus keeps only the single most-likely token. + var options = new LocalSamplingOptions { Temperature = 1.0, TopP = 0.01 }; + + for (var i = 0; i < 25; i++) + { + Assert.Equal(1, sampler.Sample(logits, options)); + } + + await Task.CompletedTask; + } + + // ---- ChatMlPromptTemplate ---- + + [Fact(Timeout = 60000)] + public async Task Template_RendersRoles_AndOpensAssistantTurn() + { + var template = new ChatMlPromptTemplate(); + var prompt = template.Render(new[] + { + ChatMessage.System("be brief"), + ChatMessage.User("hi"), + }); + + Assert.Contains("<|system|>\nbe brief\n", prompt); + Assert.Contains("<|user|>\nhi\n", prompt); + Assert.EndsWith("<|assistant|>\n", prompt); + await Task.CompletedTask; + } + + // ---- LocalEngineChatClient ---- + + [Fact(Timeout = 60000)] + public async Task Engine_GreedyGeneration_DecodesExpectedText_AndStopsOnEos() + { + var tokenizer = new WordTokenizer("hello", "world"); + // Emit "hello" (1), "world" (2), then EOS (0). + var model = new ScriptedCausalModel(tokenizer.VocabularySize, new[] { 1, 2, 0 }); + var engine = new LocalEngineChatClient(model, tokenizer); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("greet") }, + new ChatOptions { Temperature = 0.0 }); + + Assert.Equal("hello world", response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + Assert.NotNull(response.Usage); + Assert.Equal(2, response.Usage.OutputTokens); + Assert.Equal("local", response.ModelId); + } + + [Fact(Timeout = 60000)] + public async Task Engine_HitsTokenLimit_ReportsLength() + { + var tokenizer = new WordTokenizer("again"); + // Always emits the non-EOS token "again" (1), never stopping on its own. + var model = new ScriptedCausalModel(tokenizer.VocabularySize, new[] { 1 }, repeatLast: true); + var engine = new LocalEngineChatClient(model, tokenizer); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("go") }, + new ChatOptions { Temperature = 0.0, MaxOutputTokens = 3 }); + + Assert.Equal(ChatFinishReason.Length, response.FinishReason); + Assert.Equal(3, response.Usage.OutputTokens); + } + + [Fact(Timeout = 60000)] + public async Task Engine_Streaming_DeltasReconstructFullText() + { + var tokenizer = new WordTokenizer("alpha", "beta", "gamma"); + var model = new ScriptedCausalModel(tokenizer.VocabularySize, new[] { 1, 2, 3, 0 }); + var engine = new LocalEngineChatClient(model, tokenizer); + + var text = string.Empty; + ChatFinishReason? finish = null; + await foreach (var update in engine.GetStreamingResponseAsync(new[] { ChatMessage.User("x") }, + new ChatOptions { Temperature = 0.0 })) + { + if (update.TextDelta is { } delta) + { + text += delta; + } + + if (update.FinishReason is { } reason) + { + finish = reason; + } + } + + Assert.Equal("alpha beta gamma", text); + Assert.Equal(ChatFinishReason.Stop, finish); + } + + [Fact(Timeout = 60000)] + public async Task Engine_IsDropInForAgentExecutor() + { + var tokenizer = new WordTokenizer("done"); + var model = new ScriptedCausalModel(tokenizer.VocabularySize, new[] { 1, 0 }); + var engine = new LocalEngineChatClient(model, tokenizer); + + // The local engine drives a standard agent with no code changes. + var agent = new AgentExecutor(engine); + var result = await agent.RunAsync("anything"); + + Assert.True(result.Completed); + Assert.Equal("done", result.FinalText); + } + + // ---- Deterministic test doubles ---- + + // Emits a predetermined token sequence as one-hot logits, ignoring context. When the sequence is + // exhausted it emits EOS (id 0), or repeats the last token when repeatLast is set. + private sealed class ScriptedCausalModel : ICausalLanguageModel + { + private readonly int[] _sequence; + private readonly bool _repeatLast; + private int _index; + + public ScriptedCausalModel(int vocabularySize, int[] sequence, bool repeatLast = false) + { + VocabularySize = vocabularySize; + _sequence = sequence; + _repeatLast = repeatLast; + } + + public int VocabularySize { get; } + + public Vector NextTokenLogits(IReadOnlyList tokenIds) + { + int id; + if (_index < _sequence.Length) + { + id = _sequence[_index]; + } + else + { + id = _repeatLast ? _sequence[_sequence.Length - 1] : 0; + } + + _index++; + var logits = new double[VocabularySize]; + logits[id] = 1.0; + return new Vector(logits); + } + } + + // Minimal whitespace tokenizer: id 0 is EOS, words get ids 1.. in order. Unknown words map to id 1. + private sealed class WordTokenizer : IGenerationTokenizer + { + private readonly Dictionary _wordToId = new(StringComparer.Ordinal); + private readonly Dictionary _idToWord = new(); + + public WordTokenizer(params string[] words) + { + _idToWord[0] = string.Empty; // EOS decodes to nothing + var nextId = 1; + foreach (var word in words) + { + _wordToId[word] = nextId; + _idToWord[nextId] = word; + nextId++; + } + + VocabularySize = nextId; + } + + public int VocabularySize { get; } + + public int EosTokenId => 0; + + public IReadOnlyList Encode(string text) + { + var ids = new List(); + foreach (var word in text.Split(new[] { ' ', '\n', '\r', '\t' }, StringSplitOptions.RemoveEmptyEntries)) + { + ids.Add(_wordToId.TryGetValue(word, out var id) ? id : 1); + } + + if (ids.Count == 0) + { + ids.Add(1); + } + + return ids; + } + + public string Decode(IReadOnlyList tokenIds) + { + var words = tokenIds + .Select(id => _idToWord.TryGetValue(id, out var word) ? word : string.Empty) + .Where(w => w.Length > 0); + return string.Join(" ", words); + } + } + } +} From bb3dd4a332454236d7748ce9f7572a6c3d49cf24 Mon Sep 17 00:00:00 2001 From: franklinic Date: Tue, 9 Jun 2026 14:42:14 -0400 Subject: [PATCH 2/6] =?UTF-8?q?feat(agentic):=20Phase=203=20=E2=80=94=20wi?= =?UTF-8?q?re=20real=20AiDotNet=20networks=20into=20local=20inference=20(#?= =?UTF-8?q?1544)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NeuralNetworkCausalLanguageModel: adapts a trained NeuralNetworkBase LM (Mamba/GLA/Transformer-LM-head) to ICausalLanguageModel. Encodes context as the one-hot [1,seq,vocab] tensor these models expect, runs a forward pass (ResetState first so recurrent models start fresh), and extracts the final position's logits (handles rank-2 and rank-3 outputs). Full context re-fed each step; KV-cached fast path is a follow-up. - 3 integration tests over a real tiny untrained MambaLanguageModel (greedy, deterministic): adapter logit width + no-NaN, end-to-end generation honoring the token budget, streaming termination. Pinned to CpuEngine + non-parallel collection (documented GPU-autodetect mitigation). - Fixed Engine_IsDropInForAgentExecutor: it relied on the engine's stochastic default sampling while asserting a fixed output; now configures greedy (Temperature 0) so the assertion is deterministic. Root-caused the flake to default-temperature sampling, not a library/GPU issue. - Local suite green + stable (verified repeated runs) on net10.0 + net471. No null-forgiving. Part of epic #1544 (Phase 3). Co-Authored-By: Claude Opus 4.8 --- .../Local/NeuralNetworkCausalLanguageModel.cs | 126 +++++++++++++ .../Agentic/Local/LocalInferenceTests.cs | 9 +- .../Local/RealModelLocalInferenceTests.cs | 177 ++++++++++++++++++ 3 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs create mode 100644 tests/AiDotNet.Tests/UnitTests/Agentic/Local/RealModelLocalInferenceTests.cs diff --git a/src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs b/src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs new file mode 100644 index 0000000000..09c7eb047a --- /dev/null +++ b/src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs @@ -0,0 +1,126 @@ +using AiDotNet.LinearAlgebra; +using AiDotNet.NeuralNetworks; +using AiDotNet.Tensors; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// Adapts a trained AiDotNet language model (e.g., +/// MambaLanguageModel, GLALanguageModel, or a Transformer LM head) to the +/// seam, so can run real, +/// fully in-process generation over the library's own networks. +/// +/// The tensor element type of the network (e.g., or ). +/// +/// +/// AiDotNet's language models accept a one-hot input tensor of shape [1, sequence, vocab] and return +/// logits of the same leading shape. This adapter encodes the context as that one-hot tensor, runs a forward +/// pass, and returns the final position's logits. It re-feeds the full context each step (no KV-cache yet) +/// and calls before each pass so recurrent models (Mamba/GLA) +/// start fresh — correct, if not yet optimal. A KV-cached fast path is a planned follow-up. +/// +/// For Beginners: This is the bridge that lets the local chat engine talk to a real AiDotNet +/// network. It turns the running list of tokens into the exact tensor shape the network expects, asks the +/// network for its prediction, and hands back the scores for the next token — which the engine then samples +/// from. The result: a chatbot powered entirely by an in-house model, no external service. +/// +/// +public sealed class NeuralNetworkCausalLanguageModel : ICausalLanguageModel +{ + private static readonly T One = (T)Convert.ChangeType(1.0, typeof(T)); + + private readonly NeuralNetworkBase _network; + private readonly int _maxContextTokens; + + /// + /// Initializes a new adapter over a network language model. + /// + /// The trained network whose Predict maps one-hot tokens to vocab logits. + /// The model's vocabulary size (the width of the one-hot input and logits). + /// + /// The maximum number of most-recent tokens to feed the network (typically its max sequence length). + /// null or non-positive feeds the entire context. + /// + /// Thrown when is null. + /// Thrown when is not positive. + public NeuralNetworkCausalLanguageModel(NeuralNetworkBase network, int vocabularySize, int? maxContextTokens = null) + { + Guard.NotNull(network); + Guard.Positive(vocabularySize); + _network = network; + VocabularySize = vocabularySize; + _maxContextTokens = maxContextTokens is { } configured && configured > 0 ? configured : int.MaxValue; + } + + /// + public int VocabularySize { get; } + + /// + /// Thrown when is null. + /// Thrown when is empty. + /// Thrown when a token id is outside the vocabulary. + public Vector NextTokenLogits(IReadOnlyList tokenIds) + { + Guard.NotNull(tokenIds); + if (tokenIds.Count == 0) + { + throw new ArgumentException("The context must contain at least one token.", nameof(tokenIds)); + } + + var start = _maxContextTokens == int.MaxValue ? 0 : Math.Max(0, tokenIds.Count - _maxContextTokens); + var sequenceLength = tokenIds.Count - start; + + var input = new Tensor(new[] { 1, sequenceLength, VocabularySize }); + for (var position = 0; position < sequenceLength; position++) + { + var id = tokenIds[start + position]; + if (id < 0 || id >= VocabularySize) + { + throw new ArgumentOutOfRangeException( + nameof(tokenIds), id, $"Token id is outside the vocabulary [0, {VocabularySize})."); + } + + input[new[] { 0, position, id }] = One; + } + + _network.ResetState(); + var output = _network.Predict(input); + return ExtractLastPositionLogits(output); + } + + private Vector ExtractLastPositionLogits(Tensor output) + { + var shape = output.Shape.ToArray(); + var vocab = shape[shape.Length - 1]; + if (vocab != VocabularySize) + { + throw new InvalidOperationException( + $"Model produced logits of width {vocab}, but vocabulary size is {VocabularySize}."); + } + + var logits = new T[vocab]; + if (shape.Length == 3) + { + var lastPosition = shape[1] - 1; + for (var v = 0; v < vocab; v++) + { + logits[v] = output[new[] { 0, lastPosition, v }]; + } + } + else if (shape.Length == 2) + { + var lastPosition = shape[0] - 1; + for (var v = 0; v < vocab; v++) + { + logits[v] = output[new[] { lastPosition, v }]; + } + } + else + { + throw new InvalidOperationException( + $"Expected logits of rank 2 or 3, but the model returned rank {shape.Length}."); + } + + return new Vector(logits); + } +} diff --git a/tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs index 2be45338ef..acf3181e1c 100644 --- a/tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs +++ b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/LocalInferenceTests.cs @@ -155,9 +155,14 @@ public async Task Engine_Streaming_DeltasReconstructFullText() [Fact(Timeout = 60000)] public async Task Engine_IsDropInForAgentExecutor() { - var tokenizer = new WordTokenizer("done"); + var tokenizer = new WordTokenizer("done", "later"); var model = new ScriptedCausalModel(tokenizer.VocabularySize, new[] { 1, 0 }); - var engine = new LocalEngineChatClient(model, tokenizer); + // AgentExecutor does not specify a temperature, so make the engine greedy by default for a + // deterministic assertion (otherwise it samples stochastically at the default temperature). + var engine = new LocalEngineChatClient(model, tokenizer, options: new LocalEngineOptions + { + Sampling = new LocalSamplingOptions { Temperature = 0.0 } + }); // The local engine drives a standard agent with no code changes. var agent = new AgentExecutor(engine); diff --git a/tests/AiDotNet.Tests/UnitTests/Agentic/Local/RealModelLocalInferenceTests.cs b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/RealModelLocalInferenceTests.cs new file mode 100644 index 0000000000..13bff53d39 --- /dev/null +++ b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/RealModelLocalInferenceTests.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AiDotNet.Agentic.Models; +using AiDotNet.Agentic.Models.Local; +using AiDotNet.Enums; +using AiDotNet.NeuralNetworks; +using AiDotNet.Tensors.Engines; +using Xunit; + +namespace AiDotNetTests.UnitTests.Agentic.Local +{ + /// + /// Marks the real-network inference tests as a non-parallel collection. Running a real neural-network + /// forward pass mutates process-wide tensor-engine state (engine selection / LazyTensorScope), which can + /// disturb tensor ops in tests running concurrently; isolating this collection avoids that. + /// + [CollectionDefinition("RealModelInference", DisableParallelization = true)] + public sealed class RealModelInferenceCollection + { + } + + /// + /// Integration tests that run the local generation engine over a real (tiny, untrained) AiDotNet + /// . They validate the full in-process pipeline — one-hot encoding, + /// forward pass, last-position logit extraction, sampling, autoregressive loop — not output quality. + /// All use greedy decoding (temperature 0) for deterministic assertions. + /// + [Collection("RealModelInference")] + public class RealModelLocalInferenceTests + { + private const int Vocab = 20; + private const int MaxSeq = 16; + + // A real network forward dispatches through AiDotNetEngine.Current, which a module initializer may + // auto-set to a GPU engine; some GPU backends return zeros for small shapes. Pin a CPU engine for the + // duration of each test (documented mitigation) and restore it afterwards. + private static async Task PinCpuAsync(Func body) + { + var priorEngine = AiDotNetEngine.Current; + AiDotNetEngine.Current = new CpuEngine(); + try + { + await body(); + } + finally + { + AiDotNetEngine.Current = priorEngine; + } + } + + private static MambaLanguageModel BuildTinyModel() + { + var architecture = new NeuralNetworkArchitecture( + InputType.OneDimensional, + NeuralNetworkTaskType.TextGeneration, + inputSize: Vocab, + outputSize: Vocab); + + return new MambaLanguageModel( + architecture, + vocabSize: Vocab, + modelDimension: 16, + numLayers: 1, + stateDimension: 4, + maxSeqLength: MaxSeq); + } + + [Fact(Timeout = 120000)] + public async Task Adapter_NextTokenLogits_ReturnsVocabWidthLogits() + { + await PinCpuAsync(() => + { + var lm = new NeuralNetworkCausalLanguageModel(BuildTinyModel(), Vocab, maxContextTokens: MaxSeq); + + var logits = lm.NextTokenLogits(new[] { 1, 2, 3 }); + + Assert.Equal(Vocab, logits.Length); + for (var i = 0; i < logits.Length; i++) + { + Assert.False(double.IsNaN(logits[i])); + } + + return Task.CompletedTask; + }); + } + + [Fact(Timeout = 120000)] + public async Task Engine_GeneratesEndToEnd_OverRealNetwork() + { + await PinCpuAsync(async () => + { + var lm = new NeuralNetworkCausalLanguageModel(BuildTinyModel(), Vocab, maxContextTokens: MaxSeq); + var tokenizer = new BoundedTokenizer(Vocab); + var engine = new LocalEngineChatClient(lm, tokenizer); + + var response = await engine.GetResponseAsync( + new[] { ChatMessage.User("hello there") }, + new ChatOptions { Temperature = 0.0, MaxOutputTokens = 3 }); + + // The untrained model produces arbitrary tokens, but the full pipeline must run cleanly and + // honor the token budget. + Assert.Equal(ChatFinishReason.Length, response.FinishReason); + Assert.NotNull(response.Usage); + Assert.Equal(3, response.Usage.OutputTokens); + Assert.NotNull(response.Text); + }); + } + + [Fact(Timeout = 120000)] + public async Task Engine_StreamingOverRealNetwork_TerminatesWithFinish() + { + await PinCpuAsync(async () => + { + var lm = new NeuralNetworkCausalLanguageModel(BuildTinyModel(), Vocab, maxContextTokens: MaxSeq); + var engine = new LocalEngineChatClient(lm, new BoundedTokenizer(Vocab)); + + ChatFinishReason? finish = null; + var updates = 0; + await foreach (var update in engine.GetStreamingResponseAsync( + new[] { ChatMessage.User("hi") }, + new ChatOptions { Temperature = 0.0, MaxOutputTokens = 2 })) + { + updates++; + if (update.FinishReason is { } reason) + { + finish = reason; + } + } + + Assert.True(updates > 0); + Assert.Equal(ChatFinishReason.Length, finish); + }); + } + + // A whitespace tokenizer whose ids stay within the model's vocabulary. EOS is disabled (-1) so + // greedy generation runs deterministically to the token budget regardless of the untrained model. + private sealed class BoundedTokenizer : IGenerationTokenizer + { + private readonly int _vocab; + + public BoundedTokenizer(int vocab) + { + _vocab = vocab; + } + + public int EosTokenId => -1; + + public IReadOnlyList Encode(string text) + { + var ids = new List(); + foreach (var word in text.Split(new[] { ' ', '\n', '\r', '\t' }, StringSplitOptions.RemoveEmptyEntries)) + { + // Deterministic, bounded id from a stable char-sum (string.GetHashCode is randomized per process). + var sum = 0; + foreach (var ch in word) + { + sum = (sum + ch) & 0x7fffffff; + } + + ids.Add((sum % (_vocab - 1)) + 1); + } + + if (ids.Count == 0) + { + ids.Add(1); + } + + return ids; + } + + public string Decode(IReadOnlyList tokenIds) => + string.Join(" ", tokenIds.Select(id => "t" + id)); + } + } +} From fadae2092a153665c35790ccb65c3e295a503ec2 Mon Sep 17 00:00:00 2001 From: franklinic Date: Tue, 9 Jun 2026 21:40:05 -0400 Subject: [PATCH 3/6] =?UTF-8?q?feat(agentic):=20Phase=203=20=E2=80=94=20co?= =?UTF-8?q?nstrained=20decoding=20+=20stop=20sequences=20(#1544)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Native LOCAL guaranteed-structure generation — enforced at the logits, not merely prompted (a capability cloud models can't guarantee): - ITokenConstraint seam: AllowedNextTokens(generated) returns null (free) / a set (restrict) / empty (terminal -> stop). TokenSampler.Sample gains an allowed-token mask honored by both greedy (argmax within allowed) and stochastic (candidates restricted before top-k/top-p). - AllowedTokenSetConstraint (fixed allow-list) + FiniteStateTokenConstraint (finite-state grammar over token ids: start set + per-token transitions; terminal states stop) — the general mechanism a JSON-schema/regular grammar compiles to. - LocalEngineOptions.Constraint wires it into LocalEngineChatClient generation (non-streaming + streaming). - Stop sequences: ChatOptions.StopSequences now halt generation and trim the output at the earliest match (both paths). - 5 tests (green, stable, net10.0 + net471): allowed-set greedy/stochastic masking, FSA forces an exact JSON-shaped sequence despite a model biased to 'garbage', allow-list restricts whole output, stop-sequence halts+trims. No null-forgiving. Part of epic #1544 (Phase 3). JSON-schema->token-grammar compiler (to drive structured output / tool-calling off this framework) is the next follow-up. Co-Authored-By: Claude Opus 4.8 --- .../Models/Local/AllowedTokenSetConstraint.cs | 34 +++ .../Local/FiniteStateTokenConstraint.cs | 61 +++++ src/Agentic/Models/Local/ITokenConstraint.cs | 33 +++ .../Models/Local/LocalEngineChatClient.cs | 106 ++++++++- .../Models/Local/LocalEngineOptions.cs | 7 + src/Agentic/Models/Local/TokenSampler.cs | 65 +++++- .../Agentic/Local/ConstrainedDecodingTests.cs | 218 ++++++++++++++++++ 7 files changed, 500 insertions(+), 24 deletions(-) create mode 100644 src/Agentic/Models/Local/AllowedTokenSetConstraint.cs create mode 100644 src/Agentic/Models/Local/FiniteStateTokenConstraint.cs create mode 100644 src/Agentic/Models/Local/ITokenConstraint.cs create mode 100644 tests/AiDotNet.Tests/UnitTests/Agentic/Local/ConstrainedDecodingTests.cs diff --git a/src/Agentic/Models/Local/AllowedTokenSetConstraint.cs b/src/Agentic/Models/Local/AllowedTokenSetConstraint.cs new file mode 100644 index 0000000000..60bf258eb2 --- /dev/null +++ b/src/Agentic/Models/Local/AllowedTokenSetConstraint.cs @@ -0,0 +1,34 @@ +namespace AiDotNet.Agentic.Models.Local; + +/// +/// A constraint that always restricts generation to a fixed set of token ids, regardless of context — for +/// example, "only emit digit tokens" or "only emit tokens from this closed label vocabulary". +/// +/// +/// For Beginners: The simplest gate: a permanent allow-list. Whatever has been generated, only +/// these tokens are ever permitted next. Handy when the whole answer must come from a small known set. +/// +/// +public sealed class AllowedTokenSetConstraint : ITokenConstraint +{ + private readonly HashSet _allowed; + + /// + /// Initializes a new constraint permitting only the given token ids. + /// + /// The permitted token ids. Must be non-null and non-empty. + /// Thrown when is null. + /// Thrown when is empty. + public AllowedTokenSetConstraint(IEnumerable allowedTokenIds) + { + Guard.NotNull(allowedTokenIds); + _allowed = new HashSet(allowedTokenIds); + if (_allowed.Count == 0) + { + throw new ArgumentException("At least one allowed token id is required.", nameof(allowedTokenIds)); + } + } + + /// + public IReadOnlyCollection? AllowedNextTokens(IReadOnlyList generatedTokenIds) => _allowed; +} diff --git a/src/Agentic/Models/Local/FiniteStateTokenConstraint.cs b/src/Agentic/Models/Local/FiniteStateTokenConstraint.cs new file mode 100644 index 0000000000..93c6289372 --- /dev/null +++ b/src/Agentic/Models/Local/FiniteStateTokenConstraint.cs @@ -0,0 +1,61 @@ +namespace AiDotNet.Agentic.Models.Local; + +/// +/// A constraint defined by a finite-state grammar over token ids: the set of allowed next tokens depends on +/// the most recently generated token (the current state). This expresses exact sequences, branching choices, +/// and loops — the general mechanism a JSON-schema or regular grammar compiles down to. +/// +/// +/// +/// State is the last generated token. Before any token is generated, the start set applies. From a +/// state with no outgoing transitions, the empty set is returned, which tells the engine to stop — so a +/// chain of single-token transitions forces an exact output, and terminal states end generation cleanly. +/// +/// For Beginners: Picture a flowchart where each box says "from here, you may only go to these +/// tokens next". Generation walks the flowchart; it can never step off it. Give each box exactly one exit and +/// you force a precise output; give it several and you allow choices. Boxes with no exit end the answer. +/// +/// +public sealed class FiniteStateTokenConstraint : ITokenConstraint +{ + private readonly IReadOnlyCollection _start; + private readonly IReadOnlyDictionary> _transitions; + + /// + /// Initializes a new finite-state constraint. + /// + /// The tokens allowed as the very first generated token. Must be non-empty. + /// + /// A map from a just-generated token id to the tokens allowed after it. A token absent from the map (or + /// mapped to an empty set) is a terminal state at which generation stops. + /// + /// Thrown when or is null. + /// Thrown when is empty. + public FiniteStateTokenConstraint( + IEnumerable start, + IReadOnlyDictionary> transitions) + { + Guard.NotNull(start); + Guard.NotNull(transitions); + _start = new List(start); + if (_start.Count == 0) + { + throw new ArgumentException("At least one start token id is required.", nameof(start)); + } + + _transitions = transitions; + } + + /// + public IReadOnlyCollection? AllowedNextTokens(IReadOnlyList generatedTokenIds) + { + Guard.NotNull(generatedTokenIds); + if (generatedTokenIds.Count == 0) + { + return _start; + } + + var lastToken = generatedTokenIds[generatedTokenIds.Count - 1]; + return _transitions.TryGetValue(lastToken, out var allowed) ? allowed : Array.Empty(); + } +} diff --git a/src/Agentic/Models/Local/ITokenConstraint.cs b/src/Agentic/Models/Local/ITokenConstraint.cs new file mode 100644 index 0000000000..f235cdcf54 --- /dev/null +++ b/src/Agentic/Models/Local/ITokenConstraint.cs @@ -0,0 +1,33 @@ +namespace AiDotNet.Agentic.Models.Local; + +/// +/// Restricts which tokens the local engine may generate next, enabling constrained decoding: the +/// model can only emit tokens the constraint permits, so the output is guaranteed to satisfy a structure +/// (a fixed vocabulary, a grammar, a JSON shape) rather than merely being asked to in the prompt. +/// +/// +/// +/// At each step the engine calls with the tokens generated so far. Returning +/// null means "no restriction this step"; returning a set restricts sampling to exactly those token +/// ids; returning an empty set tells the engine to stop (nothing valid can follow). This is the foundation +/// for local structured output and tool-calling — capabilities cloud models approximate with prompting but +/// cannot guarantee, and which a local engine can enforce at the logits because it controls decoding. +/// +/// For Beginners: Normally a model can pick any next word-piece. A constraint is a gate that +/// only lets through the choices that keep the answer valid — for example, only digits when you want a +/// number, or only tokens that continue well-formed JSON. Because the gate is applied while generating, the +/// result is always valid by construction, not just "usually" valid. +/// +/// +public interface ITokenConstraint +{ + /// + /// Returns the token ids permitted as the next token given what has been generated so far. + /// + /// The tokens generated since the prompt (excludes the prompt itself). + /// + /// null for no restriction; a non-empty set to restrict sampling to those ids; an empty set to + /// signal that generation should stop (no valid continuation exists). + /// + IReadOnlyCollection? AllowedNextTokens(IReadOnlyList generatedTokenIds); +} diff --git a/src/Agentic/Models/Local/LocalEngineChatClient.cs b/src/Agentic/Models/Local/LocalEngineChatClient.cs index cd11c24c2e..5f30d2cf94 100644 --- a/src/Agentic/Models/Local/LocalEngineChatClient.cs +++ b/src/Agentic/Models/Local/LocalEngineChatClient.cs @@ -1,3 +1,4 @@ +using System.Linq; using System.Runtime.CompilerServices; using AiDotNet.Agentic.Models; @@ -74,10 +75,12 @@ public Task GetResponseAsync( var sampler = new TokenSampler(sampling.Seed); var maxTokens = ResolveMaxTokens(options); + var stopSequences = ResolveStopSequences(options); var generated = new List(); - var finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, generated, cancellationToken); + var finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken); var text = generated.Count > 0 ? _tokenizer.Decode(generated) : string.Empty; + text = TrimAtStopSequence(text, stopSequences); var usage = new ChatUsage(promptIds.Count, generated.Count); var response = new ChatResponse(ChatMessage.Assistant(text), finishReason, usage, ModelId); return Task.FromResult(response); @@ -98,6 +101,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( yield return new ChatResponseUpdate(role: ChatRole.Assistant); + var stopSequences = ResolveStopSequences(options); var context = new List(promptIds); var generated = new List(); var previousText = string.Empty; @@ -107,24 +111,29 @@ public async IAsyncEnumerable GetStreamingResponseAsync( { cancellationToken.ThrowIfCancellationRequested(); - var logits = _model.NextTokenLogits(context); - var next = sampler.Sample(logits, sampling); - if (next == _tokenizer.EosTokenId) + var next = NextToken(context, generated, sampler, sampling); + if (next is null || next.Value == _tokenizer.EosTokenId) { finishReason = ChatFinishReason.Stop; break; } - generated.Add(next); - context.Add(next); + generated.Add(next.Value); + context.Add(next.Value); - var fullText = _tokenizer.Decode(generated); + var fullText = TrimAtStopSequence(_tokenizer.Decode(generated), stopSequences); if (fullText.Length > previousText.Length) { var delta = fullText.Substring(previousText.Length); previousText = fullText; yield return ChatResponseUpdate.ForText(delta); } + + if (stopSequences is not null && ContainsStopSequence(_tokenizer.Decode(generated), stopSequences)) + { + finishReason = ChatFinishReason.Stop; + break; + } } yield return ChatResponseUpdate.ForFinish(finishReason, new ChatUsage(promptIds.Count, generated.Count)); @@ -135,6 +144,7 @@ private ChatFinishReason RunGeneration( int maxTokens, TokenSampler sampler, LocalSamplingOptions sampling, + IReadOnlyList? stopSequences, List generated, CancellationToken cancellationToken) { @@ -143,20 +153,92 @@ private ChatFinishReason RunGeneration( { cancellationToken.ThrowIfCancellationRequested(); - var logits = _model.NextTokenLogits(context); - var next = sampler.Sample(logits, sampling); - if (next == _tokenizer.EosTokenId) + var next = NextToken(context, generated, sampler, sampling); + if (next is null || next.Value == _tokenizer.EosTokenId) { + // null = the constraint reached a terminal state; either way generation is complete. return ChatFinishReason.Stop; } - generated.Add(next); - context.Add(next); + generated.Add(next.Value); + context.Add(next.Value); + + if (stopSequences is not null && ContainsStopSequence(_tokenizer.Decode(generated), stopSequences)) + { + return ChatFinishReason.Stop; + } } return ChatFinishReason.Length; } + // Computes the next token id honoring the configured constraint, or null when the constraint signals a + // terminal state (no valid continuation exists). + private int? NextToken( + List context, + List generated, + TokenSampler sampler, + LocalSamplingOptions sampling) + { + IReadOnlyCollection? allowed = null; + if (_options.Constraint is { } constraint) + { + allowed = constraint.AllowedNextTokens(generated); + if (allowed is not null && allowed.Count == 0) + { + return null; + } + } + + var logits = _model.NextTokenLogits(context); + return sampler.Sample(logits, sampling, allowed); + } + + private static IReadOnlyList? ResolveStopSequences(ChatOptions? options) + { + var stops = options?.StopSequences; + if (stops is null || stops.Count == 0) + { + return null; + } + + var filtered = stops.Where(s => s is not null && s.Length > 0).ToList(); + return filtered.Count > 0 ? filtered : null; + } + + private static bool ContainsStopSequence(string text, IReadOnlyList stopSequences) + { + foreach (var stop in stopSequences) + { + if (text.IndexOf(stop, StringComparison.Ordinal) >= 0) + { + return true; + } + } + + return false; + } + + private static string TrimAtStopSequence(string text, IReadOnlyList? stopSequences) + { + if (stopSequences is null) + { + return text; + } + + var earliest = -1; + foreach (var stop in stopSequences) + { + var index = text.IndexOf(stop, StringComparison.Ordinal); + if (index >= 0 && (earliest < 0 || index < earliest)) + { + earliest = index; + } + } + + return earliest >= 0 ? text.Substring(0, earliest) : text; + } + private List BuildPromptIds(IReadOnlyList messages) { Guard.NotNull(messages); diff --git a/src/Agentic/Models/Local/LocalEngineOptions.cs b/src/Agentic/Models/Local/LocalEngineOptions.cs index f9e2e13fe1..d096ca5234 100644 --- a/src/Agentic/Models/Local/LocalEngineOptions.cs +++ b/src/Agentic/Models/Local/LocalEngineOptions.cs @@ -33,4 +33,11 @@ public sealed class LocalEngineOptions /// values (temperature, top-k, top-p, seed) override the matching fields. /// public LocalSamplingOptions? Sampling { get; set; } + + /// + /// Gets or sets a token constraint applied during generation (constrained decoding). null means + /// unconstrained generation. Use this to guarantee structured output (a grammar, JSON shape, or a closed + /// vocabulary) at the logits rather than relying on prompting. + /// + public ITokenConstraint? Constraint { get; set; } } diff --git a/src/Agentic/Models/Local/TokenSampler.cs b/src/Agentic/Models/Local/TokenSampler.cs index 85b818b4aa..5fd7174b88 100644 --- a/src/Agentic/Models/Local/TokenSampler.cs +++ b/src/Agentic/Models/Local/TokenSampler.cs @@ -35,14 +35,19 @@ public TokenSampler(int? seed = null) } /// - /// Chooses the next token id from the supplied logits. + /// Chooses the next token id from the supplied logits, optionally restricted to an allowed set + /// (constrained decoding). /// /// The next-token logits (length = vocabulary size). Must be non-empty. /// The sampling settings. null uses defaults (temperature 1.0, no filters). + /// + /// When non-null, sampling is restricted to these token ids (others are excluded). Must be non-empty when + /// provided. null means all tokens are eligible. + /// /// The chosen token id (an index into ). /// Thrown when is null. - /// Thrown when is empty. - public int Sample(Vector logits, LocalSamplingOptions? options = null) + /// Thrown when or is empty. + public int Sample(Vector logits, LocalSamplingOptions? options = null, IReadOnlyCollection? allowedTokenIds = null) { Guard.NotNull(logits); var count = logits.Length; @@ -51,6 +56,11 @@ public int Sample(Vector logits, LocalSamplingOptions? options = null) throw new ArgumentException("Logits must be non-empty.", nameof(logits)); } + if (allowedTokenIds is not null && allowedTokenIds.Count == 0) + { + throw new ArgumentException("The allowed token set must be non-empty when provided.", nameof(allowedTokenIds)); + } + var settings = options ?? new LocalSamplingOptions(); var scores = new double[count]; @@ -59,10 +69,12 @@ public int Sample(Vector logits, LocalSamplingOptions? options = null) scores[i] = Convert.ToDouble(logits[i]); } + var allowed = BuildAllowedMask(count, allowedTokenIds); + var temperature = settings.Temperature ?? 1.0; if (temperature <= 0) { - return ArgMax(scores); + return ArgMax(scores, allowed); } for (var i = 0; i < count; i++) @@ -94,11 +106,15 @@ public int Sample(Vector logits, LocalSamplingOptions? options = null) probabilities[i] /= sum; } - // Candidate token ids, ordered by descending probability (so top-k / top-p slice from the front). + // Candidate token ids (allowed only), ordered by descending probability so top-k / top-p slice + // from the front. var candidates = new List(count); for (var i = 0; i < count; i++) { - candidates.Add(i); + if (allowed is null || allowed[i]) + { + candidates.Add(i); + } } candidates.Sort((a, b) => probabilities[b].CompareTo(probabilities[a])); @@ -145,19 +161,44 @@ public int Sample(Vector logits, LocalSamplingOptions? options = null) return candidates[candidates.Count - 1]; } - private static int ArgMax(double[] scores) + private static bool[]? BuildAllowedMask(int count, IReadOnlyCollection? allowedTokenIds) + { + if (allowedTokenIds is null) + { + return null; + } + + var mask = new bool[count]; + foreach (var id in allowedTokenIds) + { + if (id >= 0 && id < count) + { + mask[id] = true; + } + } + + return mask; + } + + private static int ArgMax(double[] scores, bool[]? allowed) { - var bestIndex = 0; - var best = scores[0]; - for (var i = 1; i < scores.Length; i++) + var bestIndex = -1; + var best = double.NegativeInfinity; + for (var i = 0; i < scores.Length; i++) { - if (scores[i] > best) + if (allowed is not null && !allowed[i]) + { + continue; + } + + if (bestIndex < 0 || scores[i] > best) { best = scores[i]; bestIndex = i; } } - return bestIndex; + // If the allowed set referenced only out-of-range ids, fall back to the global argmax. + return bestIndex >= 0 ? bestIndex : 0; } } diff --git a/tests/AiDotNet.Tests/UnitTests/Agentic/Local/ConstrainedDecodingTests.cs b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/ConstrainedDecodingTests.cs new file mode 100644 index 0000000000..331158ed72 --- /dev/null +++ b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/ConstrainedDecodingTests.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AiDotNet.Agentic.Models; +using AiDotNet.Agentic.Models.Local; +using AiDotNet.LinearAlgebra; +using Xunit; + +namespace AiDotNetTests.UnitTests.Agentic.Local +{ + public class ConstrainedDecodingTests + { + // ---- TokenSampler with an allowed set ---- + + [Fact(Timeout = 60000)] + public async Task Sampler_Greedy_RespectsAllowedSet() + { + var sampler = new TokenSampler(); + // Index 3 has the highest logit, but it is excluded by the allowed set. + var logits = new Vector(new[] { 0.1, 0.5, 0.2, 0.9 }); + + var id = sampler.Sample(logits, new LocalSamplingOptions { Temperature = 0.0 }, new[] { 0, 1, 2 }); + + Assert.Equal(1, id); // best among {0,1,2} + await Task.CompletedTask; + } + + [Fact(Timeout = 60000)] + public async Task Sampler_Sampling_NeverEmitsDisallowedToken() + { + var sampler = new TokenSampler(seed: 5); + var logits = new Vector(new[] { 5.0, 0.1, 5.0, 0.1 }); + var allowed = new[] { 1, 3 }; + + for (var i = 0; i < 40; i++) + { + var id = sampler.Sample(logits, new LocalSamplingOptions { Temperature = 1.0 }, allowed); + Assert.Contains(id, allowed); + } + + await Task.CompletedTask; + } + + // ---- FiniteStateTokenConstraint forces structure ---- + + [Fact(Timeout = 60000)] + public async Task FiniteStateConstraint_ForcesExactSequence_RegardlessOfModel() + { + var tokenizer = new MapTokenizer( + eos: 0, + ("{", 1), ("\"ok\"", 2), ("}", 3), ("garbage", 4)); + + // A model that, left unconstrained, always wants token 4 ("garbage"). + var model = new BiasedModel(tokenizer.VocabularySize, preferredToken: 4); + + // Grammar: start -> 1 ; 1 -> 2 ; 2 -> 3 ; 3 -> terminal (stop). + var constraint = new FiniteStateTokenConstraint( + start: new[] { 1 }, + transitions: new Dictionary> + { + [1] = new[] { 2 }, + [2] = new[] { 3 }, + [3] = Array.Empty(), + }); + + var engine = new LocalEngineChatClient(model, tokenizer, options: new LocalEngineOptions + { + Sampling = new LocalSamplingOptions { Temperature = 0.0 }, + Constraint = constraint, + }); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("emit json") }); + + // Despite the model preferring "garbage", the constraint forced the exact grammar output. + Assert.Equal("{\"ok\"}", response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + } + + [Fact(Timeout = 60000)] + public async Task AllowedTokenSetConstraint_RestrictsWholeOutput() + { + var tokenizer = new MapTokenizer(eos: 0, ("a", 1), ("b", 2), ("c", 3)); + var model = new BiasedModel(tokenizer.VocabularySize, preferredToken: 3); // wants "c" + var constraint = new AllowedTokenSetConstraint(new[] { 1, 2 }); // only a or b allowed + + var engine = new LocalEngineChatClient(model, tokenizer, options: new LocalEngineOptions + { + Sampling = new LocalSamplingOptions { Temperature = 0.0 }, + Constraint = constraint, + MaxOutputTokens = 4, + }); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("x") }); + + Assert.DoesNotContain("c", response.Text); + Assert.True(response.Text.Replace(" ", "").Length > 0); + } + + // ---- Stop sequences ---- + + [Fact(Timeout = 60000)] + public async Task StopSequence_HaltsGeneration_AndTrimsOutput() + { + var tokenizer = new MapTokenizer(eos: 0, ("hello", 1), ("STOP", 2), ("world", 3)); + // Model emits: hello, STOP, world, ... (never EOS on its own within the budget). + var model = new ScriptedModel(tokenizer.VocabularySize, new[] { 1, 2, 3, 3, 3 }); + + var engine = new LocalEngineChatClient(model, tokenizer, options: new LocalEngineOptions + { + Sampling = new LocalSamplingOptions { Temperature = 0.0 }, + }); + + var response = await engine.GetResponseAsync( + new[] { ChatMessage.User("go") }, + new ChatOptions { StopSequences = new[] { "STOP" }, MaxOutputTokens = 10 }); + + // Output is trimmed at the stop sequence and generation halts. + Assert.Equal("hello", response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + } + + // ---- Test doubles ---- + + // Always biases toward one preferred token (one-hot logits), ignoring context. + private sealed class BiasedModel : ICausalLanguageModel + { + private readonly int _preferred; + + public BiasedModel(int vocabularySize, int preferredToken) + { + VocabularySize = vocabularySize; + _preferred = preferredToken; + } + + public int VocabularySize { get; } + + public Vector NextTokenLogits(IReadOnlyList tokenIds) + { + var logits = new double[VocabularySize]; + for (var i = 0; i < VocabularySize; i++) + { + logits[i] = i == _preferred ? 10.0 : 0.0; + } + + return new Vector(logits); + } + } + + // Emits a fixed sequence (one-hot), then EOS. + private sealed class ScriptedModel : ICausalLanguageModel + { + private readonly int[] _sequence; + private int _index; + + public ScriptedModel(int vocabularySize, int[] sequence) + { + VocabularySize = vocabularySize; + _sequence = sequence; + } + + public int VocabularySize { get; } + + public Vector NextTokenLogits(IReadOnlyList tokenIds) + { + var id = _index < _sequence.Length ? _sequence[_index] : 0; + _index++; + var logits = new double[VocabularySize]; + logits[id] = 1.0; + return new Vector(logits); + } + } + + // Tokenizer over an explicit (text, id) map; id 0 reserved for EOS. + private sealed class MapTokenizer : IGenerationTokenizer + { + private readonly Dictionary _wordToId = new(StringComparer.Ordinal); + private readonly Dictionary _idToWord = new(); + + public MapTokenizer(int eos, params (string Word, int Id)[] entries) + { + EosTokenId = eos; + _idToWord[eos] = string.Empty; + var max = eos; + foreach (var (word, id) in entries) + { + _wordToId[word] = id; + _idToWord[id] = word; + max = Math.Max(max, id); + } + + VocabularySize = max + 1; + } + + public int VocabularySize { get; } + + public int EosTokenId { get; } + + public IReadOnlyList Encode(string text) + { + var ids = text + .Split(new[] { ' ', '\n', '\r', '\t' }, StringSplitOptions.RemoveEmptyEntries) + .Select(w => _wordToId.TryGetValue(w, out var id) ? id : EosTokenId) + .ToList(); + if (ids.Count == 0) + { + ids.Add(EosTokenId == 0 && _wordToId.Count > 0 ? _wordToId.Values.First() : 0); + } + + return ids; + } + + // Concatenate token texts directly (no separators) so grammar output like {"ok"} is exact. + public string Decode(IReadOnlyList tokenIds) => + string.Concat(tokenIds.Select(id => _idToWord.TryGetValue(id, out var w) ? w : string.Empty)); + } + } +} From ee6d46cdfc7c755fc871af12861451cbf768a5a6 Mon Sep 17 00:00:00 2001 From: franklinic Date: Tue, 9 Jun 2026 22:01:44 -0400 Subject: [PATCH 4/6] =?UTF-8?q?feat(agentic):=20Phase=203=20=E2=80=94=20be?= =?UTF-8?q?am-search=20decoding=20(#1544)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LocalEngineOptions.BeamWidth (>1) switches non-streaming GetResponseAsync to beam search: explores N hypotheses in parallel, expands each by its top tokens (honoring the ITokenConstraint and stop sequences per beam), prunes to the top-N by length-normalized log-probability, and returns the best completion. Deterministic; streaming stays token-by-token. - LogSoftmax (with allowed-token masking → -inf) + length-normalized scoring + Beam bookkeeping. - 3 tests (green net10.0 + net471): greedy takes the locally-best token ("A"); beam width 2 discovers the globally better "BC" path the same model would miss greedily; beam respects a finite-state constraint ("AB"). No null-forgiving (removed an introduced allowed! via proper narrowing). Part of epic #1544 (Phase 3). Co-Authored-By: Claude Opus 4.8 --- .../Models/Local/LocalEngineChatClient.cs | 166 +++++++++++++++++- .../Models/Local/LocalEngineOptions.cs | 9 + .../Agentic/Local/BeamSearchTests.cs | 142 +++++++++++++++ 3 files changed, 312 insertions(+), 5 deletions(-) create mode 100644 tests/AiDotNet.Tests/UnitTests/Agentic/Local/BeamSearchTests.cs diff --git a/src/Agentic/Models/Local/LocalEngineChatClient.cs b/src/Agentic/Models/Local/LocalEngineChatClient.cs index 5f30d2cf94..489e2008cc 100644 --- a/src/Agentic/Models/Local/LocalEngineChatClient.cs +++ b/src/Agentic/Models/Local/LocalEngineChatClient.cs @@ -71,13 +71,23 @@ public Task GetResponseAsync( CancellationToken cancellationToken = default) { var promptIds = BuildPromptIds(messages); - var sampling = ResolveSampling(options); - var sampler = new TokenSampler(sampling.Seed); var maxTokens = ResolveMaxTokens(options); - var stopSequences = ResolveStopSequences(options); - var generated = new List(); - var finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken); + + List generated; + ChatFinishReason finishReason; + var beamWidth = _options.BeamWidth is { } width && width > 1 ? width : 1; + if (beamWidth > 1) + { + (generated, finishReason) = RunBeamSearch(promptIds, maxTokens, beamWidth, stopSequences, cancellationToken); + } + else + { + var sampling = ResolveSampling(options); + var sampler = new TokenSampler(sampling.Seed); + generated = new List(); + finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken); + } var text = generated.Count > 0 ? _tokenizer.Decode(generated) : string.Empty; text = TrimAtStopSequence(text, stopSequences); @@ -194,6 +204,152 @@ private ChatFinishReason RunGeneration( return sampler.Sample(logits, sampling, allowed); } + // Beam search: explore `beamWidth` hypotheses in parallel, expanding each by its top tokens (honoring + // the constraint), and keep the highest-scoring (length-normalized log-probability) beams. Returns the + // best completion. Deterministic. + private (List Tokens, ChatFinishReason Finish) RunBeamSearch( + IReadOnlyList promptIds, + int maxTokens, + int beamWidth, + IReadOnlyList? stopSequences, + CancellationToken cancellationToken) + { + var beams = new List { new(new List(), 0.0, finished: false) }; + + for (var step = 0; step < maxTokens; step++) + { + cancellationToken.ThrowIfCancellationRequested(); + if (beams.All(b => b.Finished)) + { + break; + } + + var expanded = new List(); + foreach (var beam in beams) + { + if (beam.Finished) + { + expanded.Add(beam); + continue; + } + + IReadOnlyCollection? allowed = null; + if (_options.Constraint is { } constraint) + { + allowed = constraint.AllowedNextTokens(beam.Tokens); + if (allowed is not null && allowed.Count == 0) + { + expanded.Add(new Beam(beam.Tokens, beam.LogProbability, finished: true)); + continue; + } + } + + var context = new List(promptIds.Count + beam.Tokens.Count); + context.AddRange(promptIds); + context.AddRange(beam.Tokens); + + var logProbabilities = LogSoftmax(_model.NextTokenLogits(context), allowed); + foreach (var (tokenId, logProbability) in TopTokens(logProbabilities, beamWidth)) + { + var finished = tokenId == _tokenizer.EosTokenId; + var tokens = new List(beam.Tokens); + if (!finished) + { + tokens.Add(tokenId); + } + + if (!finished && stopSequences is not null && ContainsStopSequence(_tokenizer.Decode(tokens), stopSequences)) + { + finished = true; + } + + expanded.Add(new Beam(tokens, beam.LogProbability + logProbability, finished)); + } + } + + beams = expanded + .OrderByDescending(NormalizedScore) + .Take(beamWidth) + .ToList(); + } + + var best = beams.OrderByDescending(NormalizedScore).First(); + return (best.Tokens, best.Finished ? ChatFinishReason.Stop : ChatFinishReason.Length); + } + + private static double NormalizedScore(Beam beam) => beam.LogProbability / Math.Max(1, beam.Tokens.Count); + + private static double[] LogSoftmax(Vector logits, IReadOnlyCollection? allowed) + { + var count = logits.Length; + bool[]? allowedMask = null; + if (allowed is not null) + { + allowedMask = new bool[count]; + foreach (var id in allowed) + { + if (id >= 0 && id < count) + { + allowedMask[id] = true; + } + } + } + + var scores = new double[count]; + var max = double.NegativeInfinity; + for (var i = 0; i < count; i++) + { + var value = allowedMask is null || allowedMask[i] ? Convert.ToDouble(logits[i]) : double.NegativeInfinity; + scores[i] = value; + if (value > max) + { + max = value; + } + } + + var sumExp = 0.0; + for (var i = 0; i < count; i++) + { + if (!double.IsNegativeInfinity(scores[i])) + { + sumExp += Math.Exp(scores[i] - max); + } + } + + var logSumExp = max + Math.Log(sumExp); + for (var i = 0; i < count; i++) + { + scores[i] = double.IsNegativeInfinity(scores[i]) ? double.NegativeInfinity : scores[i] - logSumExp; + } + + return scores; + } + + private static IEnumerable<(int TokenId, double LogProbability)> TopTokens(double[] logProbabilities, int k) + { + return Enumerable.Range(0, logProbabilities.Length) + .Where(i => !double.IsNegativeInfinity(logProbabilities[i])) + .OrderByDescending(i => logProbabilities[i]) + .Take(k) + .Select(i => (i, logProbabilities[i])); + } + + private sealed class Beam + { + public Beam(List tokens, double logProbability, bool finished) + { + Tokens = tokens; + LogProbability = logProbability; + Finished = finished; + } + + public List Tokens { get; } + + public double LogProbability { get; } + + public bool Finished { get; } + } + private static IReadOnlyList? ResolveStopSequences(ChatOptions? options) { var stops = options?.StopSequences; diff --git a/src/Agentic/Models/Local/LocalEngineOptions.cs b/src/Agentic/Models/Local/LocalEngineOptions.cs index d096ca5234..ecb1560cfb 100644 --- a/src/Agentic/Models/Local/LocalEngineOptions.cs +++ b/src/Agentic/Models/Local/LocalEngineOptions.cs @@ -40,4 +40,13 @@ public sealed class LocalEngineOptions /// vocabulary) at the logits rather than relying on prompting. /// public ITokenConstraint? Constraint { get; set; } + + /// + /// Gets or sets the beam width for beam-search decoding. null or a value <= 1 uses ordinary + /// token-by-token sampling/greedy decoding. A value > 1 explores that many hypotheses in parallel and + /// returns the highest-scoring (length-normalized) completion — deterministic, and typically higher + /// quality than greedy for short structured outputs. Beam search applies to non-streaming + /// ; streaming always decodes token-by-token. + /// + public int? BeamWidth { get; set; } } diff --git a/tests/AiDotNet.Tests/UnitTests/Agentic/Local/BeamSearchTests.cs b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/BeamSearchTests.cs new file mode 100644 index 0000000000..b4a8ee444a --- /dev/null +++ b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/BeamSearchTests.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AiDotNet.Agentic.Models; +using AiDotNet.Agentic.Models.Local; +using AiDotNet.LinearAlgebra; +using Xunit; + +namespace AiDotNetTests.UnitTests.Agentic.Local +{ + public class BeamSearchTests + { + // Vocabulary: 0=EOS, 1=A, 2=B, 3=C. The model's next-token logits depend on the last token, set up + // so that the locally-best first token (A) leads to a worse overall sequence than B -> C. + private static TransitionModel BuildModel() => new( + vocabularySize: 4, + transitions: new Dictionary + { + [0] = new[] { 0.1, 2.0, 1.9, 0.1 }, // start (prompt's last token): A slightly beats B + [1] = new[] { 2.0, 0.1, 0.1, 0.1 }, // after A: EOS (A is a short, lower-total path) + [2] = new[] { 0.1, 0.1, 0.1, 5.0 }, // after B: C is almost certain + [3] = new[] { 5.0, 0.1, 0.1, 0.1 }, // after C: EOS + }); + + private static MapTokenizer BuildTokenizer() => + new(eos: 0, ("A", 1), ("B", 2), ("C", 3)); + + [Fact(Timeout = 60000)] + public async Task Greedy_TakesLocallyBestToken() + { + var engine = new LocalEngineChatClient(BuildModel(), BuildTokenizer(), options: new LocalEngineOptions + { + Sampling = new LocalSamplingOptions { Temperature = 0.0 }, // greedy, beam width 1 + }); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("go") }); + + Assert.Equal("A", response.Text); + } + + [Fact(Timeout = 60000)] + public async Task BeamSearch_FindsHigherProbabilitySequence() + { + var engine = new LocalEngineChatClient(BuildModel(), BuildTokenizer(), options: new LocalEngineOptions + { + BeamWidth = 2, + }); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("go") }); + + // Beam search explores B as well as A and discovers the globally better B -> C completion. + Assert.Equal("BC", response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + } + + [Fact(Timeout = 60000)] + public async Task BeamSearch_RespectsConstraint() + { + // Force the grammar A -> B -> (terminal) even though the model would prefer other paths. + var constraint = new FiniteStateTokenConstraint( + start: new[] { 1 }, + transitions: new Dictionary> + { + [1] = new[] { 2 }, + [2] = Array.Empty(), + }); + + var engine = new LocalEngineChatClient(BuildModel(), BuildTokenizer(), options: new LocalEngineOptions + { + BeamWidth = 3, + Constraint = constraint, + }); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("go") }); + + Assert.Equal("AB", response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + } + + // ---- Test doubles ---- + + private sealed class TransitionModel : ICausalLanguageModel + { + private readonly Dictionary _transitions; + private readonly double[] _default; + + public TransitionModel(int vocabularySize, Dictionary transitions) + { + VocabularySize = vocabularySize; + _transitions = transitions; + _default = new double[vocabularySize]; + _default[0] = 5.0; // unknown state -> prefer EOS + } + + public int VocabularySize { get; } + + public Vector NextTokenLogits(IReadOnlyList tokenIds) + { + var last = tokenIds[tokenIds.Count - 1]; + var logits = _transitions.TryGetValue(last, out var row) ? row : _default; + return new Vector((double[])logits.Clone()); + } + } + + private sealed class MapTokenizer : IGenerationTokenizer + { + private readonly Dictionary _wordToId = new(StringComparer.Ordinal); + private readonly Dictionary _idToWord = new(); + + public MapTokenizer(int eos, params (string Word, int Id)[] entries) + { + EosTokenId = eos; + _idToWord[eos] = string.Empty; + foreach (var (word, id) in entries) + { + _wordToId[word] = id; + _idToWord[id] = word; + } + } + + public int EosTokenId { get; } + + public IReadOnlyList Encode(string text) + { + var ids = text + .Split(new[] { ' ', '\n', '\r', '\t' }, StringSplitOptions.RemoveEmptyEntries) + .Select(w => _wordToId.TryGetValue(w, out var id) ? id : EosTokenId) + .ToList(); + if (ids.Count == 0) + { + ids.Add(EosTokenId); + } + + return ids; + } + + public string Decode(IReadOnlyList tokenIds) => + string.Concat(tokenIds.Select(id => _idToWord.TryGetValue(id, out var w) ? w : string.Empty)); + } + } +} From a3cd8c06eda783bdfcf15c542733b66a4c665057 Mon Sep 17 00:00:00 2001 From: franklinic Date: Tue, 9 Jun 2026 22:13:44 -0400 Subject: [PATCH 5/6] =?UTF-8?q?feat(agentic):=20Phase=203=20=E2=80=94=20in?= =?UTF-8?q?cremental=20(KV-cache)=20decoding=20seam=20+=20engine=20fast=20?= =?UTF-8?q?path=20(#1544)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - IIncrementalCausalLanguageModel : ICausalLanguageModel adds ResetCache / StartSequence(prompt) / AppendToken(id) — the contract a KV-cached model implements to advance one token at a time instead of re-feeding the full O(n^2) context. - LocalEngineChatClient auto-detects the interface and takes the incremental fast path (prime once, then feed single tokens), falling back to full re-feed otherwise. Constraint + stop-sequence + sampler logic shared via a single PickToken helper across both paths. - 2 tests (green net10.0 + net471): the engine drives the incremental path correctly (ResetCache once, StartSequence with the prompt, AppendToken per generated token) and produces output IDENTICAL to the full-refeed path. The remaining half — a real K/V cache inside Mamba/GLA/attention forward — is a model-layer change (the network Predict API has no incremental entry point yet); this lands the engine-side support + verification it plugs into. Part of epic #1544 (Phase 3). Co-Authored-By: Claude Opus 4.8 --- .../Local/IIncrementalCausalLanguageModel.cs | 46 +++++ .../Models/Local/LocalEngineChatClient.cs | 61 ++++++- .../Agentic/Local/IncrementalDecodingTests.cs | 159 ++++++++++++++++++ 3 files changed, 260 insertions(+), 6 deletions(-) create mode 100644 src/Agentic/Models/Local/IIncrementalCausalLanguageModel.cs create mode 100644 tests/AiDotNet.Tests/UnitTests/Agentic/Local/IncrementalDecodingTests.cs diff --git a/src/Agentic/Models/Local/IIncrementalCausalLanguageModel.cs b/src/Agentic/Models/Local/IIncrementalCausalLanguageModel.cs new file mode 100644 index 0000000000..8b26445554 --- /dev/null +++ b/src/Agentic/Models/Local/IIncrementalCausalLanguageModel.cs @@ -0,0 +1,46 @@ +using AiDotNet.LinearAlgebra; + +namespace AiDotNet.Agentic.Models.Local; + +/// +/// An that supports incremental (KV-cached) decoding: it processes the +/// prompt once, caches the per-position state, and then advances one token at a time without recomputing the +/// whole sequence. This is the fast path for autoregressive generation. +/// +/// The tensor element type. +/// +/// +/// The base re-feeds the full context each step, which +/// is correct but O(n²) over a generation. A model that maintains a key/value cache implements this interface +/// so can drive it incrementally: primes +/// the cache with the prompt and returns the first next-token logits, then each +/// feeds a single new token and returns the following logits. clears state between +/// independent generations. +/// +/// For Beginners: Without caching, predicting each new word re-reads the entire conversation — +/// slower and slower as it grows. With caching, the model remembers the work it already did and only looks at +/// the one new word each time. This interface is how a model advertises "I can do the fast, remember-as-you-go +/// version"; the engine uses it automatically when available and falls back to the simple way otherwise. +/// +/// +public interface IIncrementalCausalLanguageModel : ICausalLanguageModel +{ + /// + /// Clears any cached decoding state, so the next begins fresh. + /// + void ResetCache(); + + /// + /// Processes the prompt, populating the cache, and returns the logits for the first generated token. + /// + /// The prompt token ids. Must be non-empty. + /// The next-token logits following the prompt (length = ). + Vector StartSequence(IReadOnlyList promptTokenIds); + + /// + /// Appends a single token to the cached context and returns the logits for the token after it. + /// + /// The token id to append (typically the one just generated). + /// The next-token logits (length = ). + Vector AppendToken(int tokenId); +} diff --git a/src/Agentic/Models/Local/LocalEngineChatClient.cs b/src/Agentic/Models/Local/LocalEngineChatClient.cs index 489e2008cc..820f116a96 100644 --- a/src/Agentic/Models/Local/LocalEngineChatClient.cs +++ b/src/Agentic/Models/Local/LocalEngineChatClient.cs @@ -86,7 +86,9 @@ public Task GetResponseAsync( var sampling = ResolveSampling(options); var sampler = new TokenSampler(sampling.Seed); generated = new List(); - finishReason = RunGeneration(promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken); + finishReason = _model is IIncrementalCausalLanguageModel incremental + ? RunGenerationIncremental(incremental, promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken) + : RunGeneration(promptIds, maxTokens, sampler, sampling, stopSequences, generated, cancellationToken); } var text = generated.Count > 0 ? _tokenizer.Decode(generated) : string.Empty; @@ -182,11 +184,51 @@ private ChatFinishReason RunGeneration( return ChatFinishReason.Length; } - // Computes the next token id honoring the configured constraint, or null when the constraint signals a - // terminal state (no valid continuation exists). - private int? NextToken( - List context, + // Incremental (KV-cached) generation: prime with the prompt, then advance one token at a time. + private ChatFinishReason RunGenerationIncremental( + IIncrementalCausalLanguageModel model, + IReadOnlyList promptIds, + int maxTokens, + TokenSampler sampler, + LocalSamplingOptions sampling, + IReadOnlyList? stopSequences, List generated, + CancellationToken cancellationToken) + { + model.ResetCache(); + var logits = model.StartSequence(promptIds); + + for (var step = 0; step < maxTokens; step++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var next = PickToken(generated, logits, sampler, sampling); + if (next is null || next.Value == _tokenizer.EosTokenId) + { + return ChatFinishReason.Stop; + } + + generated.Add(next.Value); + + if (stopSequences is not null && ContainsStopSequence(_tokenizer.Decode(generated), stopSequences)) + { + return ChatFinishReason.Stop; + } + + if (step < maxTokens - 1) + { + logits = model.AppendToken(next.Value); + } + } + + return ChatFinishReason.Length; + } + + // Computes the next token id from precomputed logits, honoring the configured constraint, or null when + // the constraint signals a terminal state (no valid continuation exists). + private int? PickToken( + List generated, + Vector logits, TokenSampler sampler, LocalSamplingOptions sampling) { @@ -200,10 +242,17 @@ private ChatFinishReason RunGeneration( } } - var logits = _model.NextTokenLogits(context); return sampler.Sample(logits, sampling, allowed); } + // Computes the next token id by re-feeding the full context (fallback when the model has no KV-cache). + private int? NextToken( + List context, + List generated, + TokenSampler sampler, + LocalSamplingOptions sampling) => + PickToken(generated, _model.NextTokenLogits(context), sampler, sampling); + // Beam search: explore `beamWidth` hypotheses in parallel, expanding each by its top tokens (honoring // the constraint), and keep the highest-scoring (length-normalized log-probability) beams. Returns the // best completion. Deterministic. diff --git a/tests/AiDotNet.Tests/UnitTests/Agentic/Local/IncrementalDecodingTests.cs b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/IncrementalDecodingTests.cs new file mode 100644 index 0000000000..0b797359f1 --- /dev/null +++ b/tests/AiDotNet.Tests/UnitTests/Agentic/Local/IncrementalDecodingTests.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AiDotNet.Agentic.Models; +using AiDotNet.Agentic.Models.Local; +using AiDotNet.LinearAlgebra; +using Xunit; + +namespace AiDotNetTests.UnitTests.Agentic.Local +{ + public class IncrementalDecodingTests + { + // Deterministic next-token rule keyed on the last token: 0(EOS)->1, 1->2, 2->3, 3->0(EOS). + private static readonly Dictionary Transition = new() + { + [0] = 1, + [1] = 2, + [2] = 3, + [3] = 0, + }; + + private const int Vocab = 4; + + private static MapTokenizer BuildTokenizer() => new(eos: 0, ("A", 1), ("B", 2), ("C", 3)); + + private static double[] LogitsFor(int lastToken) + { + var next = Transition.TryGetValue(lastToken, out var n) ? n : 0; + var logits = new double[Vocab]; + logits[next] = 10.0; + return logits; + } + + [Fact(Timeout = 60000)] + public async Task Engine_UsesIncrementalPath_FeedingOneTokenAtATime() + { + var model = new IncrementalProbe(); + var engine = new LocalEngineChatClient(model, BuildTokenizer(), options: new LocalEngineOptions + { + Sampling = new LocalSamplingOptions { Temperature = 0.0 }, + }); + + var response = await engine.GetResponseAsync(new[] { ChatMessage.User("go") }); + + Assert.Equal("ABC", response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + + // The engine took the incremental fast path: reset once, primed once with the prompt, then fed + // each generated token singly via AppendToken. + Assert.Equal(1, model.ResetCalls); + Assert.Equal(1, model.StartCalls); + Assert.Equal(new[] { 1, 2, 3 }, model.AppendedTokens); + } + + [Fact(Timeout = 60000)] + public async Task IncrementalAndFullRefeed_ProduceIdenticalOutput() + { + var tokenizer = BuildTokenizer(); + var sampling = new LocalSamplingOptions { Temperature = 0.0 }; + + var incremental = new LocalEngineChatClient(new IncrementalProbe(), tokenizer, + options: new LocalEngineOptions { Sampling = sampling }); + var fullRefeed = new LocalEngineChatClient(new FullRefeedModel(), tokenizer, + options: new LocalEngineOptions { Sampling = sampling }); + + var a = await incremental.GetResponseAsync(new[] { ChatMessage.User("go") }); + var b = await fullRefeed.GetResponseAsync(new[] { ChatMessage.User("go") }); + + Assert.Equal(b.Text, a.Text); + Assert.Equal("ABC", a.Text); + } + + // ---- Test doubles ---- + + // A KV-cached model: tracks how the engine drives it and advances state by the last token only. + private sealed class IncrementalProbe : IIncrementalCausalLanguageModel + { + private int _lastToken; + + public int VocabularySize => Vocab; + + public int ResetCalls { get; private set; } + + public int StartCalls { get; private set; } + + public List AppendedTokens { get; } = new(); + + public void ResetCache() + { + ResetCalls++; + _lastToken = 0; + } + + public Vector StartSequence(IReadOnlyList promptTokenIds) + { + StartCalls++; + _lastToken = promptTokenIds[promptTokenIds.Count - 1]; + return new Vector(LogitsFor(_lastToken)); + } + + public Vector AppendToken(int tokenId) + { + AppendedTokens.Add(tokenId); + _lastToken = tokenId; + return new Vector(LogitsFor(_lastToken)); + } + + // Full-refeed fallback (unused when the engine takes the incremental path). + public Vector NextTokenLogits(IReadOnlyList tokenIds) => + new(LogitsFor(tokenIds[tokenIds.Count - 1])); + } + + // Same next-token rule, but no KV-cache: forces the engine's full-context path. + private sealed class FullRefeedModel : ICausalLanguageModel + { + public int VocabularySize => Vocab; + + public Vector NextTokenLogits(IReadOnlyList tokenIds) => + new(LogitsFor(tokenIds[tokenIds.Count - 1])); + } + + private sealed class MapTokenizer : IGenerationTokenizer + { + private readonly Dictionary _wordToId = new(StringComparer.Ordinal); + private readonly Dictionary _idToWord = new(); + + public MapTokenizer(int eos, params (string Word, int Id)[] entries) + { + EosTokenId = eos; + _idToWord[eos] = string.Empty; + foreach (var (word, id) in entries) + { + _wordToId[word] = id; + _idToWord[id] = word; + } + } + + public int EosTokenId { get; } + + public IReadOnlyList Encode(string text) + { + var ids = text + .Split(new[] { ' ', '\n', '\r', '\t' }, StringSplitOptions.RemoveEmptyEntries) + .Select(w => _wordToId.TryGetValue(w, out var id) ? id : EosTokenId) + .ToList(); + if (ids.Count == 0) + { + ids.Add(EosTokenId); + } + + return ids; + } + + public string Decode(IReadOnlyList tokenIds) => + string.Concat(tokenIds.Select(id => _idToWord.TryGetValue(id, out var w) ? w : string.Empty)); + } + } +} From 69a827131a947500b8e78b03d0c360542e7b2af4 Mon Sep 17 00:00:00 2001 From: franklinic Date: Tue, 9 Jun 2026 22:17:23 -0400 Subject: [PATCH 6/6] =?UTF-8?q?docs(agentic):=20Phase=203=20=E2=80=94=20do?= =?UTF-8?q?cument=20quantization=20composition=20via=20ModelCompression=20?= =?UTF-8?q?(#1544)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quantization for the local engine is achieved by quantizing the NeuralNetworkBase with the existing ModelCompression stack before wrapping it — the adapter accepts any such network unchanged. No engine-side code (no duplication of ModelCompression). Documented on NeuralNetworkCausalLanguageModel. Co-Authored-By: Claude Opus 4.8 --- .../Models/Local/NeuralNetworkCausalLanguageModel.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs b/src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs index 09c7eb047a..d0fe68a24a 100644 --- a/src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs +++ b/src/Agentic/Models/Local/NeuralNetworkCausalLanguageModel.cs @@ -19,6 +19,12 @@ namespace AiDotNet.Agentic.Models.Local; /// and calls before each pass so recurrent models (Mamba/GLA) /// start fresh — correct, if not yet optimal. A KV-cached fast path is a planned follow-up. /// +/// +/// Quantization composes through this adapter rather than being re-implemented here: quantize the +/// network with the repository's ModelCompression stack first, then wrap the quantized +/// — the adapter accepts any such network unchanged, so a smaller/faster +/// model needs no engine-side code. +/// /// For Beginners: This is the bridge that lets the local chat engine talk to a real AiDotNet /// network. It turns the running list of tokens into the exact tensor shape the network expects, asks the /// network for its prediction, and hands back the scores for the next token — which the engine then samples