Merge remote-tracking branch 'origin/main' into paper/10-ink-bleed

# Conflicts:
#	frontend/taskdeck-web/src/views/PaperStyleGuideView.vue

Author: Chris0Jeky

backend/src/Taskdeck.Api/Extensions/OptionsValidationRegistration.cs

@@ -41,6 +41,7 @@ public static IServiceCollection AddOptionsValidation(
 
         services.RegisterValidatedOptions<WorkerSettings>(configuration, "Workers");
         services.RegisterValidatedOptions<AuditRetentionSettings>(configuration, "AuditRetention");
+        services.RegisterValidatedOptions<EmbeddingBackfillSettings>(configuration, "EmbeddingBackfill");
 
         // ── Settings from CorsRegistration (Cache is used in infrastructure) ─
 

backend/src/Taskdeck.Api/Extensions/WorkerRegistration.cs

@@ -42,11 +42,15 @@ public static IServiceCollection AddTaskdeckWorkers(
         var auditRetentionSettings = configuration.GetSection("AuditRetention").Get<AuditRetentionSettings>() ?? new AuditRetentionSettings();
         services.AddSingleton(auditRetentionSettings);
 
+        var embeddingBackfillSettings = configuration.GetSection("EmbeddingBackfill").Get<EmbeddingBackfillSettings>() ?? new EmbeddingBackfillSettings();
+        services.AddSingleton(embeddingBackfillSettings);
+
         services.AddSingleton<WorkerHeartbeatRegistry>();
         services.AddHostedService<LlmQueueToProposalWorker>();
         services.AddHostedService<ProposalHousekeepingWorker>();
         services.AddHostedService<OutboundWebhookDeliveryWorker>();
         services.AddHostedService<AuditRetentionWorker>();
+        services.AddHostedService<EmbeddingBackfillWorker>();
 
         return services;
     }

backend/src/Taskdeck.Api/Workers/EmbeddingBackfillWorker.cs

@@ -0,0 +1,111 @@
+using Taskdeck.Application.Services;
+using Taskdeck.Api.Telemetry;
+
+namespace Taskdeck.Api.Workers;
+
+/// <summary>
+/// Background worker that drives the embedding backfill service on a
+/// configurable interval. Resumable across restarts (the backfill service
+/// tracks which entities have embeddings). Uses exponential backoff on
+/// consecutive failures to avoid log spam.
+/// </summary>
+public class EmbeddingBackfillWorker : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly EmbeddingBackfillSettings _settings;
+    private readonly WorkerHeartbeatRegistry _heartbeatRegistry;
+    private readonly ILogger<EmbeddingBackfillWorker> _logger;
+
+    public EmbeddingBackfillWorker(
+        IServiceScopeFactory scopeFactory,
+        EmbeddingBackfillSettings settings,
+        WorkerHeartbeatRegistry heartbeatRegistry,
+        ILogger<EmbeddingBackfillWorker> logger)
+    {
+        _scopeFactory = scopeFactory;
+        _settings = settings;
+        _heartbeatRegistry = heartbeatRegistry;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        if (!_settings.Enabled)
+        {
+            _logger.LogInformation("EmbeddingBackfillWorker is disabled via configuration");
+            return;
+        }
+
+        _logger.LogInformation(
+            "EmbeddingBackfillWorker starting (batch={BatchSize}, interval={IntervalSeconds}s)",
+            _settings.BatchSize,
+            _settings.PollIntervalSeconds);
+
+        int consecutiveErrors = 0;
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            _heartbeatRegistry.ReportHeartbeat(nameof(EmbeddingBackfillWorker));
+
+            try
+            {
+                using var scope = _scopeFactory.CreateScope();
+                var backfillService = scope.ServiceProvider
+                    .GetRequiredService<IEmbeddingBackfillService>();
+
+                var result = await backfillService.ProcessBatchAsync(
+                    _settings.BatchSize,
+                    stoppingToken);
+
+                consecutiveErrors = 0;
+
+                // If nothing to process, use normal interval
+                // If items remain, poll sooner to catch up
+                var delay = result.Remaining > 0
+                    ? TimeSpan.FromSeconds(1)
+                    : TimeSpan.FromSeconds(_settings.PollIntervalSeconds);
+
+                await Task.Delay(delay, stoppingToken);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                consecutiveErrors++;
+
+                _logger.LogError(
+                    "EmbeddingBackfillWorker iteration failed ({ConsecutiveErrors} consecutive). {Error}",
+                    consecutiveErrors,
+                    SensitiveDataRedactor.SummarizeException(ex));
+
+                var backoffSeconds = CalculateBackoffSeconds(consecutiveErrors);
+
+                await Task.Delay(TimeSpan.FromSeconds(backoffSeconds), stoppingToken);
+            }
+        }
+
+        _logger.LogInformation("EmbeddingBackfillWorker stopped");
+    }
+
+    internal int CalculateBackoffSeconds(int consecutiveErrors)
+    {
+        var cappedErrors = Math.Min(
+            Math.Max(consecutiveErrors, 0),
+            _settings.MaxConsecutiveErrors);
+
+        var delaySeconds = _settings.PollIntervalSeconds;
+        for (var i = 0; i < cappedErrors; i++)
+        {
+            if (delaySeconds >= _settings.MaxBackoffSeconds / 2)
+            {
+                return _settings.MaxBackoffSeconds;
+            }
+
+            delaySeconds *= 2;
+        }
+
+        return Math.Min(_settings.MaxBackoffSeconds, delaySeconds);
+    }
+}

backend/src/Taskdeck.Application/DTOs/VectorSearchResult.cs

@@ -0,0 +1,9 @@
+namespace Taskdeck.Application.DTOs;
+
+/// <summary>
+/// Represents a single result from a nearest-neighbor vector search.
+/// </summary>
+public sealed record VectorSearchResult(
+    string DocumentId,
+    double Score,
+    IReadOnlyDictionary<string, string>? Metadata = null);

backend/src/Taskdeck.Application/Interfaces/IEmbeddingGenerator.cs

@@ -0,0 +1,36 @@
+namespace Taskdeck.Application.Interfaces;
+
+/// <summary>
+/// Generates fixed-dimension embedding vectors from text input.
+/// All implementations must run locally -- no user content may leave the machine.
+/// </summary>
+public interface IEmbeddingGenerator
+{
+    /// <summary>
+    /// The dimensionality of vectors produced by this generator.
+    /// Callers use this to pre-allocate storage and validate index compatibility.
+    /// </summary>
+    int Dimensions { get; }
+
+    /// <summary>
+    /// Whether the generator is ready to produce embeddings.
+    /// Returns false when the underlying model failed to load or dependencies
+    /// are unavailable, allowing callers to fall back to FTS.
+    /// </summary>
+    bool IsAvailable { get; }
+
+    /// <summary>
+    /// Generates an embedding vector for a single text input.
+    /// </summary>
+    Task<ReadOnlyMemory<float>> GenerateAsync(
+        string text,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Generates embedding vectors for a batch of text inputs.
+    /// The returned list is positionally aligned with the input list.
+    /// </summary>
+    Task<IReadOnlyList<ReadOnlyMemory<float>>> GenerateBatchAsync(
+        IReadOnlyList<string> texts,
+        CancellationToken cancellationToken = default);
+}

backend/src/Taskdeck.Application/Interfaces/IKnowledgeChunkRepository.cs

@@ -4,6 +4,19 @@ namespace Taskdeck.Application.Interfaces;
 
 public interface IKnowledgeChunkRepository : IRepository<KnowledgeChunk>
 {
+    Task<IEnumerable<KnowledgeChunk>> GetUnindexedBatchAsync(
+        KnowledgeChunkBackfillCursor? cursor,
+        int batchSize,
+        CancellationToken cancellationToken = default);
+
+    Task<IReadOnlySet<Guid>> GetExistingIdsAsync(
+        IReadOnlyCollection<Guid> candidateChunkIds,
+        CancellationToken cancellationToken = default);
+
+    Task<int> CountUnindexedAsync(
+        KnowledgeChunkBackfillCursor? cursor,
+        CancellationToken cancellationToken = default);
+
     Task<IEnumerable<KnowledgeChunk>> GetByDocumentIdAsync(
         Guid documentId,
         CancellationToken cancellationToken = default);
@@ -12,3 +25,8 @@ Task DeleteByDocumentIdAsync(
         Guid documentId,
         CancellationToken cancellationToken = default);
 }
+
+public sealed record KnowledgeChunkBackfillCursor(
+    DateTimeOffset CreatedAt,
+    Guid Id,
+    IReadOnlySet<Guid>? ProcessedIdsAtCreatedAt = null);

backend/src/Taskdeck.Application/Interfaces/IVectorIndex.cs

@@ -0,0 +1,71 @@
+using Taskdeck.Application.DTOs;
+
+namespace Taskdeck.Application.Interfaces;
+
+/// <summary>
+/// Abstracts vector storage and nearest-neighbor search so the backing store
+/// (in-memory, sqlite-vec, external service) is swappable without changing
+/// application-layer code.
+///
+/// All vectors are keyed by a string document identifier.
+/// Implementations must be thread-safe for concurrent read/write.
+/// </summary>
+public interface IVectorIndex
+{
+    /// <summary>
+    /// Upserts a single document embedding into the index.
+    /// If a vector with the same <paramref name="documentId"/> already exists
+    /// it is replaced atomically.
+    /// </summary>
+    Task UpsertAsync(
+        string documentId,
+        ReadOnlyMemory<float> vector,
+        IReadOnlyDictionary<string, string>? metadata = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Upserts multiple document embeddings in a single batch.
+    /// Implementations should optimize for throughput over latency.
+    /// </summary>
+    Task UpsertBatchAsync(
+        IReadOnlyList<VectorDocument> documents,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Finds the <paramref name="topK"/> nearest neighbors of the given
+    /// <paramref name="queryVector"/>.
+    /// </summary>
+    Task<IReadOnlyList<VectorSearchResult>> QueryAsync(
+        ReadOnlyMemory<float> queryVector,
+        int topK = 10,
+        IReadOnlyDictionary<string, string>? filter = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Deletes the vector associated with <paramref name="documentId"/>.
+    /// No-op if the document does not exist.
+    /// </summary>
+    Task DeleteAsync(
+        string documentId,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Deletes all vectors whose document IDs match any of the given IDs.
+    /// </summary>
+    Task DeleteBatchAsync(
+        IReadOnlyList<string> documentIds,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Returns the number of vectors currently stored in the index.
+    /// </summary>
+    Task<long> CountAsync(CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Batch-upsert payload for <see cref="IVectorIndex.UpsertBatchAsync"/>.
+/// </summary>
+public sealed record VectorDocument(
+    string DocumentId,
+    ReadOnlyMemory<float> Vector,
+    IReadOnlyDictionary<string, string>? Metadata = null);

backend/src/Taskdeck.Application/Services/EmbeddingBackfillSettings.cs

@@ -0,0 +1,40 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace Taskdeck.Application.Services;
+
+/// <summary>
+/// Configuration for the embedding backfill worker.
+/// </summary>
+public class EmbeddingBackfillSettings
+{
+    /// <summary>
+    /// Whether the backfill worker is enabled. When false, no embedding
+    /// backfill occurs even if vector dependencies are available.
+    /// </summary>
+    public bool Enabled { get; set; } = true;
+
+    /// <summary>
+    /// Number of items to process in each backfill batch.
+    /// </summary>
+    [Range(1, 500, ErrorMessage = "BatchSize must be between 1 and 500.")]
+    public int BatchSize { get; set; } = 50;
+
+    /// <summary>
+    /// Seconds to wait between backfill iterations.
+    /// </summary>
+    [Range(1, 86400, ErrorMessage = "PollIntervalSeconds must be between 1 and 86400.")]
+    public int PollIntervalSeconds { get; set; } = 30;
+
+    /// <summary>
+    /// Maximum number of consecutive errors before the worker pauses
+    /// with exponential backoff.
+    /// </summary>
+    [Range(1, 100, ErrorMessage = "MaxConsecutiveErrors must be between 1 and 100.")]
+    public int MaxConsecutiveErrors { get; set; } = 5;
+
+    /// <summary>
+    /// Maximum backoff delay in seconds when consecutive errors occur.
+    /// </summary>
+    [Range(10, 3600, ErrorMessage = "MaxBackoffSeconds must be between 10 and 3600.")]
+    public int MaxBackoffSeconds { get; set; } = 300;
+}

backend/src/Taskdeck.Application/Services/IEmbeddingBackfillService.cs

@@ -0,0 +1,25 @@
+namespace Taskdeck.Application.Services;
+
+/// <summary>
+/// Drives embedding backfill for entities that have not yet been indexed.
+/// Implementations must be resumable (track progress) and failure-safe
+/// (individual item failures do not block the batch).
+/// </summary>
+public interface IEmbeddingBackfillService
+{
+    /// <summary>
+    /// Processes a single batch of un-embedded items. Returns the number
+    /// of items successfully embedded in this batch (0 when caught up).
+    /// </summary>
+    Task<BackfillBatchResult> ProcessBatchAsync(
+        int batchSize,
+        CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Result of a single backfill batch execution.
+/// </summary>
+public sealed record BackfillBatchResult(
+    int Processed,
+    int Failed,
+    int Remaining);

backend/src/Taskdeck.Application/Services/IFtsKnowledgeSearchService.cs

@@ -0,0 +1,7 @@
+using Taskdeck.Application.DTOs;
+
+namespace Taskdeck.Application.Services;
+
+public interface IFtsKnowledgeSearchService : IKnowledgeSearchService
+{
+}