Merge pull request #823 from Chris0Jeky/test/resilience-degraded-mode-behavior

test: resilience and degraded-mode behavior tests (#720)

Author: Chris0Jeky

backend/tests/Taskdeck.Api.Tests/Resilience/QueueAccumulationResilienceTests.cs

@@ -0,0 +1,164 @@
+using System.Net;
+using System.Net.Http.Json;
+using FluentAssertions;
+using Microsoft.Extensions.DependencyInjection;
+using Taskdeck.Api.Tests.Support;
+using Taskdeck.Application.DTOs;
+using Taskdeck.Application.Interfaces;
+using Taskdeck.Domain.Entities;
+using Taskdeck.Domain.Enums;
+using Taskdeck.Infrastructure.Persistence;
+using Xunit;
+
+namespace Taskdeck.Api.Tests.Resilience;
+
+/// <summary>
+/// Tests that queue items accumulate correctly when workers are not processing
+/// (simulated by having processing disabled or worker stopped), and that items
+/// remain consistent (no corruption) and are processable on restart.
+/// Covers issue #720 (TST-67): "All workers stopped → queue items accumulate
+/// but don't corrupt; restart processes them."
+/// </summary>
+public class QueueAccumulationResilienceTests : IClassFixture<TestWebApplicationFactory>
+{
+    private readonly TestWebApplicationFactory _factory;
+
+    public QueueAccumulationResilienceTests(TestWebApplicationFactory factory)
+    {
+        _factory = factory;
+    }
+
+    // ── Queue Items Accumulate Without Corruption ─────────────────────
+
+    [Fact]
+    public async Task QueueItems_AccumulateWithoutCorruption_WhenWorkersNotProcessing()
+    {
+        // Arrange: create a user, then enqueue multiple capture items.
+        // The background worker may process them, but the important assertion is
+        // that items are created with correct status and no data corruption occurs
+        // regardless of worker state.
+        using var client = _factory.CreateClient();
+        await ApiTestHarness.AuthenticateAsync(client, "queue-accum-resilience");
+
+        // Create multiple capture items in quick succession.
+        var itemIds = new List<Guid>();
+        for (var i = 0; i < 5; i++)
+        {
+            var response = await client.PostAsJsonAsync(
+                "/api/capture/items",
+                new CreateCaptureItemDto(null, $"Queue accumulation test item {i}"));
+            response.StatusCode.Should().Be(HttpStatusCode.Created,
+                $"capture item {i} should be accepted");
+
+            var item = await response.Content.ReadFromJsonAsync<CaptureItemDto>();
+            item.Should().NotBeNull();
+            itemIds.Add(item!.Id);
+        }
+
+        // Assert: all items should exist and have consistent state.
+        var listResponse = await client.GetAsync("/api/capture/items?limit=100");
+        listResponse.StatusCode.Should().Be(HttpStatusCode.OK);
+
+        var listPayload = await listResponse.Content.ReadFromJsonAsync<CaptureItemSummaryDto[]>();
+        listPayload.Should().NotBeNull();
+
+        // All 5 items should be present (they may have been processed already by the worker,
+        // but none should be missing or corrupted).
+        foreach (var id in itemIds)
+        {
+            listPayload!.Should().Contain(
+                i => i.Id == id,
+                $"item {id} should exist in the queue regardless of worker state");
+        }
+
+        // No item should be in an invalid/corrupted status.
+        foreach (var item in listPayload!.Where(i => itemIds.Contains(i.Id)))
+        {
+            item.Status.Should().BeDefined(
+                "item status should always be set to a valid enum value");
+        }
+    }
+
+    // ── Queue Items Are Processable After Accumulation ───────────────
+
+    [Fact]
+    public async Task QueuedItems_RemainProcessable_AfterAccumulation()
+    {
+        // Verify that items created during worker downtime have valid status
+        // and are in a state that allows future processing.
+        using var scope = _factory.Services.CreateScope();
+        var dbContext = scope.ServiceProvider.GetRequiredService<TaskdeckDbContext>();
+
+        var user = new User("queue-processable-user", "queue-processable@example.com", "hash");
+        dbContext.Users.Add(user);
+        await dbContext.SaveChangesAsync();
+
+        // Create LLM queue items directly (bypassing API to simulate accumulated items).
+        var items = new List<LlmRequest>();
+        for (var i = 0; i < 3; i++)
+        {
+            var item = new LlmRequest(user.Id, "instruction", $"Create card {i}", null);
+            items.Add(item);
+            dbContext.Add(item);
+        }
+        await dbContext.SaveChangesAsync();
+
+        // Assert: all items should start as Pending and be individually processable.
+        foreach (var item in items)
+        {
+            await dbContext.Entry(item).ReloadAsync();
+            item.Status.Should().Be(RequestStatus.Pending,
+                "accumulated items should be in Pending status, ready for processing");
+            item.RetryCount.Should().Be(0,
+                "fresh items should have zero retry count");
+        }
+
+        // Simulate a worker picking up the first item (MarkAsProcessing).
+        items[0].MarkAsProcessing();
+        await dbContext.SaveChangesAsync();
+        await dbContext.Entry(items[0]).ReloadAsync();
+
+        items[0].Status.Should().Be(RequestStatus.Processing,
+            "first item should transition to Processing when claimed by a worker");
+
+        // Other items should remain Pending (not affected by the first item's transition).
+        await dbContext.Entry(items[1]).ReloadAsync();
+        await dbContext.Entry(items[2]).ReloadAsync();
+        items[1].Status.Should().Be(RequestStatus.Pending,
+            "other items should remain Pending when one is claimed");
+        items[2].Status.Should().Be(RequestStatus.Pending);
+    }
+
+    // ── Capture Items Do Not Corrupt Under Rapid Submission ──────────
+
+    [Fact]
+    public async Task RapidCaptureSubmission_DoesNotCorruptQueue()
+    {
+        using var client = _factory.CreateClient();
+        await ApiTestHarness.AuthenticateAsync(client, "queue-rapid-submit");
+
+        // Submit captures as fast as possible (no await between sends).
+        var tasks = Enumerable.Range(0, 10).Select(i =>
+            client.PostAsJsonAsync(
+                "/api/capture/items",
+                new CreateCaptureItemDto(null, $"Rapid item {i}")));
+
+        var responses = await Task.WhenAll(tasks);
+
+        // All submissions should succeed (201 Created).
+        foreach (var response in responses)
+        {
+            response.StatusCode.Should().Be(HttpStatusCode.Created,
+                "every rapid submission should succeed without corruption");
+        }
+
+        // Verify items are retrievable.
+        var listResponse = await client.GetAsync("/api/capture/items?limit=100");
+        listResponse.StatusCode.Should().Be(HttpStatusCode.OK);
+
+        var payload = await listResponse.Content.ReadFromJsonAsync<CaptureItemSummaryDto[]>();
+        payload.Should().NotBeNull();
+        payload!.Should().HaveCountGreaterThanOrEqualTo(10,
+            "all rapidly submitted items should be present");
+    }
+}