DSI-8836: Initial integration test setup for Internal API

.github/workflows/dotnet-checks.yml

@@ -80,6 +80,7 @@ jobs:
         run: |
           dotnet test ./build.sln `
             --no-build `
+            --filter "Category!=Integration" `
             --collect "XPlat Code Coverage" `
             --logger "trx;LogFileName=TestResults.trx" `
             /p:AutomatedRun=true

Directory.Packages.props

@@ -21,7 +21,7 @@
     <PackageVersion Include="Aspire.Hosting.NodeJs" Version="9.5.2" />
     <PackageVersion Include="Aspire.Hosting.Redis" Version="13.4.0" />
     <PackageVersion Include="CommunityToolkit.Aspire.Hosting.NodeJS.Extensions" Version="9.9.0" />
-    <!-- Dependency injection -->   
+    <!-- Dependency injection -->
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.2" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.2" />
     <!-- Options -->
@@ -104,8 +104,11 @@
     <PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.2" />
     <PackageVersion Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="10.0.2" />
     <!-- Database -->
+    <PackageVersion Include="Microsoft.EntityFrameworkCore" Version="9.0.9" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.9" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.InMemory" Version="9.0.9" />
+    <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.1.6" />
+    <PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="9.0.9" />
     <!-- Entity Framework tooling -->
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.9" />
   </ItemGroup>
@@ -130,13 +133,20 @@
   <ItemGroup>
     <!-- Code coverage -->
     <PackageVersion Include="coverlet.collector" Version="6.0.4" />
+    <!-- xUnit testing framework -->
+    <PackageVersion Include="xunit" Version="2.9.2" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
     <!-- Microsoft testing framework -->
     <PackageVersion Include="MSTest.TestAdapter" Version="3.10.4" />
     <PackageVersion Include="MSTest.TestFramework" Version="3.10.4" />
     <PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.0" />
+    <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
     <!-- Mocking -->
     <PackageVersion Include="Moq" Version="4.20.72" />
     <PackageVersion Include="Moq.AutoMock" Version="3.5.0" />
+    <!-- Integration testing -->
+    <PackageVersion Include="Testcontainers.MsSql" Version="4.3.0" />
+    <PackageVersion Include="Respawn" Version="7.0.0" />
   </ItemGroup>
 </Project>

docs/testing/Integration-Tests.md

@@ -0,0 +1,217 @@
+# DfE Sign-In Internal API Integration Testing Guide
+
+This guide explains the purpose, architecture, and implementation details of the integration testing framework in the `Dfe.SignIn.InternalApi.IntegrationTests` project. Use this document as a reference for writing new integration tests and troubleshooting test environments.
+
+---
+
+## 1. Purpose of Integration Tests
+
+While unit tests validate business logic in isolation by mocking all external dependencies, **integration tests** verify that the entire application stack works together correctly.
+
+In the DfE Sign-In platform, integration tests ensure:
+*   **Database Schema Compatibility**: Entity Framework Core mappings align perfectly with real SQL Server database schemas.
+*   **API Route and Contract Validation**: HTTP endpoints are correctly mapped, accept the expected request payloads, and return correct response payloads.
+*   **Security & Interception**: Custom middlewares, filters, logging contexts, and exception handling blocks function correctly in a real HTTP lifecycle.
+*   **Resilience & DI Setup**: Dependency injection registrations resolve correctly without runtime exceptions during host startup.
+
+---
+
+## 2. How the Testing Flow Works
+
+The framework spins up a lightweight, isolated SQL Server instance inside a Docker container, runs EF Core database schema creation, hosts the API in-memory, and executes tests against it.
+
+```mermaid
+sequenceDiagram
+    participant TestRunner as xUnit Runner
+    participant Factory as WebApplicationFactory
+    participant Container as Docker (SQL Server)
+    participant API as In-Memory API Host
+    participant Db as Test Database
+
+    Note over TestRunner,Db: Class Initialisation (via collection fixture)
+    TestRunner->>Factory: Instantiate Factory (Starts container once)
+    Factory->>Container: Boot SQL Server container (Testcontainers)
+    Container-->>Factory: Connection string returned
+    Factory->>Factory: Map JSON settings to Env variables
+    Factory->>Db: EnsureCreated() (directories & organisations schemas)
+    Factory->>Factory: Initialise Respawner
+
+    Note over TestRunner,Db: Test Initialization (Before Each Test)
+    TestRunner->>Factory: ResetDatabasesAsync()
+    Factory->>Db: Respawn wipes tables (FKs disabled/re-enabled)
+    TestRunner->>Factory: Create HttpClient
+
+    Note over TestRunner,Db: Test Execution (Arrange, Act, Assert)
+    TestRunner->>Db: Seed entities (using scoped DbContext)
+    TestRunner->>API: HTTP POST request (using TestAuthHandler bypass)
+    API->>Db: Query/Update tables
+    Db-->>API: Data returned
+    API-->>TestRunner: HTTP Response (JSON)
+    TestRunner->>TestRunner: Assert status code and response payload
+```
+
+---
+
+## 3. Key Libraries Used
+
+We utilize three main libraries to keep integration tests fast, isolated, and easy to maintain:
+
+### 1. `Microsoft.AspNetCore.Mvc.Testing` (`WebApplicationFactory<T>`)
+Hosts the API in-memory using a test-specific web server.
+*   **Why**: It allows sending real HTTP requests to your endpoints and testing the full middleware pipeline without running a slow, external IIS or Kestrel process.
+
+### 2. `Testcontainers.MsSql`
+Provides lightweight, disposable SQL Server instances running in Docker.
+*   **Why**: It avoids the need for a shared, local database server that could contain stale data. Each test run gets a 100% fresh, isolated SQL Server instance.
+
+### 3. `Respawn`
+Resets the state of database tables back to a blank schema.
+*   **Why**: Dropping and recreating database schemas for every test takes seconds. Respawn queries the database metadata, disables foreign keys, deletes all data using `DELETE`/`TRUNCATE` statements, and re-enables constraints in **under 15 milliseconds**, ensuring total test isolation without a speed penalty.
+
+---
+
+## 4. How to Write a New Test
+
+Follow these steps to add integration tests for a new endpoint:
+
+### Step 1: Add a Test Class
+Create a new test file under the `Endpoints/` folder. Use the following class template:
+
+```csharp
+using System.Net;
+using System.Net.Http.Json;
+using Dfe.SignIn.Core.Contracts.Organisations; // Replace with your request/response contracts
+using Dfe.SignIn.Core.Entities.Organisations;  // Replace with your database entity models
+using Dfe.SignIn.Gateways.EntityFramework;
+using Dfe.SignIn.InternalApi.Contracts;
+using Microsoft.Extensions.DependencyInjection;
+using Xunit;
+
+namespace Dfe.SignIn.InternalApi.IntegrationTests.Endpoints;
+
+[Collection("IntegrationTestsCollection")]
+[Trait("Category", "Integration")]
+public class GetOrganisationByIdTests : IAsyncLifetime
+{
+    private readonly InternalApiWebApplicationFactory _factory;
+    private readonly HttpClient _client;
+
+    public GetOrganisationByIdTests(InternalApiWebApplicationFactory factory)
+    {
+        _factory = factory;
+        _client = _factory.CreateClient();
+    }
+
+    public async Task InitializeAsync()
+    {
+        // Clear database state between tests
+        await _factory.ResetDatabasesAsync();
+    }
+
+    public Task DisposeAsync() => Task.CompletedTask;
+
+    [Fact]
+    public async Task GetOrganisationById_ReturnsOrganisation_WhenExists()
+    {
+        // 1. Arrange: Seed your data using the scoped DbContext
+        var orgId = Guid.NewGuid();
+        var expectedName = "Test Academy Trust";
+
+        await using var scope = _factory.Services.CreateAsyncScope();
+        var dbContext = scope.ServiceProvider.GetRequiredService<DbOrganisationsContext>();
+        dbContext.Organisations.Add(new OrganisationEntity
+        {
+            Id = orgId,
+            Name = expectedName,
+            Category = "001", // Required field for DB mapping
+            Status = 1,       // Required field for DB mapping
+            CreatedAt = DateTime.UtcNow,
+            UpdatedAt = DateTime.UtcNow
+        });
+        await dbContext.SaveChangesAsync();
+
+        var request = new GetOrganisationByIdRequest { OrganisationId = orgId };
+
+        // 2. Act: Send request to the endpoint
+        var response = await _client.PostAsJsonAsync("interaction/Organisations.GetOrganisationById", request);
+
+        // 3. Assert: Verify response
+        if (response.StatusCode != HttpStatusCode.OK)
+        {
+            var errorBody = await response.Content.ReadAsStringAsync();
+            Assert.Fail($"Request failed with status {response.StatusCode}. Response: {errorBody}");
+        }
+
+        var body = await response.Content.ReadFromJsonAsync<InteractionResponse<GetOrganisationByIdResponse>>();
+        Assert.NotNull(body);
+        Assert.NotNull(body.Data);
+        Assert.Equal(orgId, body.Data.Organisation.Id);
+        Assert.Equal(expectedName, body.Data.Organisation.Name);
+    }
+}
+```
+
+### Step 2: Implement Common Seeding Helpers (Optional)
+If you find yourself seeding the same entities (like a default user or organisation) across multiple test classes, create an extension class to reuse the code:
+
+```csharp
+public static class SeedExtensions
+{
+    public static async Task<Guid> SeedOrganisationAsync(this InternalApiWebApplicationFactory factory, string name)
+    {
+        var id = Guid.NewGuid();
+        await using var scope = factory.Services.CreateAsyncScope();
+        var context = scope.ServiceProvider.GetRequiredService<DbOrganisationsContext>();
+
+        context.Organisations.Add(new OrganisationEntity
+        {
+            Id = id,
+            Name = name,
+            Category = "001",
+            Status = 1,
+            CreatedAt = DateTime.UtcNow,
+            UpdatedAt = DateTime.UtcNow
+        });
+
+        await context.SaveChangesAsync();
+        return id;
+    }
+}
+```
+
+---
+
+## 5. Configuration & Under-The-Hood Details
+
+### Minimal API Startup Limitation
+In .NET 6+ Minimal APIs, using `ConfigureWebHost` to override configuration properties happens **after** `Program.cs` starts parsing `builder.Configuration`. Since the API project checks configuration keys immediately during bootstrap (`CreateBuilder(args)`), standard `WebApplicationFactory` configuration overrides are too late and cause "Section not found" exceptions.
+
+### Solution: JSON-to-Environment Variable Mapping
+We solve this by loading static test settings from [appsettings.IntegrationTests.json](file:///c:/Work/Playground/dsi-workspace/dsi-platform/tests/Dfe.SignIn.InternalApi.IntegrationTests/appsettings.IntegrationTests.json) (or custom test settings file defined by the factory) in the constructor and dumping them as process environment variables. 
+The configuration binder translates double underscores (`__`) into colons (`:`), resulting in a perfect representation of configuration sections (e.g. `AzureAd__Audience` maps to `AzureAd:Audience`) which are processed in time by `Program.cs`.
+
+---
+
+## 6. Troubleshooting & Tips for Developers
+
+### "An Application Control policy has blocked this file (0x800711C7)"
+This occurs on corporate-managed laptops when Windows AppLocker blocks the execution/loading of DLLs in arbitrary folders (like `C:\Users\` or user sandbox folders).
+*   **Fix 1**: Move/clone your workspace folder to a whitelisted developer directory, such as `C:\git\` or `C:\source\`.
+*   **Fix 2**: Run tests inside **WSL 2** (Ubuntu/Linux terminal) where Windows AppLocker policies do not apply:
+    ```bash
+    dotnet test tests/Dfe.SignIn.InternalApi.IntegrationTests/Dfe.SignIn.InternalApi.IntegrationTests.csproj
+    ```
+
+### "Docker is either not running or misconfigured"
+Testcontainers requires a running Docker daemon.
+*   **Fix**: Ensure **Docker Desktop** is running. If Docker is active and the error persists:
+    *   Disable **"Resource Saver Mode"** in Docker Desktop settings (known to freeze background container boot requests).
+    *   If using WSL 2 backend, ensure integration is enabled for your current distro.
+
+### Test execution is stuck at `StartAsync` / Taking too long
+The first time integration tests run, Testcontainers has to pull the large Microsoft SQL Server image. Because it pulls the image silently over the Docker API, it looks like the test runner is frozen.
+*   **Fix**: Run a manual pull in your terminal once to download the image with progress indicators:
+    ```bash
+    docker pull mcr.microsoft.com/mssql/server:2022-latest
+    ```
+    All subsequent runs will boot instantly since the image will be cached in your local Docker registry.

dsi-platform.sln

@@ -119,6 +119,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		README.md = README.md
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Dfe.SignIn.InternalApi.IntegrationTests", "tests\Dfe.SignIn.InternalApi.IntegrationTests\Dfe.SignIn.InternalApi.IntegrationTests.csproj", "{E92E522D-3383-4356-AD07-D1C491916E26}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -729,6 +731,18 @@ Global
 		{5100CDC3-5C26-B655-51E7-C5738A1DCD44}.Release|x64.Build.0 = Release|Any CPU
 		{5100CDC3-5C26-B655-51E7-C5738A1DCD44}.Release|x86.ActiveCfg = Release|Any CPU
 		{5100CDC3-5C26-B655-51E7-C5738A1DCD44}.Release|x86.Build.0 = Release|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Debug|x64.Build.0 = Debug|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Debug|x86.Build.0 = Debug|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Release|x64.ActiveCfg = Release|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Release|x64.Build.0 = Release|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Release|x86.ActiveCfg = Release|Any CPU
+		{E92E522D-3383-4356-AD07-D1C491916E26}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -784,6 +798,7 @@ Global
 		{282680D1-4103-4AC6-9238-0038A937DAA2} = {3F245DBD-544F-4DD4-9B56-06B116BB9CFE}
 		{36F11936-D413-4D86-B5E7-0BF41EB2885A} = {3F245DBD-544F-4DD4-9B56-06B116BB9CFE}
 		{5100CDC3-5C26-B655-51E7-C5738A1DCD44} = {3F245DBD-544F-4DD4-9B56-06B116BB9CFE}
+		{E92E522D-3383-4356-AD07-D1C491916E26} = {66605B0E-3683-4A9D-B8C6-73C7146BC2D6}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {86FC62DB-8432-46A5-BBDD-C58FDCB6403E}

src/Dfe.SignIn.Base.Framework/ExceptionReflectionHelpers.cs

@@ -26,12 +26,19 @@
                     return exceptionTypesByFullName;
                 }
 
                 exceptionTypesByFullName = AppDomain.CurrentDomain.GetAssemblies()
-                    .SelectMany(assembly => assembly.GetTypes())
+                    .SelectMany(assembly => {
+                        try {
+                            return assembly.GetTypes();
+                        }
+                        catch (ReflectionTypeLoadException e) {
+                            return e.Types.Where(t => t != null)!;
+                        }
+                    })
                     .Where(type => typeof(Exception).IsAssignableFrom(type))
                     .GroupBy(type => type.FullName)
                     .ToDictionary(
                         keySelector: type => type.First().FullName!,
                         elementSelector: type => type.First()
                     );
                 return exceptionTypesByFullName;
@@ -117,7 +124,7 @@
 
         IEnumerable<PropertyInfo> result;
         lock (@lock) {
             if (!propertiesByType.TryGetValue(exceptionType, out result!)) {
                 result = [.. exceptionType.GetProperties().Where(IsSerializableProperty)];
                 propertiesByType[exceptionType] = result;
             }

src/Dfe.SignIn.InternalApi/Program.cs

@@ -125,3 +125,9 @@
 app.UseUserEndpoints();
 
 await app.RunAsync();
+
+// Expose the Program class to the integration tests project
+/// <summary>
+/// The entry point class for the application.
+/// </summary>
+public partial class Program { }

tests/Dfe.SignIn.InternalApi.IntegrationTests/Dfe.SignIn.InternalApi.IntegrationTests.csproj

@@ -0,0 +1,55 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <ProjectGuid>{E92E522D-3383-4356-AD07-D1C491916E26}</ProjectGuid>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
+    <PackageReference Include="Testcontainers.MsSql" />
+    <PackageReference Include="Respawn" />
+    <PackageReference Include="Moq" />
+    <PackageReference Include="Moq.AutoMock" />
+    <PackageReference Include="OpenTelemetry.Api" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Xunit" />
+    <Using Include="Dfe.SignIn.TestHelpers" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Dfe.SignIn.TestHelpers\Dfe.SignIn.TestHelpers.csproj" />
+    <ProjectReference Include="..\..\src\Dfe.SignIn.InternalApi\Dfe.SignIn.InternalApi.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Endpoints\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="appsettings.Test.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>