Merge remote-tracking branch 'origin/main' into agents/ai-agent-security-review

# Conflicts:
#	EssentialCSharp.Chat.Shared/Services/AIChatService.cs

Co-authored-by: BenjaminMichaelis <22186029+BenjaminMichaelis@users.noreply.github.com>

Author: Copilot

.github/workflows/Build-Test-And-Deploy.yml

@@ -63,23 +63,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
-      - name: Generate NuGet auth config for Docker build
-        if: github.event_name != 'pull_request' && github.event_name != 'merge_group'
-        env:
-          NUGET_AUTH_TOKEN: ${{ secrets.AZURE_DEVOPS_PAT }}
-        run: |
-          cat > /tmp/nuget-auth.config << EOF
-          <?xml version="1.0" encoding="utf-8"?>
-          <configuration>
-            <packageSourceCredentials>
-              <EssentialCSharp>
-                <add key="Username" value="docker" />
-                <add key="ClearTextPassword" value="${NUGET_AUTH_TOKEN}" />
-              </EssentialCSharp>
-            </packageSourceCredentials>
-          </configuration>
-          EOF
-
 # Build but no push with a PR
       - name: Docker build (no push)
         if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
@@ -99,7 +82,7 @@ jobs:
           file: ./EssentialCSharp.Web/Dockerfile
           context: .
           secrets: |
-            "id=nugetconfig,src=/tmp/nuget-auth.config"
+            "nuget_pat=${{ secrets.AZURE_DEVOPS_PAT }}"
           outputs: type=docker,dest=${{ github.workspace }}/essentialcsharpwebimage.tar
           cache-from: type=gha
           cache-to: type=gha,mode=max
@@ -109,10 +92,6 @@ jobs:
           name: essentialcsharpwebimage
           path: ${{ github.workspace }}/essentialcsharpwebimage.tar
 
-      - name: Clean up NuGet auth config
-        if: always()
-        run: rm -f /tmp/nuget-auth.config
-
   deploy-development:
     if: github.event_name != 'pull_request_target' && github.event_name != 'pull_request'
     runs-on: ubuntu-latest

.gitignore

@@ -10,6 +10,7 @@ TestResults/
 ~$*.do[ct]x
 ~*.tmp
 ~$*.dotm
+.playwright-mcp/
 
 # Files to keep (primarily book content)
 !.gitignore

Directory.Packages.props

@@ -4,13 +4,10 @@
     <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
     <ToolingPackagesVersion>1.1.1.19025</ToolingPackagesVersion>
     <AccessToNugetFeed Condition="'$(AccessToNugetFeed)' == ''">false</AccessToNugetFeed>
-    <RestoreSources>
-      https://api.nuget.org/v3/index.json;
-    </RestoreSources>
-    <RestoreSources Condition="$(AccessToNugetFeed)">
-      $(RestoreSources);
-      https://pkgs.dev.azure.com/intelliTect/_packaging/EssentialCSharp/nuget/v3/index.json;
-    </RestoreSources>
+    <!-- Disable NuGet vulnerability audit when the private feed is unavailable (e.g. CI without credentials).
+         NuGet audit queries all sources in nuget.config regardless of RestoreSources, causing NU1900 which is
+         escalated to an error by TreatWarningsAsErrors. -->
+    <NuGetAudit Condition="'$(AccessToNugetFeed)' != 'true'">false</NuGetAudit>
   </PropertyGroup>
   <PropertyGroup>
     <SemanticKernelVersion>1.72.0</SemanticKernelVersion>

EssentialCSharp.Chat.Shared/Services/AIChatService.cs

@@ -63,9 +63,9 @@ public AIChatService(IOptions<AIOptions> options, AISearchService searchService,
         string? endUserId = null,
         CancellationToken cancellationToken = default)
     {
-        var responseOptions = await CreateResponseOptionsAsync(previousResponseId, tools, reasoningEffortLevel, mcpClient: mcpClient, endUserId: endUserId, cancellationToken: cancellationToken);
+        var responseOptions = await CreateResponseOptionsAsync(systemPrompt, previousResponseId, tools, reasoningEffortLevel, mcpClient: mcpClient, endUserId: endUserId, cancellationToken: cancellationToken);
         var enrichedPrompt = await EnrichPromptWithContext(prompt, enableContextualSearch, cancellationToken);
-        return await GetChatCompletionCore(enrichedPrompt, responseOptions, systemPrompt, mcpClient, endUserId, cancellationToken);
+        return await GetChatCompletionCore(enrichedPrompt, responseOptions, mcpClient, endUserId, cancellationToken);
     }
 
     /// <summary>
@@ -94,17 +94,12 @@ public AIChatService(IOptions<AIOptions> options, AISearchService searchService,
         string? endUserId = null,
         [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        var responseOptions = await CreateResponseOptionsAsync(previousResponseId, tools, reasoningEffortLevel, mcpClient: mcpClient, endUserId: endUserId, cancellationToken: cancellationToken);
+        var responseOptions = await CreateResponseOptionsAsync(systemPrompt, previousResponseId, tools, reasoningEffortLevel, mcpClient: mcpClient, endUserId: endUserId, cancellationToken: cancellationToken);
         var enrichedPrompt = await EnrichPromptWithContext(prompt, enableContextualSearch, cancellationToken);
 
-        // Construct the user input with system context if provided
-        var systemContext = !string.IsNullOrWhiteSpace(systemPrompt) ? systemPrompt : _Options.SystemPrompt;
-
         // Create the streaming response using the Responses API
 #pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-        List<ResponseItem> responseItems = systemContext is not null
-            ? [ResponseItem.CreateSystemMessageItem(systemContext), ResponseItem.CreateUserMessageItem(enrichedPrompt)]
-            : [ResponseItem.CreateUserMessageItem(enrichedPrompt)];
+        List<ResponseItem> responseItems = [ResponseItem.CreateUserMessageItem(enrichedPrompt)];
 #pragma warning restore OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
         var streamingUpdates = _ResponseClient.CreateResponseStreamingAsync(
             responseItems,
@@ -327,6 +322,7 @@ private static string SanitizeForXmlContext(string? input) =>
     /// </summary>
 #pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
     private async Task<ResponseCreationOptions> CreateResponseOptionsAsync(
+        string? systemPrompt = null,
         string? previousResponseId = null,
         IEnumerable<ResponseTool>? tools = null,
         ResponseReasoningEffortLevel? reasoningEffortLevel = null,
@@ -338,6 +334,14 @@ private async Task<ResponseCreationOptions> CreateResponseOptionsAsync(
         var options = new ResponseCreationOptions();
 #pragma warning restore OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
 
+        // Set the system prompt via Instructions — this is stateless across turns when using previous_response_id,
+        // preventing accumulation of system messages in the conversation context.
+        var resolvedSystemPrompt = !string.IsNullOrWhiteSpace(systemPrompt) ? systemPrompt : _Options.SystemPrompt;
+        if (!string.IsNullOrWhiteSpace(resolvedSystemPrompt))
+        {
+            options.Instructions = resolvedSystemPrompt;
+        }
+
         // Add conversation context if available
         if (!string.IsNullOrEmpty(previousResponseId))
         {
@@ -401,18 +405,13 @@ private async Task<ResponseCreationOptions> CreateResponseOptionsAsync(
 #pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
         ResponseCreationOptions responseOptions,
 #pragma warning restore OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-        string? systemPrompt = null,
         McpClient? mcpClient = null,
         string? endUserId = null,
         CancellationToken cancellationToken = default)
     {
-        // Construct the user input with system context if provided
-        var systemContext = !string.IsNullOrWhiteSpace(systemPrompt) ? systemPrompt : _Options.SystemPrompt;
-
+        // Create the response using the Responses API
 #pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-        List<ResponseItem> responseItems = systemContext is not null
-            ? [ResponseItem.CreateSystemMessageItem(systemContext), ResponseItem.CreateUserMessageItem(prompt)]
-            : [ResponseItem.CreateUserMessageItem(prompt)];
+        List<ResponseItem> responseItems = [ResponseItem.CreateUserMessageItem(prompt)];
 #pragma warning restore OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
 
         const int MaxToolCallIterations = 10;

EssentialCSharp.Web.Tests/SitemapXmlHelpersTests.cs

@@ -1,4 +1,3 @@
-using System.IO;
 using System.Globalization;
 using DotnetSitemapGenerator;
 using EssentialCSharp.Web.Helpers;
@@ -70,14 +69,12 @@ await Assert.That(() => SitemapXmlHelpers.EnsureSitemapHealthy(siteMappings))
     public async Task GenerateSitemapXml_DoesNotIncludeIdentityRoutes()
     {
         // Arrange
-        var tempDir = Directory.CreateTempSubdirectory("SitemapTest_");
         var siteMappings = new List<SiteMapping> { CreateSiteMapping(1, 1, true) };
         var baseUrl = "https://test.example.com/";
 
         // Act & Assert
         var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
         SitemapXmlHelpers.GenerateSitemapXml(
-            tempDir,
             siteMappings,
             routeConfigurationService,
             baseUrl,
@@ -98,14 +95,12 @@ public async Task GenerateSitemapXml_DoesNotIncludeIdentityRoutes()
     public async Task GenerateSitemapXml_IncludesBaseUrl()
     {
         // Arrange
-        var tempDir = Directory.CreateTempSubdirectory("SitemapTest_");
         var siteMappings = new List<SiteMapping>();
         var baseUrl = "https://test.example.com/";
 
         // Act & Assert
         var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
         SitemapXmlHelpers.GenerateSitemapXml(
-            tempDir,
             siteMappings,
             routeConfigurationService,
             baseUrl,
@@ -126,7 +121,6 @@ public async Task GenerateSitemapXml_IncludesBaseUrl()
     public async Task GenerateSitemapXml_IncludesSiteMappingsMarkedForXml()
     {
         // Arrange
-        var tempDir = Directory.CreateTempSubdirectory("SitemapTest_");
         var baseUrl = "https://test.example.com/";
 
         var siteMappings = new List<SiteMapping>
@@ -139,7 +133,6 @@ public async Task GenerateSitemapXml_IncludesSiteMappingsMarkedForXml()
         // Act & Assert
         var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
         SitemapXmlHelpers.GenerateSitemapXml(
-            tempDir,
             siteMappings,
             routeConfigurationService,
             baseUrl,
@@ -156,14 +149,12 @@ public async Task GenerateSitemapXml_IncludesSiteMappingsMarkedForXml()
     public async Task GenerateSitemapXml_DoesNotIncludeIndexRoutes()
     {
         // Arrange
-        var tempDir = Directory.CreateTempSubdirectory("SitemapTest_");
         var siteMappings = new List<SiteMapping>();
         var baseUrl = "https://test.example.com/";
 
         // Act & Assert
         var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
         SitemapXmlHelpers.GenerateSitemapXml(
-            tempDir,
             siteMappings,
             routeConfigurationService,
             baseUrl,
@@ -179,14 +170,12 @@ public async Task GenerateSitemapXml_DoesNotIncludeIndexRoutes()
     public async Task GenerateSitemapXml_DoesNotIncludeErrorRoutes()
     {
         // Arrange
-        var tempDir = Directory.CreateTempSubdirectory("SitemapTest_");
         var siteMappings = new List<SiteMapping>();
         var baseUrl = "https://test.example.com/";
 
         // Act & Assert
         var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
         SitemapXmlHelpers.GenerateSitemapXml(
-            tempDir,
             siteMappings,
             routeConfigurationService,
             baseUrl,
@@ -198,11 +187,31 @@ public async Task GenerateSitemapXml_DoesNotIncludeErrorRoutes()
         await Assert.That(allUrls).DoesNotContain(url => url.Contains("/Error", StringComparison.OrdinalIgnoreCase));
     }
 
+    [Test]
+    public async Task GenerateSitemapXml_DoesNotIncludeSitemapRoute()
+    {
+        // Arrange
+        var siteMappings = new List<SiteMapping>();
+        var baseUrl = "https://test.example.com/";
+
+        // Act
+        var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
+        SitemapXmlHelpers.GenerateSitemapXml(
+            siteMappings,
+            routeConfigurationService,
+            baseUrl,
+            out var nodes);
+
+        var allUrls = nodes.Select(n => n.Url).ToList();
+
+        // /sitemap.xml should not list itself
+        await Assert.That(allUrls).DoesNotContain(url => url.EndsWith("/sitemap.xml", StringComparison.OrdinalIgnoreCase));
+    }
+
     [Test]
     public async Task GenerateSitemapXml_UsesLastModifiedDateFromSiteMapping()
     {
         // Arrange
-        var tempDir = Directory.CreateTempSubdirectory("SitemapTest_");
         var baseUrl = "https://test.example.com/";
         var specificLastModified = new DateTime(2023, 5, 15, 10, 30, 0, DateTimeKind.Utc);
 
@@ -214,7 +223,6 @@ public async Task GenerateSitemapXml_UsesLastModifiedDateFromSiteMapping()
         // Act
         var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
         SitemapXmlHelpers.GenerateSitemapXml(
-            tempDir,
             siteMappings,
             routeConfigurationService,
             baseUrl,
@@ -225,6 +233,29 @@ public async Task GenerateSitemapXml_UsesLastModifiedDateFromSiteMapping()
         await Assert.That(siteMappingNode.LastModificationDate).IsEqualTo(specificLastModified);
     }
 
+    [Test]
+    public async Task GenerateSitemapXml_DoesNotSetLastModifiedDateWhenSiteMappingDateIsMissing()
+    {
+        // Arrange
+        var baseUrl = "https://test.example.com/";
+        var siteMappings = new List<SiteMapping>
+        {
+            CreateSiteMapping(1, 1, true, "test-page-1")
+        };
+
+        // Act
+        var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
+        SitemapXmlHelpers.GenerateSitemapXml(
+            siteMappings,
+            routeConfigurationService,
+            baseUrl,
+            out var nodes);
+
+        // Assert
+        var siteMappingNode = nodes.First(node => node.Url.Contains("test-page-1"));
+        await Assert.That(siteMappingNode.LastModificationDate).IsNull();
+    }
+
     private static SiteMapping CreateSiteMapping(
         int chapterNumber,
         int pageNumber,

EssentialCSharp.Web/Controllers/BaseController.cs

@@ -16,6 +16,8 @@ protected BaseController(IRouteConfigurationService routeConfigurationService, I
         _HttpContextAccessor = httpContextAccessor;
     }
 
+    protected IRouteConfigurationService RouteConfigurationService => _RouteConfigurationService;
+
     public override void OnActionExecuting(ActionExecutingContext context)
     {
         // Automatically add static routes to all views

EssentialCSharp.Web/Controllers/ChatController.cs

@@ -12,6 +12,7 @@ namespace EssentialCSharp.Web.Controllers;
 [Route("api/[controller]")]
 [Authorize]
 [EnableRateLimiting("ChatEndpoint")]
+[IgnoreAntiforgeryToken]
 public partial class ChatController : ControllerBase
 {
     private readonly AIChatService _AiChatService;

EssentialCSharp.Web/Controllers/HomeController.cs

@@ -1,14 +1,18 @@
+using DotnetSitemapGenerator;
 using EssentialCSharp.Web.Extensions;
+using EssentialCSharp.Web.Helpers;
 using EssentialCSharp.Web.Models;
 using EssentialCSharp.Web.Services;
 using HtmlAgilityPack;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.OutputCaching;
 using Microsoft.AspNetCore.RateLimiting;
 using Microsoft.CodeAnalysis;
+using Microsoft.Extensions.Options;
 
 namespace EssentialCSharp.Web.Controllers;
 
-public class HomeController(ILogger<HomeController> logger, IWebHostEnvironment hostingEnvironment, ISiteMappingService siteMappingService, IHttpContextAccessor httpContextAccessor, IRouteConfigurationService routeConfigurationService) : BaseController(routeConfigurationService, httpContextAccessor)
+public class HomeController(ILogger<HomeController> logger, IWebHostEnvironment hostingEnvironment, ISiteMappingService siteMappingService, IHttpContextAccessor httpContextAccessor, IRouteConfigurationService routeConfigurationService, IOptions<SiteSettings> siteSettings) : BaseController(routeConfigurationService, httpContextAccessor)
 {
     [EnableRateLimiting("content")]
     public IActionResult Index()
@@ -86,6 +90,15 @@ public IActionResult Guidelines()
         return View();
     }
 
+    [Route("/sitemap.xml")]
+    [OutputCache(Duration = 3600)]
+    [EnableRateLimiting("content")]
+    public IActionResult SitemapXml()
+    {
+        SitemapXmlHelpers.GenerateSitemapXml(siteMappingService.SiteMappings, RouteConfigurationService, siteSettings.Value.BaseUrl, out var nodes);
+        return new SitemapProvider().CreateSitemap(new SitemapModel(nodes));
+    }
+
     private string FlipPage(int currentChapter, int currentPage, bool next)
     {
         if (siteMappingService.SiteMappings.Count == 0)

EssentialCSharp.Web/Controllers/McpTokenController.cs

@@ -9,6 +9,7 @@ namespace EssentialCSharp.Web.Controllers;
 [Route("api/[controller]")]
 [Authorize]
 [ResponseCache(Location = ResponseCacheLocation.None, NoStore = true)]
+[IgnoreAntiforgeryToken]
 public class McpTokenController(McpApiTokenService tokenService) : ControllerBase
 {
     public record CreateTokenRequest(string? Name, DateOnly? ExpiresOn = null);

EssentialCSharp.Web/Dockerfile

@@ -1,6 +1,6 @@
-#syntax=docker/dockerfile:1.2
+# syntax=docker/dockerfile:1.4
 
-FROM mcr.microsoft.com/dotnet/aspnet:10.0-noble-chiseled AS base
+FROM mcr.microsoft.com/dotnet/aspnet:10.0-noble-chiseled-extra AS base
 WORKDIR /app
 EXPOSE 8080
 EXPOSE 8081
@@ -18,15 +18,19 @@ ENV ACCESS_TO_NUGET_FEED=$ACCESS_TO_NUGET_FEED
 WORKDIR /src
 COPY . .
 COPY --from=frontend-build /frontend/EssentialCSharp.Web/wwwroot/dist ./EssentialCSharp.Web/wwwroot/dist
-RUN --mount=type=secret,id=nugetconfig,required=false \
-    if [ "$ACCESS_TO_NUGET_FEED" = "true" ] && [ -f /run/secrets/nugetconfig ]; then \
-      mkdir -p ~/.nuget/config && \
-      cp /run/secrets/nugetconfig ~/.nuget/config/credentials.config; \
+RUN --mount=type=secret,id=nuget_pat,required=false \
+    if [ "$ACCESS_TO_NUGET_FEED" = "true" ] && [ ! -s /run/secrets/nuget_pat ]; then \
+      echo "ERROR: ACCESS_TO_NUGET_FEED=true but nuget_pat secret is missing or empty" >&2; exit 1; \
+    fi && \
+    if [ "$ACCESS_TO_NUGET_FEED" = "true" ]; then \
+      mkdir -p /root/.nuget/NuGet && \
+      printf '<?xml version="1.0" encoding="utf-8"?>\n<configuration>\n  <packageSourceCredentials>\n    <EssentialCSharp>\n      <add key="Username" value="az" />\n      <add key="ClearTextPassword" value="%s" />\n    </EssentialCSharp>\n  </packageSourceCredentials>\n</configuration>\n' \
+        "$(cat /run/secrets/nuget_pat)" > /root/.nuget/NuGet/NuGet.Config; \
     fi && \
     dotnet restore "EssentialCSharp.Web.slnx" -p:AccessToNugetFeed=$ACCESS_TO_NUGET_FEED && \
     dotnet build "EssentialCSharp.Web.slnx" -c Release --no-restore -p:AccessToNugetFeed=$ACCESS_TO_NUGET_FEED -p:ReleaseDateAttribute=True -p:SkipFrontendBuild=true && \
     dotnet publish "EssentialCSharp.Web.slnx" -c Release -p:PublishDir=/app/publish -p:UseAppHost=false -p:SkipFrontendBuild=true --no-build && \
-    rm -f ~/.nuget/config/credentials.config
+    rm -f /root/.nuget/NuGet/NuGet.Config
 
 FROM base AS final
 WORKDIR /app