fix: address valid review comments from PR 1074 (#1075)

## Summary
- evaluated unresolved review comments from #1074 and applied fixes for
the valid ones
- updated sitemap route filtering to exclude only the `/sitemap.xml`
route template
- stopped using per-request `UtcNow` as sitemap `lastmod` fallback when
content has no timestamp
- moved `UseOutputCache()` to run after
authentication/authorization/rate-limiting middleware
- renamed sitemap startup validation log methods to match their behavior
- tightened sitemap exclusion assertion and added null-`LastModified`
coverage

## Validation
- `dotnet restore /p:AccessToNugetFeed=false`
- `dotnet build --configuration Release --no-restore
/p:AccessToNugetFeed=false` ✅
- `dotnet test --project
EssentialCSharp.Web.Tests/EssentialCSharp.Web.Tests.csproj --no-build
--configuration Release` ⚠️ fails in existing integration tests
(`CaptchaService_Verify_Success`, `KnownBreachedPassword_IsDetected`)
unrelated to this change
- `parallel_validation` code review ✅
- `parallel_validation` CodeQL timed out after one successful run
earlier in session (0 alerts), then timed out on rerun after final tiny
polish

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: BenjaminMichaelis <22186029+BenjaminMichaelis@users.noreply.github.com>

Author: Copilot

EssentialCSharp.Web.Tests/SitemapXmlHelpersTests.cs

@@ -205,7 +205,7 @@ public async Task GenerateSitemapXml_DoesNotIncludeSitemapRoute()
         var allUrls = nodes.Select(n => n.Url).ToList();
 
         // /sitemap.xml should not list itself
-        await Assert.That(allUrls).DoesNotContain(url => url.Contains("sitemap", StringComparison.OrdinalIgnoreCase));
+        await Assert.That(allUrls).DoesNotContain(url => url.EndsWith("/sitemap.xml", StringComparison.OrdinalIgnoreCase));
     }
 
     [Test]
@@ -233,6 +233,29 @@ public async Task GenerateSitemapXml_UsesLastModifiedDateFromSiteMapping()
         await Assert.That(siteMappingNode.LastModificationDate).IsEqualTo(specificLastModified);
     }
 
+    [Test]
+    public async Task GenerateSitemapXml_DoesNotSetLastModifiedDateWhenSiteMappingDateIsMissing()
+    {
+        // Arrange
+        var baseUrl = "https://test.example.com/";
+        var siteMappings = new List<SiteMapping>
+        {
+            CreateSiteMapping(1, 1, true, "test-page-1")
+        };
+
+        // Act
+        var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
+        SitemapXmlHelpers.GenerateSitemapXml(
+            siteMappings,
+            routeConfigurationService,
+            baseUrl,
+            out var nodes);
+
+        // Assert
+        var siteMappingNode = nodes.First(node => node.Url.Contains("test-page-1"));
+        await Assert.That(siteMappingNode.LastModificationDate).IsNull();
+    }
+
     private static SiteMapping CreateSiteMapping(
         int chapterNumber,
         int pageNumber,

EssentialCSharp.Web/Helpers/SitemapXmlHelpers.cs

@@ -20,8 +20,6 @@ public static void EnsureSitemapHealthy(List<SiteMapping> siteMappings)
 
     public static void GenerateSitemapXml(IEnumerable<SiteMapping> siteMappings, IRouteConfigurationService routeConfigurationService, string baseUrl, out List<SitemapNode> nodes)
     {
-        DateTime newDateTime = DateTime.UtcNow;
-
         // Routes should end up with leading slash
         baseUrl = baseUrl.TrimEnd('/');
 
@@ -40,7 +38,7 @@ public static void GenerateSitemapXml(IEnumerable<SiteMapping> siteMappings, IRo
             .Where(route => !route.Contains("error", StringComparison.OrdinalIgnoreCase))
             .Where(route => !route.Contains("index", StringComparison.OrdinalIgnoreCase))
             .Where(route => !route.Contains("identity", StringComparison.OrdinalIgnoreCase))
-            .Where(route => !route.Contains("sitemap", StringComparison.OrdinalIgnoreCase))
+            .Where(route => !IsSitemapRoute(route))
             .Select(route => $"/{route}")
             .ToList();
 
@@ -54,14 +52,26 @@ public static void GenerateSitemapXml(IEnumerable<SiteMapping> siteMappings, IRo
         }
 
         // Add site mappings from content
-        nodes.AddRange(siteMappings.Where(item => item.IncludeInSitemapXml).Select<SiteMapping, SitemapNode>(siteMapping => new($"{baseUrl.TrimEnd('/')}/{siteMapping.Keys.First()}")
+        nodes.AddRange(siteMappings.Where(item => item.IncludeInSitemapXml).Select(siteMapping =>
         {
-            LastModificationDate = siteMapping.LastModified ?? newDateTime,
-            ChangeFrequency = ChangeFrequency.Daily,
-            Priority = 0.8M
+            SitemapNode node = new($"{baseUrl.TrimEnd('/')}/{siteMapping.Keys.First()}")
+            {
+                ChangeFrequency = ChangeFrequency.Daily,
+                Priority = 0.8M
+            };
+
+            if (siteMapping.LastModified is DateTime lastModified)
+            {
+                node.LastModificationDate = lastModified;
+            }
+
+            return node;
         }));
     }
 
+    private static bool IsSitemapRoute(string route) =>
+        route.TrimStart('/').Equals("sitemap.xml", StringComparison.OrdinalIgnoreCase);
+
     private static ChangeFrequency GetChangeFrequencyForRoute(string route)
     {
         return route.ToLowerInvariant() switch

EssentialCSharp.Web/Program.cs

@@ -505,8 +505,7 @@ await McpJsonRpcResponseWriter.WriteErrorAsync(
         }
         app.UseStaticFiles();
 
-        app.UseRouting();
-        app.UseOutputCache();
+        app.UseRouting();
 
         app.UseWhen(
             context => context.Request.Path.StartsWithSegments("/mcp"),
@@ -537,9 +536,10 @@ await McpJsonRpcResponseWriter.WriteErrorAsync(
                 await next(context);
             }));
 
-        app.UseRateLimiter();
-
-        app.UseAuthorization();
+        app.UseRateLimiter();
+
+        app.UseAuthorization();
+        app.UseOutputCache();
 
         app.UseMiddleware<ReferralMiddleware>();
 
@@ -578,13 +578,13 @@ await McpJsonRpcResponseWriter.WriteErrorAsync(
         try
         {
             SitemapXmlHelpers.EnsureSitemapHealthy(siteMappingService.SiteMappings.ToList());
-            LogSitemapGenerationSucceeded(logger);
-        }
-        catch (Exception ex)
-        {
-            LogSitemapGenerationFailed(logger, ex);
-            // Continue startup even if sitemap validation fails
-        }
+            LogSitemapValidationSucceeded(logger);
+        }
+        catch (Exception ex)
+        {
+            LogSitemapValidationFailed(logger, ex);
+            // Continue startup even if sitemap validation fails
+        }
 
         app.Run();
     }
@@ -598,11 +598,11 @@ private static bool IsMcpTransportRequest(HttpRequest request) =>
     [LoggerMessage(Level = LogLevel.Error, Message = "Unhandled exception on {Path}")]
     private static partial void LogUnhandledException(ILogger<Program> logger, Exception? exception, PathString path);
 
-    [LoggerMessage(Level = LogLevel.Information, Message = "Sitemap validation completed successfully during application startup")]
-    private static partial void LogSitemapGenerationSucceeded(ILogger<Program> logger);
-
-    [LoggerMessage(Level = LogLevel.Error, Message = "Failed to validate sitemap during application startup")]
-    private static partial void LogSitemapGenerationFailed(ILogger<Program> logger, Exception exception);
+    [LoggerMessage(Level = LogLevel.Information, Message = "Sitemap validation completed successfully during application startup")]
+    private static partial void LogSitemapValidationSucceeded(ILogger<Program> logger);
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "Failed to validate sitemap during application startup")]
+    private static partial void LogSitemapValidationFailed(ILogger<Program> logger, Exception exception);
 
     [LoggerMessage(Level = LogLevel.Warning, Message = "Ignoring invalid TryDotNet origin in CSP: {Origin}")]
     private static partial void LogIgnoringInvalidTryDotNetOrigin(ILogger logger, string origin);