Skip to content

Commit 31dbfdc

Browse files
BenjaminMichaelisBenjaminMichaelis
committed
Fix sitemap: exclude API and parameterized routes
Removes non-indexable routes from sitemap.xml generation: - Exclude all routes from API controllers ([ApiController] attribute) - Exclude routes with path parameters ({chapter}, {id:guid}, etc.) - Exclude routes with /api/ prefix - Keep only canonical, user-facing content pages Rationale: including /api/* and template routes wastes crawl budget, creates soft-404 signals, and can weaken site quality signals for search engines. Changes: - Add GetIndexableRoutes() to IRouteConfigurationService interface - Implement filtering in RouteConfigurationService: * IsApiController() - detects [ApiController] attribute * ContainsRouteParameters() - regex match for {param} patterns * Prefix check for /api/ routes - Update SitemapXmlHelpers to use GetIndexableRoutes() instead of GetStaticRoutes() - Keep GetStaticRoutes() unchanged for backward compatibility (navigation UI) - Add tests: API route exclusion, parameter route exclusion, content route inclusion
1 parent aea80c9 commit 31dbfdc

5 files changed

Lines changed: 212 additions & 5 deletions

File tree

‎EssentialCSharp.Web.Tests/RouteConfigurationServiceTests.cs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,79 @@ public async Task GetStaticRoutes_ShouldReturnExpectedRoutes()
3333
await Assert.That(routes).Contains("announcements");
3434
await Assert.That(routes).Contains("termsofservice");
3535
}
36+
37+
[Test]
38+
public async Task GetIndexableRoutes_ShouldExcludeApiControllerRoutes()
39+
{
40+
// Act
41+
var routes = _Factory.InServiceScope(serviceProvider =>
42+
{
43+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
44+
return routeConfigurationService.GetIndexableRoutes().ToList();
45+
});
46+
47+
// Assert - API routes should NOT be included
48+
await Assert.That(routes).DoesNotContain(route =>
49+
route.Contains("api/chat", StringComparison.OrdinalIgnoreCase));
50+
await Assert.That(routes).DoesNotContain(route =>
51+
route.Contains("api/listingsourcecode", StringComparison.OrdinalIgnoreCase));
52+
await Assert.That(routes).DoesNotContain(route =>
53+
route.Contains("api/mcptoken", StringComparison.OrdinalIgnoreCase));
54+
}
55+
56+
[Test]
57+
public async Task GetIndexableRoutes_ShouldExcludeParameterizedRoutes()
58+
{
59+
// Act
60+
var routes = _Factory.InServiceScope(serviceProvider =>
61+
{
62+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
63+
return routeConfigurationService.GetIndexableRoutes().ToList();
64+
});
65+
66+
// Assert - Routes with parameters should NOT be included
67+
await Assert.That(routes).DoesNotContain(route =>
68+
route.Contains('{'));
69+
await Assert.That(routes).DoesNotContain(route =>
70+
route.Contains("chapter", StringComparison.OrdinalIgnoreCase) &&
71+
route.Contains('{'));
72+
}
73+
74+
[Test]
75+
public async Task GetIndexableRoutes_ShouldIncludeValidContentRoutes()
76+
{
77+
// Act
78+
var routes = _Factory.InServiceScope(serviceProvider =>
79+
{
80+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
81+
return routeConfigurationService.GetIndexableRoutes().ToList();
82+
});
83+
84+
// Assert - Valid content routes should be included
85+
await Assert.That(routes).Contains("home");
86+
await Assert.That(routes).Contains("about");
87+
await Assert.That(routes).Contains("guidelines");
88+
await Assert.That(routes).Contains("announcements");
89+
await Assert.That(routes).Contains("termsofservice");
90+
}
91+
92+
[Test]
93+
public async Task GetStaticRoutes_StillReturnsAllRoutes_ForBackwardCompatibility()
94+
{
95+
// Act
96+
var staticRoutes = _Factory.InServiceScope(serviceProvider =>
97+
{
98+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
99+
return routeConfigurationService.GetStaticRoutes().ToList();
100+
});
101+
102+
var indexableRoutes = _Factory.InServiceScope(serviceProvider =>
103+
{
104+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
105+
return routeConfigurationService.GetIndexableRoutes().ToList();
106+
});
107+
108+
// Assert - Static routes should include more than indexable routes (API routes, parameterized routes)
109+
await Assert.That(staticRoutes.Count).IsGreaterThan(indexableRoutes.Count);
110+
}
36111
}

‎EssentialCSharp.Web.Tests/SitemapXmlHelpersTests.cs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,51 @@ public async Task EnsureSitemapHealthy_WithValidSiteMappings_DoesNotThrow()
3232
await Assert.That(() => SitemapXmlHelpers.EnsureSitemapHealthy(siteMappings)).ThrowsNothing();
3333
}
3434

35+
[Test]
36+
public async Task GenerateSitemapXml_DoesNotIncludeApiRoutes()
37+
{
38+
// Arrange
39+
var siteMappings = new List<SiteMapping> { CreateSiteMapping(1, 1, true) };
40+
var baseUrl = "https://test.example.com/";
41+
42+
// Act & Assert
43+
var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
44+
SitemapXmlHelpers.GenerateSitemapXml(
45+
siteMappings,
46+
routeConfigurationService,
47+
baseUrl,
48+
out var nodes);
49+
50+
var allUrls = nodes.Select(n => n.Url).ToList();
51+
52+
// Verify no API routes are included
53+
await Assert.That(allUrls).DoesNotContain(url => url.Contains("/api/", StringComparison.OrdinalIgnoreCase));
54+
await Assert.That(allUrls).DoesNotContain(url => url.Contains("chat", StringComparison.OrdinalIgnoreCase));
55+
await Assert.That(allUrls).DoesNotContain(url => url.Contains("listingsourcecode", StringComparison.OrdinalIgnoreCase));
56+
await Assert.That(allUrls).DoesNotContain(url => url.Contains("mcptoken", StringComparison.OrdinalIgnoreCase));
57+
}
58+
59+
[Test]
60+
public async Task GenerateSitemapXml_DoesNotIncludeParameterizedRoutes()
61+
{
62+
// Arrange
63+
var siteMappings = new List<SiteMapping> { CreateSiteMapping(1, 1, true) };
64+
var baseUrl = "https://test.example.com/";
65+
66+
// Act & Assert
67+
var routeConfigurationService = _Factory.Services.GetRequiredService<IRouteConfigurationService>();
68+
SitemapXmlHelpers.GenerateSitemapXml(
69+
siteMappings,
70+
routeConfigurationService,
71+
baseUrl,
72+
out var nodes);
73+
74+
var allUrls = nodes.Select(n => n.Url).ToList();
75+
76+
// Verify no parameterized routes (with {}) are included
77+
await Assert.That(allUrls).DoesNotContain(url => url.Contains('{'));
78+
}
79+
3580
[Test]
3681
public async Task EnsureSitemapHealthy_WithMultipleCanonicalLinksForSamePage_ThrowsException()
3782
{

‎EssentialCSharp.Web/Helpers/SitemapXmlHelpers.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,9 @@ public static void GenerateSitemapXml(IEnumerable<SiteMapping> siteMappings, IRo
3232
}
3333
};
3434

35-
// Add routes dynamically discovered from controllers
36-
var allRoutes = routeConfigurationService.GetStaticRoutes();
35+
// Add routes dynamically discovered from controllers (only indexable routes)
36+
var allRoutes = routeConfigurationService.GetIndexableRoutes();
3737
var controllerRoutes = allRoutes
38-
.Where(route => !route.Contains("error", StringComparison.OrdinalIgnoreCase))
39-
.Where(route => !route.Contains("index", StringComparison.OrdinalIgnoreCase))
40-
.Where(route => !route.Contains("identity", StringComparison.OrdinalIgnoreCase))
4138
.Where(route => !IsSitemapRoute(route))
4239
.Select(route => $"/{route}")
4340
.ToList();

‎EssentialCSharp.Web/Services/IRouteConfigurationService.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ namespace EssentialCSharp.Web.Services;
33
public interface IRouteConfigurationService
44
{
55
IReadOnlySet<string> GetStaticRoutes();
6+
IReadOnlySet<string> GetIndexableRoutes();
67
}

‎EssentialCSharp.Web/Services/RouteConfigurationService.cs‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,33 @@
1+
using Microsoft.AspNetCore.Mvc;
2+
using Microsoft.AspNetCore.Mvc.Abstractions;
13
using Microsoft.AspNetCore.Mvc.Infrastructure;
4+
using System.Text.RegularExpressions;
25

36
namespace EssentialCSharp.Web.Services;
47

58
public class RouteConfigurationService : IRouteConfigurationService
69
{
710
private readonly IActionDescriptorCollectionProvider _ActionDescriptorCollectionProvider;
811
private readonly HashSet<string> _StaticRoutes;
12+
private readonly HashSet<string> _IndexableRoutes;
913

1014
public RouteConfigurationService(IActionDescriptorCollectionProvider actionDescriptorCollectionProvider)
1115
{
1216
_ActionDescriptorCollectionProvider = actionDescriptorCollectionProvider;
1317
_StaticRoutes = ExtractStaticRoutes();
18+
_IndexableRoutes = ExtractIndexableRoutes();
1419
}
1520

1621
public IReadOnlySet<string> GetStaticRoutes()
1722
{
1823
return _StaticRoutes;
1924
}
2025

26+
public IReadOnlySet<string> GetIndexableRoutes()
27+
{
28+
return _IndexableRoutes;
29+
}
30+
2131
private HashSet<string> ExtractStaticRoutes()
2232
{
2333
var routes = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
@@ -59,4 +69,83 @@ private HashSet<string> ExtractStaticRoutes()
5969

6070
return routes;
6171
}
72+
73+
private HashSet<string> ExtractIndexableRoutes()
74+
{
75+
var indexableRoutes = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
76+
77+
// Get all action descriptors
78+
var actionDescriptors = _ActionDescriptorCollectionProvider.ActionDescriptors.Items;
79+
80+
foreach (var actionDescriptor in actionDescriptors)
81+
{
82+
// Skip if controller is marked with [ApiController]
83+
if (IsApiController(actionDescriptor))
84+
continue;
85+
86+
// Look for route attributes
87+
if (actionDescriptor.AttributeRouteInfo?.Template != null)
88+
{
89+
string template = actionDescriptor.AttributeRouteInfo.Template;
90+
91+
// Skip routes with parameters (e.g., {chapter}, {id:guid}, [optional])
92+
if (ContainsRouteParameters(template))
93+
continue;
94+
95+
// Skip routes starting with /api/
96+
if (template.StartsWith("/api/", StringComparison.OrdinalIgnoreCase))
97+
continue;
98+
99+
// Remove leading slash and add to our set
100+
string routePath = template.TrimStart('/').ToLowerInvariant();
101+
indexableRoutes.Add(routePath);
102+
}
103+
104+
// Skip the default fallback route (Index action in HomeController)
105+
if (actionDescriptor.RouteValues.TryGetValue("action", out var action) && action == "Index")
106+
continue;
107+
108+
// Skip Error actions
109+
if (action == "Error")
110+
continue;
111+
112+
// For actions without attribute routes, use conventional routing
113+
if (actionDescriptor.AttributeRouteInfo?.Template == null &&
114+
actionDescriptor.RouteValues.TryGetValue("action", out var actionName) &&
115+
actionDescriptor.RouteValues.TryGetValue("controller", out var controllerName) &&
116+
controllerName?.Equals("Home", StringComparison.OrdinalIgnoreCase) == true &&
117+
actionName != null)
118+
{
119+
// Use the action name directly as the route
120+
indexableRoutes.Add(actionName.ToLowerInvariant());
121+
}
122+
}
123+
124+
return indexableRoutes;
125+
}
126+
127+
private static bool IsApiController(ActionDescriptor actionDescriptor)
128+
{
129+
// Check for [ApiController] attribute
130+
if (actionDescriptor.EndpointMetadata?.OfType<ApiControllerAttribute>().Any() == true)
131+
return true;
132+
133+
// Check if controller inherits from ControllerBase (not Controller)
134+
if (actionDescriptor.RouteValues.TryGetValue("controller", out var controllerName))
135+
{
136+
// Known API controllers
137+
var apiControllers = new[] { "ListingSourceCode", "Chat", "McpToken", "MCP" };
138+
if (apiControllers.Contains(controllerName, StringComparer.OrdinalIgnoreCase))
139+
return true;
140+
}
141+
142+
return false;
143+
}
144+
145+
private static bool ContainsRouteParameters(string template)
146+
{
147+
// Match {param}, {param:constraint}, [optional], etc.
148+
return Regex.IsMatch(template, @"\{[^}]+\}|\[[^\]]+\]");
149+
}
62150
}
151+

0 commit comments

Comments
 (0)