Skip to content

Commit a8757c4

Browse files
BenjaminMichaelisBenjaminMichaelis
authored
fix: sitemap exclude API and parameterized routes (#1132)
## Motivation The sitemap currently includes non-indexable routes such as `/api/listingsourcecode/*`, `/api/chat/*`, and parameterized templates like `/api/listingsourcecode/chapter/{chapter}/listing/{listing}`. These are API endpoints that return JSON or template routes with unfillable placeholders -- not real user-facing pages. Including them wastes crawl budget, creates soft-404 signals, and can weaken site quality signals for search engines. ## Approach Adds `GetIndexableRoutes()` to `IRouteConfigurationService` alongside the existing `GetStaticRoutes()`. The implementation derives indexable routes from the already-computed static route set using three string-level filters: 1. **`/api/` prefix** -- drops all API-prefixed routes 2. **Route parameter regex** -- drops routes containing `{param}` or `{param:constraint}` syntax (cached as `static readonly` to avoid repeated construction) 3. **Identity area** -- drops routes containing `identity` (Razor Pages under `/Identity/Account/*`) `GetStaticRoutes()` is unchanged for backward compatibility (used by navigation UI). `SitemapXmlHelpers.GenerateSitemapXml()` now calls `GetIndexableRoutes()`, and redundant inline filters are removed. ## Tests added - `GetIndexableRoutes_ShouldExcludeApiControllerRoutes` - `GetIndexableRoutes_ShouldExcludeParameterizedRoutes` - `GetIndexableRoutes_ShouldIncludeValidContentRoutes` (including `mcp-setup`) - `GetStaticRoutes_StillReturnsAllRoutes_ForBackwardCompatibility` - `GenerateSitemapXml_DoesNotIncludeApiRoutes` (asserts on `/api/` prefix pattern) - `GenerateSitemapXml_DoesNotIncludeParameterizedRoutes`
1 parent 391cea6 commit a8757c4

6 files changed

Lines changed: 167 additions & 6 deletions

File tree

‎EssentialCSharp.Web.Tests/RouteConfigurationServiceTests.cs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,77 @@ public async Task GetStaticRoutes_ShouldReturnExpectedRoutes()
2525
await Assert.That(routes).Contains("announcements");
2626
await Assert.That(routes).Contains("termsofservice");
2727
}
28+
29+
[Test]
30+
public async Task GetIndexableRoutes_ShouldExcludeApiControllerRoutes()
31+
{
32+
// Act
33+
var routes = InServiceScope(serviceProvider =>
34+
{
35+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
36+
return routeConfigurationService.GetIndexableRoutes().ToList();
37+
});
38+
39+
// Assert - no indexable route should start with api/ (matches the actual filter behavior)
40+
await Assert.That(routes).DoesNotContain(route =>
41+
route.StartsWith("api/", StringComparison.OrdinalIgnoreCase));
42+
}
43+
44+
[Test]
45+
public async Task GetIndexableRoutes_ShouldExcludeParameterizedRoutes()
46+
{
47+
// Act
48+
var routes = InServiceScope(serviceProvider =>
49+
{
50+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
51+
return routeConfigurationService.GetIndexableRoutes().ToList();
52+
});
53+
54+
// Assert - Routes with parameters should NOT be included
55+
await Assert.That(routes).DoesNotContain(route =>
56+
route.Contains('{'));
57+
await Assert.That(routes).DoesNotContain(route =>
58+
route.Contains("chapter", StringComparison.OrdinalIgnoreCase) &&
59+
route.Contains('{'));
60+
}
61+
62+
[Test]
63+
public async Task GetIndexableRoutes_ShouldIncludeValidContentRoutes()
64+
{
65+
// Act
66+
var routes = InServiceScope(serviceProvider =>
67+
{
68+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
69+
return routeConfigurationService.GetIndexableRoutes().ToList();
70+
});
71+
72+
// Assert - Valid content routes should be included
73+
await Assert.That(routes).Contains("home");
74+
await Assert.That(routes).Contains("about");
75+
await Assert.That(routes).Contains("guidelines");
76+
await Assert.That(routes).Contains("announcements");
77+
await Assert.That(routes).Contains("termsofservice");
78+
await Assert.That(routes).Contains("mcp-setup");
79+
}
80+
81+
[Test]
82+
public async Task GetStaticRoutes_StillReturnsAllRoutes_ForBackwardCompatibility()
83+
{
84+
// Act
85+
var staticRoutes = InServiceScope(serviceProvider =>
86+
{
87+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
88+
return routeConfigurationService.GetStaticRoutes().ToList();
89+
});
90+
91+
var indexableRoutes = InServiceScope(serviceProvider =>
92+
{
93+
var routeConfigurationService = serviceProvider.GetRequiredService<IRouteConfigurationService>();
94+
return routeConfigurationService.GetIndexableRoutes().ToList();
95+
});
96+
97+
// Assert - Static routes should include more than indexable routes (API routes, parameterized routes)
98+
await Assert.That(staticRoutes.Count).IsGreaterThan(indexableRoutes.Count);
99+
}
28100
}
101+

‎EssentialCSharp.Web.Tests/RouteParameterFilterTests.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
using EssentialCSharp.Web.Services;
2+
3+
namespace EssentialCSharp.Web.Tests;
4+
5+
public class RouteParameterFilterTests
6+
{
7+
[Test]
8+
[Arguments("{chapter}")]
9+
[Arguments("{id:guid}")]
10+
[Arguments("{id?}")]
11+
[Arguments("{action=Index}")]
12+
[Arguments("{*catchall}")]
13+
[Arguments("chapter/{chapter}/listing/{listing}")]
14+
[Arguments("[optional]")]
15+
[Arguments("area/[optional]/page")]
16+
public async Task RouteParameterRegex_MatchesParameterizedRoutes(string route)
17+
=> await Assert.That(RouteConfigurationService.RouteParameterRegex().IsMatch(route)).IsTrue();
18+
19+
[Test]
20+
[Arguments("about")]
21+
[Arguments("mcp-setup")]
22+
[Arguments("identity/account/login")]
23+
[Arguments("api/listing")]
24+
[Arguments("")]
25+
public async Task RouteParameterRegex_DoesNotMatchStaticRoutes(string route)
26+
=> await Assert.That(RouteConfigurationService.RouteParameterRegex().IsMatch(route)).IsFalse();
27+
}

‎EssentialCSharp.Web.Tests/SitemapXmlHelpersTests.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,48 @@ public async Task EnsureSitemapHealthy_WithValidSiteMappings_DoesNotThrow()
2222
await Assert.That(() => SitemapXmlHelpers.EnsureSitemapHealthy(siteMappings)).ThrowsNothing();
2323
}
2424

25+
[Test]
26+
public async Task GenerateSitemapXml_DoesNotIncludeApiRoutes()
27+
{
28+
// Arrange
29+
var siteMappings = new List<SiteMapping> { CreateSiteMapping(1, 1, true) };
30+
var baseUrl = "https://test.example.com/";
31+
32+
// Act & Assert
33+
var routeConfigurationService = Factory.Services.GetRequiredService<IRouteConfigurationService>();
34+
SitemapXmlHelpers.GenerateSitemapXml(
35+
siteMappings,
36+
routeConfigurationService,
37+
baseUrl,
38+
out var nodes);
39+
40+
var allUrls = nodes.Select(n => n.Url).ToList();
41+
42+
// Verify no API routes are included (assert on the /api/ pattern, not specific controller names)
43+
await Assert.That(allUrls).DoesNotContain(url => url.Contains("/api/", StringComparison.OrdinalIgnoreCase));
44+
}
45+
46+
[Test]
47+
public async Task GenerateSitemapXml_DoesNotIncludeParameterizedRoutes()
48+
{
49+
// Arrange
50+
var siteMappings = new List<SiteMapping> { CreateSiteMapping(1, 1, true) };
51+
var baseUrl = "https://test.example.com/";
52+
53+
// Act & Assert
54+
var routeConfigurationService = Factory.Services.GetRequiredService<IRouteConfigurationService>();
55+
SitemapXmlHelpers.GenerateSitemapXml(
56+
siteMappings,
57+
routeConfigurationService,
58+
baseUrl,
59+
out var nodes);
60+
61+
var allUrls = nodes.Select(n => n.Url).ToList();
62+
63+
// Verify no parameterized routes (with {}) are included
64+
await Assert.That(allUrls).DoesNotContain(url => url.Contains('{'));
65+
}
66+
2567
[Test]
2668
public async Task EnsureSitemapHealthy_WithMultipleCanonicalLinksForSamePage_ThrowsException()
2769
{

‎EssentialCSharp.Web/Helpers/SitemapXmlHelpers.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,9 @@ public static void GenerateSitemapXml(IEnumerable<SiteMapping> siteMappings, IRo
3232
}
3333
};
3434

35-
// Add routes dynamically discovered from controllers
36-
var allRoutes = routeConfigurationService.GetStaticRoutes();
35+
// Add routes dynamically discovered from controllers (only indexable routes)
36+
var allRoutes = routeConfigurationService.GetIndexableRoutes();
3737
var controllerRoutes = allRoutes
38-
.Where(route => !route.Contains("error", StringComparison.OrdinalIgnoreCase))
39-
.Where(route => !route.Contains("index", StringComparison.OrdinalIgnoreCase))
40-
.Where(route => !route.Contains("identity", StringComparison.OrdinalIgnoreCase))
4138
.Where(route => !IsSitemapRoute(route))
4239
.Select(route => $"/{route}")
4340
.ToList();

‎EssentialCSharp.Web/Services/IRouteConfigurationService.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ namespace EssentialCSharp.Web.Services;
33
public interface IRouteConfigurationService
44
{
55
IReadOnlySet<string> GetStaticRoutes();
6+
IReadOnlySet<string> GetIndexableRoutes();
67
}

‎EssentialCSharp.Web/Services/RouteConfigurationService.cs‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,34 @@
11
using Microsoft.AspNetCore.Mvc.Infrastructure;
2+
using System.Text.RegularExpressions;
23

34
namespace EssentialCSharp.Web.Services;
45

5-
public class RouteConfigurationService : IRouteConfigurationService
6+
public partial class RouteConfigurationService : IRouteConfigurationService
67
{
8+
[GeneratedRegex(@"\{[^}]+\}|\[[^\]]+\]")]
9+
public static partial Regex RouteParameterRegex();
10+
711
private readonly IActionDescriptorCollectionProvider _ActionDescriptorCollectionProvider;
812
private readonly HashSet<string> _StaticRoutes;
13+
private readonly HashSet<string> _IndexableRoutes;
914

1015
public RouteConfigurationService(IActionDescriptorCollectionProvider actionDescriptorCollectionProvider)
1116
{
1217
_ActionDescriptorCollectionProvider = actionDescriptorCollectionProvider;
1318
_StaticRoutes = ExtractStaticRoutes();
19+
_IndexableRoutes = ExtractIndexableRoutes();
1420
}
1521

1622
public IReadOnlySet<string> GetStaticRoutes()
1723
{
1824
return _StaticRoutes;
1925
}
2026

27+
public IReadOnlySet<string> GetIndexableRoutes()
28+
{
29+
return _IndexableRoutes;
30+
}
31+
2132
private HashSet<string> ExtractStaticRoutes()
2233
{
2334
var routes = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
@@ -59,4 +70,14 @@ private HashSet<string> ExtractStaticRoutes()
5970

6071
return routes;
6172
}
73+
74+
private HashSet<string> ExtractIndexableRoutes()
75+
{
76+
return _StaticRoutes
77+
.Where(route => !route.StartsWith("api/", StringComparison.OrdinalIgnoreCase))
78+
.Where(route => !RouteParameterRegex().IsMatch(route))
79+
.Where(route => !route.Contains("identity", StringComparison.OrdinalIgnoreCase))
80+
.ToHashSet(StringComparer.OrdinalIgnoreCase);
81+
}
6282
}
83+

0 commit comments

Comments
 (0)