Surface deprecated prebuilt detection rules as a sibling nav page (#3167)

Co-authored-by: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Felipe Cotti <felipe.cotti@elastic.co>
Co-authored-by: elastic-observability-automation[bot] <180520183+elastic-observability-automation[bot]@users.noreply.github.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: snyk-bot <snyk-bot@snyk.io>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Martijn Laarman <Mpdreamz@gmail.com>
Co-authored-by: Fabrizio Ferri-Benedetti <fabri.ferribenedetti@elastic.co>
Co-authored-by: Lisa Cawley <lcawley@elastic.co>
Co-authored-by: shainaraskas <58563081+shainaraskas@users.noreply.github.com>
Co-authored-by: Nassim Kammah <nkammah@gmail.com>
Co-authored-by: Janeen Mikell Roberts <57149392+jmikell821@users.noreply.github.com>
Co-authored-by: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com>
Co-authored-by: Jan Calanog <jan.calanog@elastic.co>
Co-authored-by: Martijn Laarman <ml@elastic.co>
Signed-off-by: dependabot[bot] <support@github.com>

Author: 17 people

Directory.Packages.props

@@ -57,6 +57,7 @@
     <PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.7.1" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
     <PackageVersion Include="Microsoft.OpenApi" Version="3.1.1" />
+    <PackageVersion Include="Tomlyn" Version="2.3.2" />
     <PackageVersion Include="TUnit" Version="0.25.21" />
     <PackageVersion Include="xunit.v3.extensibility.core" Version="2.0.2" />
     <PackageVersion Include="WireMock.Net" Version="1.6.11" />

src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj

@@ -21,7 +21,7 @@
     <PackageReference Include="NetEscapades.EnumGenerators" />
     <PackageReference Include="Nullean.Argh.Interfaces" />
     <PackageReference Include="Nullean.ScopedFileSystem" />
-    <PackageReference Include="Samboy063.Tomlet" />
+    <PackageReference Include="Tomlyn" />
     <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" />
     <PackageReference Include="YamlDotNet" />
   </ItemGroup>

src/Elastic.Documentation.Configuration/Toc/DetectionRules/DetectionRuleOverviewRef.cs

@@ -10,19 +10,30 @@ public record DetectionRuleOverviewRef : FileRef
 {
 	public IReadOnlyCollection<string> DetectionRuleFolders { get; }
 
+	/// <summary>Optional path to a markdown file whose content prefixes the deprecated rules listing page.</summary>
+	public string? DeprecatedFile { get; init; }
+
+	/// <summary>
+	/// The resolved deprecated-rules overview FileRef that should appear as a sibling to this ref in the nav.
+	/// Set by <c>ResolveRuleOverviewReference</c> when a <c>_deprecated</c> subfolder is detected.
+	/// </summary>
+	public FileRef? DeprecatedSiblingRef { get; init; }
+
 	public DetectionRuleOverviewRef(
 		string pathRelativeToDocumentationSet,
 		string pathRelativeToContainer,
 		IReadOnlyCollection<string> detectionRulesFolders,
 		IReadOnlyCollection<ITableOfContentsItem> children,
-		string context
+		string context,
+		string? deprecatedFile = null
 	) : base(pathRelativeToDocumentationSet, pathRelativeToContainer, false, children, context)
 	{
 		PathRelativeToDocumentationSet = pathRelativeToDocumentationSet;
 		PathRelativeToContainer = pathRelativeToContainer;
 		DetectionRuleFolders = detectionRulesFolders;
 		Children = children;
 		Context = context;
+		DeprecatedFile = deprecatedFile;
 	}
 
 	public static IReadOnlyCollection<ITableOfContentsItem> CreateTableOfContentItems(IReadOnlyCollection<IDirectoryInfo> sourceFolders, string context, IDirectoryInfo baseDirectory)
@@ -38,6 +49,18 @@ public static IReadOnlyCollection<ITableOfContentsItem> CreateTableOfContentItem
 			.ToArray();
 	}
 
+	public static IReadOnlyCollection<ITableOfContentsItem> CreateDeprecatedTableOfContentItems(IReadOnlyCollection<IDirectoryInfo> sourceFolders, string context, IDirectoryInfo baseDirectory)
+	{
+		var tocItems = new List<ITableOfContentsItem>();
+		foreach (var detectionRuleFolder in sourceFolders)
+		{
+			var children = ReadDeprecatedDetectionRuleFolder(detectionRuleFolder, context, baseDirectory);
+			tocItems.AddRange(children);
+		}
+
+		return tocItems.ToArray();
+	}
+
 	private static IReadOnlyCollection<ITableOfContentsItem> ReadDetectionRuleFolder(IDirectoryInfo directory, string context, IDirectoryInfo baseDirectory)
 	{
 		IReadOnlyCollection<ITableOfContentsItem> children = directory
@@ -62,4 +85,25 @@ private static IReadOnlyCollection<ITableOfContentsItem> ReadDetectionRuleFolder
 
 		return children;
 	}
+
+	private static IReadOnlyCollection<ITableOfContentsItem> ReadDeprecatedDetectionRuleFolder(IDirectoryInfo directory, string context, IDirectoryInfo baseDirectory)
+	{
+		IReadOnlyCollection<ITableOfContentsItem> children = directory
+			.EnumerateFiles("*.*", SearchOption.AllDirectories)
+			.Where(f => !f.Attributes.HasFlag(FileAttributes.Hidden) && !f.Attributes.HasFlag(FileAttributes.System))
+			.Where(f => !f.Directory!.Attributes.HasFlag(FileAttributes.Hidden) && !f.Directory!.Attributes.HasFlag(FileAttributes.System))
+			// skip symlinks
+			.Where(f => f.LinkTarget == null)
+			.Where(f => f.Extension == ".toml")
+			// only include files inside _deprecated subdirectories
+			.Where(f => f.FullName.Contains($"{Path.DirectorySeparatorChar}_deprecated{Path.DirectorySeparatorChar}"))
+			.Select(f =>
+			{
+				var relativePath = Path.GetRelativePath(baseDirectory.Parent!.FullName, f.FullName);
+				return (ITableOfContentsItem)new DetectionRuleRef(f, relativePath, context);
+			})
+			.ToArray();
+
+		return children;
+	}
 }

src/Elastic.Documentation.Configuration/Toc/DocumentationSetFile.cs

@@ -174,7 +174,12 @@ private static TableOfContents ResolveTableOfContents(
 			};
 
 			if (resolvedItem != null)
+			{
 				resolved.Add(resolvedItem);
+				// Emit the deprecated rules overview as a sibling immediately after the active rules ref
+				if (resolvedItem is DetectionRuleOverviewRef { DeprecatedSiblingRef: { } deprecatedSibling })
+					resolved.Add(deprecatedSibling);
+			}
 		}
 
 		return resolved;
@@ -453,9 +458,33 @@ private static ITableOfContentsItem ResolveRuleOverviewReference(IDiagnosticsCol
 			.ToList();
 		var tomlChildren = DetectionRuleOverviewRef.CreateTableOfContentItems(tocSourceFolders, context, baseDirectory);
 
-		var children = resolvedChildren.Concat(tomlChildren).ToList();
+		var children = resolvedChildren.ToList();
+		children.AddRange(tomlChildren);
 
-		return new DetectionRuleOverviewRef(fullPath, pathRelativeToContainer, detectionRuleRef.DetectionRuleFolders, children, context);
+		// Auto-detect _deprecated subdirectories. When found, build the deprecated overview FileRef
+		// and attach it as DeprecatedSiblingRef so ResolveTableOfContents can emit it as a sibling,
+		// not as a child nested under the active rules.
+		FileRef? deprecatedSiblingRef = null;
+		var hasDeprecatedRules = tocSourceFolders.Any(d =>
+			d.Exists && d.EnumerateDirectories("_deprecated", SearchOption.TopDirectoryOnly).Any());
+		if (hasDeprecatedRules)
+		{
+			var deprecatedFileName = detectionRuleRef.DeprecatedFile ?? "deprecated-detection-rules.md";
+			var overviewDir = fileSystem.Path.GetDirectoryName(fullPath);
+			var deprecatedFullPath = string.IsNullOrEmpty(overviewDir)
+				? deprecatedFileName
+				: $"{overviewDir}/{deprecatedFileName}";
+			var deprecatedPathRelativeToContainer = string.IsNullOrEmpty(containerPath)
+				? deprecatedFullPath
+				: deprecatedFullPath.Substring(containerPath.Length + 1);
+			var deprecatedTomlChildren = DetectionRuleOverviewRef.CreateDeprecatedTableOfContentItems(tocSourceFolders, context, baseDirectory);
+			deprecatedSiblingRef = new FileRef(deprecatedFullPath, deprecatedPathRelativeToContainer, false, deprecatedTomlChildren, context);
+		}
+
+		return new DetectionRuleOverviewRef(fullPath, pathRelativeToContainer, detectionRuleRef.DetectionRuleFolders, children, context, detectionRuleRef.DeprecatedFile)
+		{
+			DeprecatedSiblingRef = deprecatedSiblingRef
+		};
 	}
 
 

src/Elastic.Documentation.Configuration/Toc/TableOfContentsYamlConverters.cs

@@ -73,7 +73,7 @@ public class TocItemYamlConverter : IYamlTypeConverter
 					}
 					value = childrenList;
 				}
-				else if (key.Value is "detection_rules" or "exclude")
+				else if (key.Value is "detection_rules" or "exclude" or "deprecated_detection_rules")
 				{
 					// Parse the children list manually
 					var childrenList = new List<string>();
@@ -144,11 +144,8 @@ public class TocItemYamlConverter : IYamlTypeConverter
 		if (dictionary.TryGetValue("detection_rules", out var detectionRulesObj) && detectionRulesObj is string[] detectionRulesFolders &&
 			dictionary.TryGetValue("file", out var detectionRulesFilePath) && detectionRulesFilePath is string detectionRulesFile)
 		{
-			// Create the index file reference (FolderIndexFileRef to mark it as the folder's index)
-			// Store ONLY the file name - the folder path will be prepended during resolution
-			// This allows validation to check if the file itself has deep paths
-			// PathRelativeToContainer will be set during resolution
-			return new DetectionRuleOverviewRef(detectionRulesFile, detectionRulesFile, detectionRulesFolders, children, placeholderContext);
+			var deprecatedFile = dictionary.TryGetValue("deprecated_file", out var deprecatedFileObj) && deprecatedFileObj is string df ? df : null;
+			return new DetectionRuleOverviewRef(detectionRulesFile, detectionRulesFile, detectionRulesFolders, children, placeholderContext, deprecatedFile);
 		}
 
 		// Check for file reference (file: or hidden:)

src/Elastic.Documentation.Site/Assets/pages-nav.ts

@@ -134,8 +134,16 @@ export function initNav() {
 
     // Normalize pathname by removing trailing slash to handle both URL variants
     const pathname = window.location.pathname.replace(/\/$/, '')
+
+    // When the page is a hidden nav item (e.g. an individual detection rule), the server
+    // emits docs:nav-active pointing to the nearest visible ancestor so we can highlight it.
+    const navActiveMeta = document.querySelector<HTMLMetaElement>(
+        'meta[name="docs:nav-active"]'
+    )
+    const activePathname = navActiveMeta?.content ?? pathname
+
     const navItems = $$(
-        'a[href="' + pathname + '"], a[href="' + pathname + '/"]',
+        'a[href="' + activePathname + '"], a[href="' + activePathname + '/"]',
         pagesNav
     )
     navItems.forEach((el) => {

src/Elastic.Documentation.Site/Layout/_Head.cshtml

@@ -48,6 +48,10 @@
 	@await RenderPartialAsync(_Favicon.Create(Model))
 	<meta name="robots" content="@(Model.AllowIndexing ? "index, follow" : "noindex, nofollow")">
 	<meta name="htmx-config" content='{"selfRequestsOnly": true}'>
+	@if (!string.IsNullOrEmpty(Model.NavigationActiveUrl))
+	{
+		<meta name="docs:nav-active" content="@Model.NavigationActiveUrl">
+	}
 	<meta property="og:type" content="website"/>
 	<meta property="og:title" content="@Model.Title"/>
 	<meta property="og:description" content="@Model.Description"/>

src/Elastic.Documentation.Site/_ViewModels.cs

@@ -46,6 +46,13 @@ public record GlobalLayoutViewModel
 	/// <summary>Breadcrumb trail for codex sub-header (Home / Group / Docset).</summary>
 	public IReadOnlyList<CodexBreadcrumb>? CodexBreadcrumbs { get; init; }
 
+	/// <summary>
+	/// When the current page is a hidden nav item (e.g. an individual detection rule page),
+	/// the URL of its nearest visible ancestor. The client uses this to highlight the correct
+	/// nav entry when the page has no rendered nav link of its own.
+	/// </summary>
+	public string? NavigationActiveUrl { get; init; }
+
 
 	// Header properties for isolated mode
 	public string? HeaderTitle { get; init; }

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -47,6 +47,10 @@
     <ProjectReference Include="..\Elastic.Documentation.Svg\Elastic.Documentation.Svg.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="Elastic.Markdown.Tests"/>
+  </ItemGroup>
+
   <ItemGroup>
     <UpToDateCheckInput Remove="Myst\Directives\AppliesSwitch\AppliesItemView.cshtml" />
     <UpToDateCheckInput Remove="Myst\Directives\AppliesSwitch\AppliesSwitchView.cshtml" />