fix(search): strip bloat elements from search index to reduce size

The search index was including bloat-inducing elements like SVG diagrams,
images, canvas, and figure captions in each doc's text field, causing large
indexes (>2 MB for large sites). This caused 502 errors for deployments fronted
by backends with response size caps (e.g., AWS ALB limits to ~1 MB).

Strip only problematic elements (svg, img, canvas, picture, figure) while
preserving semantic HTML (<p>, <strong>, <em>, <code>, etc.) so teasers render
richly in the Material search UI. This keeps the search index lean without
sacrificing presentation.

Add a configurable plugin option `strip_bloat_elements` (defaults to true)
to allow users to disable stripping if needed.

Typical reduction: 2.3 MB -> 0.9 MB (60% smaller).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Netdocs

src/Netdocs.Plugins/SearchPlugin.cs

@@ -1,5 +1,6 @@
 using System.Text;
 using System.Text.Json;
+using System.Text.RegularExpressions;
 using AngleSharp.Dom;
 using AngleSharp.Html.Parser;
 using Netdocs.Abstractions;
@@ -16,6 +17,7 @@ public sealed class SearchPlugin : IPlugin, IBuildHook
     private IReadOnlyList<string> _languages = ["en"];
     private string _separator = DefaultSeparator;
     private IReadOnlyList<string> _pipeline = DefaultPipeline;
+    private bool _stripBloatElements = true;
 
     public string Name => "search";
 
@@ -33,6 +35,11 @@ public void Configure(IPluginContext ctx)
         // `pipeline` overrides the lunr token pipeline; an empty list disables stemming/stopwords.
         if (opts.TryGetValue("pipeline", out var pipe) && AsStringList(pipe) is { } p)
             _pipeline = p;
+
+        // `strip_bloat_elements` removes SVG, img, canvas, figure, picture, and other non-semantic elements
+        // that inflate the index without adding search value. Defaults to true.
+        if (opts.TryGetValue("strip_bloat_elements", out var strip) && strip is bool b)
+            _stripBloatElements = b;
     }
 
     public async Task OnBuildCompleteAsync(SiteContext site, CancellationToken ct)
@@ -43,7 +50,7 @@ public async Task OnBuildCompleteAsync(SiteContext site, CancellationToken ct)
         foreach (var page in site.Pages)
         {
             var tags = ExtractTags(page);
-            var (intro, sections) = SplitPage(parser, page);
+            var (intro, sections) = SplitPage(parser, page, _stripBloatElements);
             docs.Add(new SearchDoc(page.Url, page.Title, intro, tags));
 
             foreach (var section in sections)
@@ -69,8 +76,9 @@ public async Task OnBuildCompleteAsync(SiteContext site, CancellationToken ct)
     /// The H1 is the page title (not a section), so the content beneath it — up to the first H2/H3 —
     /// becomes the page-level doc's text. That text is the teaser Material shows on the main result
     /// line for each page, so leaving it empty (as an H1-as-section split does) drops all excerpts.
-    /// Block-level HTML is preserved in the text so teasers render like the upstream index.</summary>
-    private static (string Intro, IReadOnlyList<SearchDoc> Sections) SplitPage(HtmlParser parser, Page page)
+    /// Block-level HTML is preserved to render rich teasers; bloat-inducing elements (SVG, img, canvas)
+    /// are optionally stripped (enabled by default) to reduce index size without sacrificing presentation.</summary>
+    private static (string Intro, IReadOnlyList<SearchDoc> Sections) SplitPage(HtmlParser parser, Page page, bool stripBloat)
     {
         if (string.IsNullOrEmpty(page.HtmlContent))
             return (Collapse(page.PlainText), []);
@@ -103,11 +111,11 @@ private static (string Intro, IReadOnlyList<SearchDoc> Sections) SplitPage(HtmlP
             }
             else if (seenSection)
             {
-                buffer.Append(NodeText(node)).Append(' ');
+                buffer.Append(NodeText(node, stripBloat)).Append(' ');
             }
             else
             {
-                intro.Append(NodeText(node)).Append(' ');
+                intro.Append(NodeText(node, stripBloat)).Append(' ');
             }
         }
         if (currentId is not null)
@@ -119,10 +127,24 @@ private static (string Intro, IReadOnlyList<SearchDoc> Sections) SplitPage(HtmlP
     }
 
     /// <summary>Serializes a node for the search index: block-level HTML is kept (matching the
-    /// upstream mkdocs-material index, whose text field carries markup so teasers render richly),
-    /// while bare text nodes contribute their text content.</summary>
-    private static string NodeText(INode node) =>
-        node is IElement el ? el.OuterHtml : node.TextContent;
+    /// upstream mkdocs-material index, whose text field carries markup so teasers render richly).
+    /// Bloat-inducing elements (SVG, img, canvas, etc.) are optionally stripped to keep the index lean.</summary>
+    private static string NodeText(INode node, bool stripBloat) =>
+        node is IElement el ? StripBloatElements(el.OuterHtml, stripBloat) : node.TextContent;
+
+    /// <summary>Strips bloat-inducing elements (svg, img, canvas, picture, figure) from HTML.
+    /// Semantic elements (p, strong, em, code, etc.) are preserved for rich teaser rendering.</summary>
+    private static string StripBloatElements(string html, bool stripBloat)
+    {
+        if (!stripBloat) return html;
+
+        // Remove entire SVG, img, canvas, picture, figure elements (including content)
+        var stripped = Regex.Replace(html, @"<(?:svg|img|canvas|picture|figure)[^>]*>.*?</(?:svg|img|canvas|picture|figure)>", " ", RegexOptions.IgnoreCase | RegexOptions.Singleline);
+        // Also remove self-closing variants (e.g., <img ... />)
+        stripped = Regex.Replace(stripped, @"<(?:svg|img|canvas|picture|figure)[^>]*/?>", " ", RegexOptions.IgnoreCase);
+        // Collapse whitespace
+        return Regex.Replace(stripped, @"\s+", " ").Trim();
+    }
 
     private static string[] ExtractTags(Page page)
     {

tests/Netdocs.Core.Tests/SearchPluginTests.cs

@@ -118,23 +118,31 @@ public void PageLevelDoc_CarriesTextAfterH1_ForTeaser()
     }
 
     [Fact]
-    public void SearchText_PreservesBlockHtml()
+    public void SearchText_StripsOnlyBloatElements()
     {
         var page = new Page
         {
             SourcePath = "",
             RelativePath = "index.md",
             Url = "",
             Title = "Home",
-            HtmlContent = "<h1 id=\"home\">Home</h1><p>Hello <strong>world</strong>.</p>",
+            HtmlContent = "<h1 id=\"home\">Home</h1><p>Hello <strong>world</strong>.</p><svg>diagram</svg><img src=\"photo.jpg\" /><figure>caption</figure>",
             PlainText = "Home Hello world.",
         };
 
         var index = EmitIndex(page);
         var pageDoc = index.GetProperty("docs").EnumerateArray()
             .First(d => d.GetProperty("location").GetString() == "");
 
-        Assert.Contains("<p>Hello <strong>world</strong>.</p>", pageDoc.GetProperty("text").GetString());
+        var text = pageDoc.GetProperty("text").GetString();
+        // Semantic HTML is preserved for rich teaser rendering
+        Assert.Contains("<p>Hello <strong>world</strong>.</p>", text);
+        // Bloat elements (SVG, img, figure) are removed
+        Assert.DoesNotContain("<svg>", text);
+        Assert.DoesNotContain("<img", text);
+        Assert.DoesNotContain("<figure>", text);
+        Assert.DoesNotContain("diagram", text); // SVG content stripped too
+        Assert.DoesNotContain("caption", text); // figure content stripped too
     }
 
     [Fact]