Netdocs/src/Netdocs.Plugins/SearchPlugin.cs at 387e3396f4d94e7dc977218e378972c5b8e384ff · XtremeOwnage/Netdocs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using Netdocs.Abstractions;
using Netdocs.Core.Content;

namespace Netdocs.Plugins;

/// <summary>Emits a Material-compatible <c>search/search_index.json</c> (page + per-section docs).</summary>
public sealed class SearchPlugin : IPlugin, IBuildHook
{
    private static readonly string[] DefaultPipeline = ["stemmer", "stopWordFilter", "trimmer"];
    private const string DefaultSeparator = "[\\s\\-]+";

    private IReadOnlyList<string> _languages = ["en"];
    private string _separator = DefaultSeparator;
    private IReadOnlyList<string> _pipeline = DefaultPipeline;
    private bool _stripBloatElements = true;

    public string Name => "search";

    public void Configure(IPluginContext ctx)
    {
        var opts = ctx.PluginOptions;

        // `lang` accepts a single language ("en") or a list (["en", "de"]).
        if (opts.TryGetValue("lang", out var lang))
            _languages = AsStringList(lang) is { Count: > 0 } list ? list : _languages;

        if (opts.TryGetValue("separator", out var sep) && sep is string s && s.Length > 0)
            _separator = s;

        // `pipeline` overrides the lunr token pipeline; an empty list disables stemming/stopwords.
        if (opts.TryGetValue("pipeline", out var pipe) && AsStringList(pipe) is { } p)
            _pipeline = p;

        // `strip_bloat_elements` removes SVG, img, canvas, figure, picture, and other non-semantic elements
        // that inflate the index without adding search value. Defaults to true.
        if (opts.TryGetValue("strip_bloat_elements", out var strip) && strip is bool b)
            _stripBloatElements = b;
    }

    public async Task OnBuildCompleteAsync(SiteContext site, CancellationToken ct)
    {
        var parser = new HtmlParser();
        var docs = new List<SearchDoc>();

        foreach (var page in site.Pages)
        {
            var tags = ExtractTags(page);
            var (intro, sections) = SplitPage(parser, page, _stripBloatElements);
            docs.Add(new SearchDoc(page.Url, page.Title, intro, tags));

            foreach (var section in sections)
                docs.Add(section with { Tags = tags });
        }

        var fields = new Dictionary<string, object?>
        {
            ["title"] = new Dictionary<string, object?> { ["boost"] = 1000.0 },
            ["text"] = new Dictionary<string, object?> { ["boost"] = 1.0 },
            ["tags"] = new Dictionary<string, object?> { ["boost"] = 1000000.0 },
        };
        var index = new SearchIndex(
            new SearchConfig(_languages, _separator, _pipeline, fields),
            docs);

        var dir = Path.Combine(site.Config.AbsoluteSiteDir, "search");
        var json = JsonSerializer.Serialize(index, SearchJson.Options);
        await OutputWriter.WriteTextIfChangedAsync(site, Path.Combine(dir, "search_index.json"), json, ct);
    }

    /// <summary>Splits a page into its page-level text and per-section docs, matching mkdocs-material.
    /// The H1 is the page title (not a section), so the content beneath it — up to the first H2/H3 —
    /// becomes the page-level doc's text. That text is the teaser Material shows on the main result
    /// line for each page, so leaving it empty (as an H1-as-section split does) drops all excerpts.
    /// Block-level HTML is preserved to render rich teasers; bloat-inducing elements (SVG, img, canvas)
    /// are optionally stripped (enabled by default) to reduce index size without sacrificing presentation.</summary>
    private static (string Intro, IReadOnlyList<SearchDoc> Sections) SplitPage(HtmlParser parser, Page page, bool stripBloat)
    {
        if (string.IsNullOrEmpty(page.HtmlContent))
            return (Collapse(page.PlainText), []);

        var doc = parser.ParseDocument($"<body>{page.HtmlContent}</body>");
        var body = doc.Body;
        if (body is null) return (Collapse(page.PlainText), []);

        var sections = new List<SearchDoc>();
        var intro = new StringBuilder();
        string? currentId = null, currentTitle = null;
        var buffer = new StringBuilder();
        var seenSection = false;

        foreach (var node in body.ChildNodes)
        {
            // The H1 is the page title, not a searchable section; its text is captured by page.Title
            // and the content that follows it flows into the page-level teaser.
            if (node is IElement h1 && h1.TagName == "H1")
                continue;

            if (node is IElement el && el.TagName is "H2" or "H3" && !string.IsNullOrEmpty(el.Id))
            {
                if (currentId is not null)
                    sections.Add(new SearchDoc($"{page.Url}#{currentId}", currentTitle ?? "", Collapse(buffer.ToString()), []));
                currentId = el.Id;
                currentTitle = el.TextContent.Trim();
                buffer.Clear();
                seenSection = true;
            }
            else if (seenSection)
            {
                buffer.Append(NodeText(node, stripBloat)).Append(' ');
            }
            else
            {
                intro.Append(NodeText(node, stripBloat)).Append(' ');
            }
        }
        if (currentId is not null)
            sections.Add(new SearchDoc($"{page.Url}#{currentId}", currentTitle ?? "", Collapse(buffer.ToString()), []));

        var introText = Collapse(intro.ToString());
        if (introText.Length == 0 && sections.Count == 0) introText = Collapse(page.PlainText);
        return (introText, sections);
    }

    /// <summary>Serializes a node for the search index: block-level HTML is kept (matching the
    /// upstream mkdocs-material index, whose text field carries markup so teasers render richly).
    /// Bloat-inducing elements (SVG, img, canvas, etc.) are optionally stripped to keep the index lean.</summary>
    private static string NodeText(INode node, bool stripBloat) =>
        node is IElement el ? StripBloatElements(el.OuterHtml, stripBloat) : node.TextContent;

    /// <summary>Strips bloat-inducing elements (svg, img, canvas, picture, figure) from HTML.
    /// Semantic elements (p, strong, em, code, etc.) are preserved for rich teaser rendering.</summary>
    private static string StripBloatElements(string html, bool stripBloat)
    {
        if (!stripBloat) return html;

        // Remove entire SVG, img, canvas, picture, figure elements (including content)
        var stripped = Regex.Replace(html, @"<(?:svg|img|canvas|picture|figure)[^>]*>.*?</(?:svg|img|canvas|picture|figure)>", " ", RegexOptions.IgnoreCase | RegexOptions.Singleline);
        // Also remove self-closing variants (e.g., <img ... />)
        stripped = Regex.Replace(stripped, @"<(?:svg|img|canvas|picture|figure)[^>]*/?>", " ", RegexOptions.IgnoreCase);
        // Collapse whitespace
        return Regex.Replace(stripped, @"\s+", " ").Trim();
    }

    private static string[] ExtractTags(Page page)
    {
        if (page.FrontMatter.TryGetValue("tags", out var t) && t is IEnumerable<object?> list)
            return list.Select(x => x?.ToString() ?? "").Where(s => s.Length > 0).ToArray();
        return [];
    }

    private static string Collapse(string text) =>
        string.Join(' ', text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));

    /// <summary>Normalizes a plugin option that may be a single string or a list into a string list.
    /// Returns null when the value is neither (so callers can keep their default).</summary>
    private static IReadOnlyList<string>? AsStringList(object? value) => value switch
    {
        string s => [s],
        IEnumerable<object?> items => items.Select(x => x?.ToString() ?? "").Where(s => s.Length > 0).ToList(),
        _ => null,
    };
}

public sealed record SearchIndex(SearchConfig Config, IReadOnlyList<SearchDoc> Docs);
public sealed record SearchConfig(IReadOnlyList<string> Lang, string Separator, IReadOnlyList<string> Pipeline, IReadOnlyDictionary<string, object?> Fields);
public sealed record SearchDoc(string Location, string Title, string Text, string[] Tags);

internal static class SearchJson
{
    public static readonly JsonSerializerOptions Options = new()
    {
        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
        DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
    };
}