|
| 1 | +using System.Security.Cryptography; |
| 2 | +using System.Text; |
| 3 | +using System.Text.RegularExpressions; |
| 4 | +using Microsoft.Extensions.Logging; |
| 5 | +using Netdocs.Abstractions; |
| 6 | + |
| 7 | +namespace Netdocs.Core.Optimization; |
| 8 | + |
| 9 | +/// <summary> |
| 10 | +/// Self-hosts external CDN assets for offline usage. Scans emitted HTML for external |
| 11 | +/// <c><script src></c>, <c><link href></c>, <c><img src></c>, and the Mermaid |
| 12 | +/// dynamic <c>import()</c>, downloads each once into <c>assets/external/</c> (recursively fetching |
| 13 | +/// <c>url(...)</c> references inside downloaded CSS, e.g. web-font files), and rewrites every page |
| 14 | +/// to point at the local copies with page-relative paths so the site works from <c>file://</c>. |
| 15 | +/// </summary> |
| 16 | +public static partial class SelfHostAssets |
| 17 | +{ |
| 18 | + private static readonly HttpClient Http = new() { Timeout = TimeSpan.FromSeconds(30) }; |
| 19 | + private const string ExternalDir = "assets/external"; |
| 20 | + |
| 21 | + /// <summary>Fetches a URL, returning its bytes and response media type (or null on failure).</summary> |
| 22 | + public delegate Task<(byte[] Bytes, string? MediaType)?> Fetcher(string url, CancellationToken ct); |
| 23 | + |
| 24 | + public static Task RunAsync(SiteContext site, ILogger log, CancellationToken ct) => |
| 25 | + RunAsync(site, log, DefaultFetch, ct); |
| 26 | + |
| 27 | + private static async Task<(byte[], string?)?> DefaultFetch(string url, CancellationToken ct) |
| 28 | + { |
| 29 | + var resp = await Http.GetAsync(url, ct); |
| 30 | + resp.EnsureSuccessStatusCode(); |
| 31 | + var bytes = await resp.Content.ReadAsByteArrayAsync(ct); |
| 32 | + return (bytes, resp.Content.Headers.ContentType?.MediaType); |
| 33 | + } |
| 34 | + |
| 35 | + public static async Task RunAsync(SiteContext site, ILogger log, Fetcher fetch, CancellationToken ct) |
| 36 | + { |
| 37 | + var siteDir = Path.GetFullPath(site.Config.AbsoluteSiteDir); |
| 38 | + var htmlFiles = Directory.EnumerateFiles(siteDir, "*.html", SearchOption.AllDirectories).ToList(); |
| 39 | + if (htmlFiles.Count == 0) return; |
| 40 | + |
| 41 | + // 1. Collect every external URL referenced by an asset tag across all pages. |
| 42 | + var urls = new HashSet<string>(StringComparer.Ordinal); |
| 43 | + var fileText = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); |
| 44 | + foreach (var f in htmlFiles) |
| 45 | + { |
| 46 | + var text = await File.ReadAllTextAsync(f, ct); |
| 47 | + fileText[f] = text; |
| 48 | + foreach (var u in ExtractAssetUrls(text)) urls.Add(u); |
| 49 | + } |
| 50 | + if (urls.Count == 0) return; |
| 51 | + |
| 52 | + // 2. Download each URL once (plus CSS-referenced assets), mapping url -> local filename. |
| 53 | + var absExternalDir = Path.Combine(siteDir, ExternalDir); |
| 54 | + Directory.CreateDirectory(absExternalDir); |
| 55 | + var map = new Dictionary<string, string>(StringComparer.Ordinal); |
| 56 | + var failures = 0; |
| 57 | + foreach (var url in urls) |
| 58 | + { |
| 59 | + var local = await DownloadAsync(url, absExternalDir, site, fetch, log, ct); |
| 60 | + if (local is null) { failures++; continue; } |
| 61 | + map[url] = local; |
| 62 | + } |
| 63 | + |
| 64 | + // 3. Rewrite each HTML file, replacing external URLs with page-relative local paths. |
| 65 | + var rewritten = 0; |
| 66 | + foreach (var (path, original) in fileText) |
| 67 | + { |
| 68 | + var pageDir = Path.GetDirectoryName(path)!; |
| 69 | + var updated = original; |
| 70 | + foreach (var (url, local) in map) |
| 71 | + { |
| 72 | + var rel = RelativePath(pageDir, Path.Combine(absExternalDir, local)); |
| 73 | + updated = updated.Replace(url, rel, StringComparison.Ordinal); |
| 74 | + } |
| 75 | + if (updated != original) |
| 76 | + { |
| 77 | + await File.WriteAllTextAsync(path, updated, ct); |
| 78 | + rewritten++; |
| 79 | + } |
| 80 | + } |
| 81 | + |
| 82 | + if (map.Count > 0) |
| 83 | + log.LogInformation("Offline: self-hosted {Count} external asset(s) into {Dir} across {Files} page(s).", |
| 84 | + map.Count, ExternalDir, rewritten); |
| 85 | + if (failures > 0) |
| 86 | + log.LogWarning("Offline: {Count} external asset(s) could not be downloaded and still point at their CDN.", failures); |
| 87 | + } |
| 88 | + |
| 89 | + /// <summary>Downloads one URL into <paramref name="dir"/>, returning the local filename, and |
| 90 | + /// recursively self-hosts <c>url(...)</c> references inside downloaded CSS.</summary> |
| 91 | + private static async Task<string?> DownloadAsync(string url, string dir, SiteContext site, Fetcher fetch, ILogger log, CancellationToken ct) |
| 92 | + { |
| 93 | + try |
| 94 | + { |
| 95 | + var result = await fetch(url, ct); |
| 96 | + if (result is not { } r) { log.LogWarning("Offline: failed to download {Url}.", url); return null; } |
| 97 | + var (bytes, mediaType) = r; |
| 98 | + var name = LocalNameFor(url, mediaType); |
| 99 | + |
| 100 | + if (name.EndsWith(".css", StringComparison.OrdinalIgnoreCase)) |
| 101 | + { |
| 102 | + var css = Encoding.UTF8.GetString(bytes); |
| 103 | + css = await InlineCssReferencesAsync(css, url, dir, site, fetch, log, ct); |
| 104 | + bytes = Encoding.UTF8.GetBytes(css); |
| 105 | + } |
| 106 | + |
| 107 | + var outPath = Path.Combine(dir, name); |
| 108 | + await File.WriteAllBytesAsync(outPath, bytes, ct); |
| 109 | + site.TrackOutput(outPath); |
| 110 | + return name; |
| 111 | + } |
| 112 | + catch (Exception ex) |
| 113 | + { |
| 114 | + log.LogWarning("Offline: failed to download {Url}: {Message}", url, ex.Message); |
| 115 | + return null; |
| 116 | + } |
| 117 | + } |
| 118 | + |
| 119 | + /// <summary>Rewrites absolute <c>url(...)</c> targets in a downloaded stylesheet to local |
| 120 | + /// filenames (same directory), downloading each referenced file (fonts, images).</summary> |
| 121 | + private static async Task<string> InlineCssReferencesAsync(string css, string cssUrl, string dir, |
| 122 | + SiteContext site, Fetcher fetch, ILogger log, CancellationToken ct) |
| 123 | + { |
| 124 | + var matches = CssUrlRegex().Matches(css).Select(m => m.Groups[1].Value).Distinct(StringComparer.Ordinal).ToList(); |
| 125 | + foreach (var raw in matches) |
| 126 | + { |
| 127 | + var abs = ToAbsolute(raw, cssUrl); |
| 128 | + if (abs is null) continue; // data: or unresolvable |
| 129 | + var local = await DownloadAsync(abs, dir, site, fetch, log, ct); |
| 130 | + if (local is not null) css = css.Replace(raw, local, StringComparison.Ordinal); |
| 131 | + } |
| 132 | + return css; |
| 133 | + } |
| 134 | + |
| 135 | + private static IEnumerable<string> ExtractAssetUrls(string html) |
| 136 | + { |
| 137 | + foreach (Match m in ScriptImgRegex().Matches(html)) yield return m.Groups[1].Value; |
| 138 | + foreach (Match m in LinkRegex().Matches(html)) |
| 139 | + { |
| 140 | + var tag = m.Value; |
| 141 | + // Preconnect / dns-prefetch hints don't fetch an asset — skip them. |
| 142 | + if (tag.Contains("preconnect", StringComparison.OrdinalIgnoreCase) |
| 143 | + || tag.Contains("dns-prefetch", StringComparison.OrdinalIgnoreCase)) continue; |
| 144 | + yield return m.Groups[1].Value; |
| 145 | + } |
| 146 | + foreach (Match m in ImportRegex().Matches(html)) yield return m.Groups[1].Value; |
| 147 | + } |
| 148 | + |
| 149 | + private static string LocalNameFor(string url, string? mediaType) |
| 150 | + { |
| 151 | + var uri = new Uri(url); |
| 152 | + var last = Path.GetFileName(uri.AbsolutePath); |
| 153 | + if (string.IsNullOrEmpty(last) || !last.Contains('.')) |
| 154 | + { |
| 155 | + var ext = ExtensionFor(mediaType) ?? ".bin"; |
| 156 | + last = (string.IsNullOrEmpty(last) ? "asset" : last) + ext; |
| 157 | + } |
| 158 | + // Prefix a short hash of the full URL to avoid collisions between same-named files. |
| 159 | + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(url)))[..8].ToLowerInvariant(); |
| 160 | + return $"{hash}-{Sanitize(last)}"; |
| 161 | + } |
| 162 | + |
| 163 | + private static string? ExtensionFor(string? mediaType) => mediaType switch |
| 164 | + { |
| 165 | + "text/css" => ".css", |
| 166 | + "application/javascript" or "text/javascript" => ".js", |
| 167 | + "font/woff2" => ".woff2", |
| 168 | + "font/woff" => ".woff", |
| 169 | + "image/png" => ".png", |
| 170 | + "image/svg+xml" => ".svg", |
| 171 | + _ => null, |
| 172 | + }; |
| 173 | + |
| 174 | + private static string? ToAbsolute(string reference, string baseUrl) |
| 175 | + { |
| 176 | + if (reference.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) return null; |
| 177 | + return Uri.TryCreate(new Uri(baseUrl), reference, out var abs) ? abs.ToString() : null; |
| 178 | + } |
| 179 | + |
| 180 | + private static string RelativePath(string fromDir, string toFile) => |
| 181 | + Path.GetRelativePath(fromDir, toFile).Replace('\\', '/'); |
| 182 | + |
| 183 | + private static string Sanitize(string name) => |
| 184 | + InvalidCharRegex().Replace(name, "_"); |
| 185 | + |
| 186 | + [GeneratedRegex("<(?:script|img)\\b[^>]*?\\ssrc=[\"'](https?://[^\"']+)[\"'][^>]*>", RegexOptions.IgnoreCase)] |
| 187 | + private static partial Regex ScriptImgRegex(); |
| 188 | + |
| 189 | + [GeneratedRegex("<link\\b[^>]*?\\shref=[\"'](https?://[^\"']+)[\"'][^>]*>", RegexOptions.IgnoreCase)] |
| 190 | + private static partial Regex LinkRegex(); |
| 191 | + |
| 192 | + [GeneratedRegex("import\\(\\s*[\"'](https?://[^\"']+)[\"']\\s*\\)", RegexOptions.IgnoreCase)] |
| 193 | + private static partial Regex ImportRegex(); |
| 194 | + |
| 195 | + [GeneratedRegex("url\\(\\s*[\"']?([^\"')]+)[\"']?\\s*\\)", RegexOptions.IgnoreCase)] |
| 196 | + private static partial Regex CssUrlRegex(); |
| 197 | + |
| 198 | + [GeneratedRegex("[^A-Za-z0-9._-]")] |
| 199 | + private static partial Regex InvalidCharRegex(); |
| 200 | +} |
0 commit comments