Revert "Revert "feat(assembler): on-demand assembler serve with live reload and improved frontend DX""

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
           dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler config init --local 
           dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler clone -c local --skip-private-repositories
           dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler build -c local --skip-private-repositories
-          dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler serve &
+          dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler serve-static &
 
       - name: Wait for docs
         working-directory: src/Elastic.Documentation.Site

aspire/AppHost.cs

@@ -129,7 +129,7 @@ internal static async Task Run(
 			.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
 			.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
 			.WithHttpEndpoint(port: 4000, isProxied: false)
-			.WithArgs(["assembler", "serve", .. GlobalArguments])
+			.WithArgs(["assembler", "serve-static", .. GlobalArguments])
 			.WithHttpHealthCheck("/", 200)
 			.WaitForCompletion(buildAll)
 			.WithParentRelationship(cloneAll);

build/CommandLine.fs

@@ -43,6 +43,7 @@ type Build =
 
     | [<CliPrefix(CliPrefix.None);SubCommand>] Format of ParseResults<FormatArgs>
     | [<CliPrefix(CliPrefix.None);SubCommand>] Watch
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Watch_Full
 
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] Lint of ParseResults<LintArgs>
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] PristineCheck
@@ -84,6 +85,7 @@ with
             | Format _ -> "runs dotnet format"
 
             | Watch -> "runs dotnet watch to continuous build code/templates and web assets on the fly"
+            | Watch_Full -> "runs assembler serve with dotnet watch — watches checkout dirs and live-reloads assembled docs"
 
             // steps
             | Lint _ -> "runs dotnet format --verify-no-changes"

build/Targets.fs

@@ -42,6 +42,8 @@ let private format (formatArgs: ParseResults<FormatArgs>) =
 
 let private watch _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--configuration" "debug" "--" "serve" "--watch" }
 
+let private watchFull _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--configuration" "debug" "--" "assembler" "serve" }
+
 let private lint (lintArgs: ParseResults<LintArgs>) =
     let includeFiles = lintArgs.TryGetResult LintArgs.Include |> Option.defaultValue []
     let includeArgs = 
@@ -256,6 +258,7 @@ let Setup (parsed:ParseResults<Build>) =
 
         | Format formatArgs -> Build.Step (fun _ -> format formatArgs)
         | Watch -> Build.Step watch
+        | Watch_Full -> Build.Step watchFull
 
         // steps
         | Lint lintArgs -> Build.Step (fun _ -> lint lintArgs)

docs/cli-schema.json

@@ -1329,9 +1329,85 @@
             "assembler"
           ],
           "name": "serve",
+          "summary": "Serve assembled documentation with live reload and on-demand per-request rendering. Requires assembler clone to have been run first. No prior build is needed. Pages are rendered on demand; file changes invalidate the repo and trigger a live reload.",
+          "notes": null,
+          "usage": "docs-builder assembler serve [options]",
+          "examples": [],
+          "parameters": [
+            {
+              "role": "flag",
+              "name": "port",
+              "shortName": null,
+              "type": "integer",
+              "required": false,
+              "summary": "Port to listen on. Default: 4000.",
+              "defaultValue": "4000"
+            },
+            {
+              "role": "flag",
+              "name": "environment",
+              "shortName": null,
+              "type": "string",
+              "required": false,
+              "summary": "Named deployment target. Determines which repositories are used."
+            },
+            {
+              "role": "flag",
+              "name": "no-watch-md",
+              "shortName": null,
+              "type": "boolean",
+              "required": false,
+              "summary": "Disable watching checkout directories for markdown changes. Static asset live reload still works. Useful when doing frontend (CSS/JS) work.",
+              "defaultValue": "false"
+            },
+            {
+              "role": "flag",
+              "name": "log-level",
+              "shortName": "l",
+              "type": "enum",
+              "required": false,
+              "summary": "Minimum log level. Default: information",
+              "enumValues": [
+                "trace",
+                "debug",
+                "information",
+                "warning",
+                "error",
+                "critical",
+                "none"
+              ]
+            },
+            {
+              "role": "flag",
+              "name": "config-source",
+              "shortName": "c",
+              "type": "enum",
+              "required": false,
+              "summary": "Override the configuration source: local, remote",
+              "enumValues": [
+                "local",
+                "remote",
+                "embedded"
+              ]
+            },
+            {
+              "role": "flag",
+              "name": "skip-private-repositories",
+              "shortName": null,
+              "type": "boolean",
+              "required": false,
+              "summary": "Skip cloning private repositories"
+            }
+          ]
+        },
+        {
+          "path": [
+            "assembler"
+          ],
+          "name": "serve-static",
           "summary": "Serve the output of a completed assembler build at http://localhost:4000.",
           "notes": "Run after assembler build. Does not watch for file changes.",
-          "usage": "docs-builder assembler serve [options]",
+          "usage": "docs-builder assembler serve-static [options]",
           "examples": [],
           "parameters": [
             {
@@ -1347,7 +1423,7 @@
               "name": "path",
               "type": "string",
               "required": false,
-              "summary": "Path to the built site. Defaults to .artifacts/docs/.",
+              "summary": "Path to the built site. Defaults to .artifacts/assembly/.",
               "validations": [
                 {
                   "kind": "rejectSymbolicLinks"

src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs

@@ -9,7 +9,7 @@
 
 namespace Elastic.Documentation.LinkIndex;
 
-public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader
+public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader, IDisposable
 {
 
 	// <summary>
@@ -52,4 +52,10 @@ public async Task<RepositoryLinks> GetRepositoryLinks(string key, Cancel cancell
 	}
 
 	public string RegistryUrl { get; } = $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}";
+
+	public void Dispose()
+	{
+		s3Client.Dispose();
+		GC.SuppressFinalize(this);
+	}
 }

src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs

@@ -58,9 +58,10 @@ public record FetchedCrossLinks
 	};
 }
 
-public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, ScopedFileSystem? fileSystem = null) : IDisposable
+public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, ScopedFileSystem? fileSystem = null, bool ownsReader = false) : IDisposable
 {
 	protected ILogger Logger { get; } = logFactory.CreateLogger(nameof(CrossLinkFetcher));
+	protected ILinkIndexReader LinkIndexProvider => linkIndexProvider;
 	private readonly IFileSystem _fileSystem = fileSystem ?? FileSystemFactory.AppData;
 	private LinkRegistry? _linkIndex;
 
@@ -206,7 +207,11 @@ private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkR
 
 	public void Dispose()
 	{
-		logFactory.Dispose();
+		// Only dispose linkIndexProvider when this fetcher created it (ownsReader = true).
+		// When the reader was injected by the caller, the caller retains ownership and must dispose it.
+		// logFactory is always injected — never disposed here.
+		if (ownsReader && linkIndexProvider is IDisposable disposableReader)
+			disposableReader.Dispose();
 		GC.SuppressFinalize(this);
 	}
 }

src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs

@@ -16,9 +16,11 @@ public class DocSetConfigurationCrossLinkFetcher(
 	ConfigurationFile configuration,
 	ILinkIndexReader? linkIndexProvider = null,
 	ILinkIndexReader? codexLinkIndexReader = null)
-	: CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous())
+	: CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous(), ownsReader: linkIndexProvider is null)
 {
 	private readonly ILogger _logger = logFactory.CreateLogger(nameof(DocSetConfigurationCrossLinkFetcher));
+	// _codexReader is injected by the caller who retains ownership and is responsible for disposal.
+	// ReloadableGeneratorState, the primary caller, disposes it directly in its own Dispose().
 	private readonly ILinkIndexReader? _codexReader = codexLinkIndexReader;
 
 	public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
@@ -31,7 +33,7 @@ public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 		var codexRepositories = new HashSet<string>();
 		var declaredRepositories = new HashSet<string>();
 
-		var publicReader = linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous();
+		var publicReader = LinkIndexProvider;
 		var useDualRegistry = configuration.Registry != DocSetRegistry.Public && _codexReader is not null;
 
 		// Fetch each registry once up front so per-repository lookups don't trigger N S3 round-trips.

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -217,14 +217,26 @@ public INavigationItem FindNavigationByMarkdown(MarkdownFile markdown)
 	}
 
 	private bool _resolved;
+	private long _version;
+
+	public void InvalidateResolved()
+	{
+		_ = Interlocked.Increment(ref _version);
+		_resolved = false;
+	}
+
 	public async Task ResolveDirectoryTree(Cancel ctx)
 	{
 		if (_resolved)
 			return;
 
+		// Capture the version before parsing so that if InvalidateResolved() fires
+		// mid-flight we do not incorrectly mark the (now stale) result as resolved.
+		var capturedVersion = Interlocked.Read(ref _version);
 		await Parallel.ForEachAsync(MarkdownFiles, ctx, async (file, token) => await file.MinimalParseAsync(TryFindDocumentByRelativePath, token));
 
-		_resolved = true;
+		if (Interlocked.Read(ref _version) == capturedVersion)
+			_resolved = true;
 	}
 
 	public RepositoryLinks CreateLinkReference()

src/services/Elastic.Documentation.Assembler/AssembleSources.cs

@@ -42,13 +42,16 @@ Cancel ctx
 	{
 		var logger = logFactory.CreateLogger<AssembleSources>();
 
-		var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous();
 		var navigationTocMappings = GetTocMappings(context);
 		var uriResolver = new PublishEnvironmentUriResolver(navigationTocMappings, context.Environment);
 
 		var sw = System.Diagnostics.Stopwatch.StartNew();
-		var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexProvider);
-		var crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx);
+		FetchedCrossLinks crossLinks;
+		// Use a separate using for the reader so ownership is explicit: the caller (this method)
+		// disposes it, not the fetcher (ownsReader stays false/default on AssemblerCrossLinkFetcher).
+		using var linkIndexReader = Aws3LinkIndexReader.CreateAnonymous();
+		using (var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader))
+			crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx);
 		var crossLinkResolver = new CrossLinkResolver(crossLinks, uriResolver);
 		logger.LogInformation("  AssembleAsync: FetchCrossLinks in {Elapsed:mm\\:ss\\.fff}", sw.Elapsed);
 

src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs

@@ -62,14 +62,7 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
 				continue;
 			}
 
-			// Create inferrer per-repository with git context
-			var documentInferrer = new DocumentInferrerService(
-				context.ProductsConfiguration,
-				context.VersionsConfiguration,
-				context.LegacyUrlMappings,
-				set.DocumentationSet.Configuration,
-				set.DocumentationSet.Context.Git
-			);
+			var documentInferrer = CreateInferrer(set);
 
 			var stopwatch = Stopwatch.StartNew();
 			try
@@ -157,6 +150,33 @@ string Resolve(string path)
 		}
 	}
 
+	private DocumentInferrerService CreateInferrer(AssemblerDocumentationSet set) =>
+		new(
+			context.ProductsConfiguration,
+			context.VersionsConfiguration,
+			context.LegacyUrlMappings,
+			set.DocumentationSet.Configuration,
+			set.DocumentationSet.Context.Git
+		);
+
+	public DocumentationGenerator CreateGenerator(AssemblerDocumentationSet set)
+	{
+		SetFeatureFlags(set);
+		return new DocumentationGenerator(
+			set.DocumentationSet,
+			logFactory, NavigationTraversable, HtmlWriter,
+			pathProvider,
+			legacyUrlMapper: LegacyUrlMapper,
+			documentInferrer: CreateInferrer(set)
+		);
+	}
+
+	public async Task BuildOneAsync(AssemblerDocumentationSet set, Cancel ctx)
+	{
+		await set.DocumentationSet.ResolveDirectoryTree(ctx);
+		_ = await BuildAsync(set, null, CreateInferrer(set), ctx);
+	}
+
 	private async Task<GenerationResult> BuildAsync(AssemblerDocumentationSet set, IMarkdownExporter[]? markdownExporters, IDocumentInferrerService documentInferrer, Cancel ctx)
 	{
 		SetFeatureFlags(set);

src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs

@@ -6,15 +6,21 @@
 using Actions.Core.Services;
 using Documentation.Builder.Http;
 using Elastic.Documentation;
+using Elastic.Documentation.Assembler;
 using Elastic.Documentation.Assembler.Building;
+using Elastic.Documentation.Assembler.Navigation;
 using Elastic.Documentation.Assembler.Sourcing;
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Assembler;
+using Elastic.Documentation.Configuration.Toc;
 using Elastic.Documentation.Diagnostics;
+using Elastic.Documentation.LegacyDocs;
+using Elastic.Documentation.Navigation.Assembler;
 using Elastic.Documentation.Services;
 using Microsoft.Extensions.Logging;
 using Nullean.Argh;
 using Nullean.Argh.Documentation;
+using Nullean.ScopedFileSystem;
 
 namespace Documentation.Builder.Commands.Assembler;
 
@@ -155,13 +161,60 @@ static async (s, col, state, ctx) => await s.BuildAll(col, state.options, state.
 		return await serviceInvoker.InvokeAsync(ct);
 	}
 
+	/// <summary>
+	/// Serve assembled documentation with live reload and on-demand per-request rendering.
+	/// Requires <c>assembler clone</c> to have been run first. No prior build is needed.
+	/// Pages are rendered on demand; file changes invalidate the repo and trigger a live reload.
+	/// </summary>
+	/// <param name="port">Port to listen on. Default: 4000.</param>
+	/// <param name="environment">Named deployment target. Determines which repositories are used.</param>
+	/// <param name="noWatchMd">Disable watching checkout directories for markdown changes. Static asset live reload still works. Useful when doing frontend (CSS/JS) work.</param>
+	[NoOptionsInjection]
+	public async Task Serve(int port = 4000, string? environment = null, bool noWatchMd = false, CancellationToken ct = default)
+	{
+		environment ??= "dev";
+		var readFs = FileSystemFactory.RealRead;
+		var writeFs = FileSystemFactory.RealWrite;
+
+		var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, readFs, writeFs, null, null);
+
+		var cloner = new AssemblerRepositorySourcer(logFactory, assembleContext);
+		var checkoutResult = cloner.GetAll();
+		var checkouts = checkoutResult.Checkouts.ToArray();
+
+		if (checkouts.Length == 0)
+			throw new Exception("No checkouts found. Run 'assembler clone' first.");
+
+		var exporters = ExportOptions.Default
+			.Except([Exporter.DocumentationState])
+			.ToHashSet();
+
+		var assembleSources = await AssembleSources.AssembleAsync(logFactory, assembleContext, checkouts, configurationContext, exporters, ct);
+
+		var navigationFileInfo = configurationContext.ConfigurationFileProvider.NavigationFile;
+		var siteNavigationFile = SiteNavigationFile.Deserialize(await readFs.File.ReadAllTextAsync(navigationFileInfo.FullName, ct));
+		var documentationSets = assembleSources.AssembleSets.Values.Select(s => s.DocumentationSet.Navigation).ToArray();
+		var navigation = new SiteNavigation(siteNavigationFile, assembleContext, documentationSets, assembleContext.Environment.PathPrefix);
+
+		var pathProvider = new GlobalNavigationPathProvider(navigation, assembleSources, assembleContext);
+		using var htmlWriter = new GlobalNavigationHtmlWriter(logFactory, navigation, collector);
+		var legacyPageChecker = new LegacyPageService(logFactory);
+		var historyMapper = new PageLegacyUrlMapper(legacyPageChecker, assembleContext.VersionsConfiguration, assembleSources.LegacyUrlMappings);
+		var builder = new AssemblerBuilder(logFactory, assembleContext, navigation, htmlWriter, pathProvider, historyMapper);
+
+		var host = new AssemblerServeWebHost(port, assembleSources, builder, logFactory, watchMarkdown: !noWatchMd);
+		await host.RunAsync(ct);
+		await host.StopAsync(ct);
+		await collector.StopAsync(ct);
+	}
+
 	/// <summary>Serve the output of a completed assembler build at <c>http://localhost:4000</c>.</summary>
 	/// <remarks>Run after <c>assembler build</c>. Does not watch for file changes.</remarks>
 	/// <param name="port">Port to listen on. Default: 4000.</param>
-	/// <param name="path">Path to the built site. Defaults to <c>.artifacts/docs/</c>.</param>
-
+	/// <param name="path">Path to the built site. Defaults to <c>.artifacts/assembly/</c>.</param>
 	[NoOptionsInjection]
-	public async Task Serve(int port = 4000, [Existing, ExpandUserProfile, RejectSymbolicLinks] DirectoryInfo? path = null, CancellationToken ct = default)
+	[CommandName("serve-static")]
+	public async Task ServeStatic(int port = 4000, [Existing, ExpandUserProfile, RejectSymbolicLinks] DirectoryInfo? path = null, CancellationToken ct = default)
 	{
 		var host = new StaticWebHost(port, path?.FullName);
 		await host.RunAsync(ct);