+semver:major - Merge pull request #2087 from thomhurst/feature/immediate-module-output

feat: immediate module output with progress coordination

Author: thomhurst

.gitignore

@@ -402,4 +402,4 @@ requirements
 
 # Git worktrees directory
 .worktrees/
-docs/plans/
+docs/plans/

src/ModularPipelines/Console/ConsoleCoordinator.cs

@@ -64,6 +64,7 @@ internal class ConsoleCoordinator : IConsoleCoordinator, IProgressDisplay
 
     // Logger for output
     private ILogger? _outputLogger;
+    private readonly IOutputCoordinator _outputCoordinator;
 
     public ConsoleCoordinator(
         IBuildSystemFormatterProvider formatterProvider,
@@ -72,7 +73,8 @@ public ConsoleCoordinator(
         IOptions<PipelineOptions> options,
         ILoggerFactory loggerFactory,
         IBuildSystemDetector buildSystemDetector,
-        IServiceProvider serviceProvider)
+        IServiceProvider serviceProvider,
+        IOutputCoordinator outputCoordinator)
     {
         _formatterProvider = formatterProvider;
         _resultsPrinter = resultsPrinter;
@@ -81,6 +83,7 @@ public ConsoleCoordinator(
         _loggerFactory = loggerFactory;
         _buildSystemDetector = buildSystemDetector;
         _serviceProvider = serviceProvider;
+        _outputCoordinator = outputCoordinator;
         _unattributedBuffer = new ModuleOutputBuffer("Pipeline", typeof(void));
     }
 
@@ -221,8 +224,12 @@ public async Task<IProgressSession> BeginProgressAsync(
             this,
             modules,
             _options,
+            _loggerFactory,
             cancellationToken);
 
+        // Wire up the progress controller for output coordination
+        _outputCoordinator.SetProgressController(session);
+
         _activeSession = session;
 
         // Start the progress display
@@ -238,6 +245,7 @@ internal void EndProgressPhase()
     {
         lock (_phaseLock)
         {
+            _outputCoordinator.SetProgressController(NoOpProgressController.Instance);
             _isProgressActive = false;
             _activeSession = null;
         }
@@ -255,38 +263,22 @@ public IModuleOutputBuffer GetModuleBuffer(Type moduleType)
     /// <inheritdoc />
     public void FlushModuleOutput()
     {
+        // Output is now flushed immediately when modules complete.
+        // This method remains for API compatibility but only flushes
+        // unattributed output (pipeline-level logs).
         if (_originalConsoleOut == null)
         {
-            throw new InvalidOperationException("ConsoleCoordinator is not installed.");
+            return; // Not installed, nothing to flush
         }
 
         var formatter = _formatterProvider.GetFormatter();
 
-        // Flush unattributed output first (if any)
+        // Flush unattributed output (if any)
         if (_unattributedBuffer.HasOutput)
         {
             var unattributedLogger = _outputLogger ?? _loggerFactory.CreateLogger("ModularPipelines.Output");
             _unattributedBuffer.FlushTo(_originalConsoleOut, formatter, unattributedLogger);
         }
-
-        // Flush module buffers in completion order
-        var orderedBuffers = _moduleBuffers.Values
-            .Where(b => b.HasOutput)
-            .OrderBy(b => b.CompletedAtUtc ?? DateTime.MaxValue)
-            .ToList();
-
-        foreach (var buffer in orderedBuffers)
-        {
-            // Resolve the registered ILogger<T> from DI to use any custom loggers injected by tests
-            var loggerType = typeof(ILogger<>).MakeGenericType(buffer.ModuleType);
-            var moduleLogger = (ILogger)_serviceProvider.GetService(loggerType)
-                               ?? _loggerFactory.CreateLogger(buffer.ModuleType);
-            buffer.FlushTo(_originalConsoleOut, formatter, moduleLogger);
-        }
-
-        // Clear buffers after flush to release memory
-        // This prevents accumulation in long-running pipelines
-        _moduleBuffers.Clear();
     }
 
     /// <inheritdoc />

src/ModularPipelines/Console/IConsoleCoordinator.cs

@@ -72,10 +72,9 @@ internal interface IConsoleCoordinator : IAsyncDisposable
     IModuleOutputBuffer GetUnattributedBuffer();
 
     /// <summary>
-    /// Flushes all module output in completion order.
-    /// Should be called after progress phase ends.
+    /// Flushes any remaining unattributed output.
+    /// Module output is flushed immediately when modules complete.
     /// </summary>
-    /// <exception cref="InvalidOperationException">Thrown if not installed.</exception>
     void FlushModuleOutput();
 
     /// <summary>

src/ModularPipelines/Console/IModuleOutputBuffer.cs

@@ -15,8 +15,8 @@ namespace ModularPipelines.Console;
 /// <b>Thread Safety:</b> All methods are thread-safe and can be called concurrently.
 /// </para>
 /// <para>
-/// <b>Flush Ordering:</b> Buffers are flushed in completion order (by CompletedAtUtc)
-/// to maintain logical output sequence.
+/// <b>Flush Behavior:</b> Buffers are flushed immediately when modules complete,
+/// via the OutputCoordinator which ensures ordered output.
 /// </para>
 /// </remarks>
 internal interface IModuleOutputBuffer
@@ -26,18 +26,6 @@ internal interface IModuleOutputBuffer
     /// </summary>
     Type ModuleType { get; }
 
-    /// <summary>
-    /// Gets when the module completed (for ordering during flush).
-    /// Null if not yet completed.
-    /// </summary>
-    DateTime? CompletedAtUtc { get; }
-
-    /// <summary>
-    /// Records completion time for flush ordering.
-    /// Called when the module finishes execution.
-    /// </summary>
-    void MarkCompleted();
-
     /// <summary>
     /// Adds a plain string line to the buffer.
     /// Used for Console.WriteLine interceptions.

src/ModularPipelines/Console/IOutputCoordinator.cs

@@ -0,0 +1,24 @@
+namespace ModularPipelines.Console;
+
+/// <summary>
+/// Coordinates immediate flushing of module output with synchronization.
+/// Ensures modules write output in FIFO completion order without interleaving.
+/// </summary>
+internal interface IOutputCoordinator
+{
+    /// <summary>
+    /// Sets the progress controller for pause/resume coordination.
+    /// Must be called before any modules complete.
+    /// </summary>
+    /// <param name="controller">The progress controller.</param>
+    void SetProgressController(IProgressController controller);
+
+    /// <summary>
+    /// Enqueues a module's buffer for immediate flushing and waits until flushed.
+    /// Called when a module's logger is disposed.
+    /// </summary>
+    /// <param name="buffer">The buffer to flush.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>A task that completes when the buffer has been flushed.</returns>
+    Task EnqueueAndFlushAsync(IModuleOutputBuffer buffer, CancellationToken cancellationToken = default);
+}

src/ModularPipelines/Console/IProgressController.cs

@@ -0,0 +1,26 @@
+namespace ModularPipelines.Console;
+
+/// <summary>
+/// Controls pausing and resuming the progress display for output flushing.
+/// </summary>
+internal interface IProgressController
+{
+    /// <summary>
+    /// Gets whether this controller supports dynamic display (interactive terminal).
+    /// </summary>
+    bool IsInteractive { get; }
+
+    /// <summary>
+    /// Pauses the progress display to allow console output.
+    /// On interactive terminals, this clears the progress display.
+    /// On CI, this is a no-op.
+    /// </summary>
+    Task PauseAsync();
+
+    /// <summary>
+    /// Resumes the progress display after output.
+    /// On interactive terminals, this redraws the progress state.
+    /// On CI, this is a no-op.
+    /// </summary>
+    Task ResumeAsync();
+}

src/ModularPipelines/Console/ModuleOutputBuffer.cs

@@ -31,9 +31,6 @@ internal class ModuleOutputBuffer : IModuleOutputBuffer
     /// <inheritdoc />
     public Type ModuleType { get; }
 
-    /// <inheritdoc />
-    public DateTime? CompletedAtUtc { get; private set; }
-
     /// <summary>
     /// Initializes a new buffer for the specified module type.
     /// </summary>
@@ -80,15 +77,6 @@ public void AddLogEvent(
         }
     }
 
-    /// <inheritdoc />
-    public void MarkCompleted()
-    {
-        lock (_lock)
-        {
-            CompletedAtUtc ??= DateTime.UtcNow;
-        }
-    }
-
     /// <inheritdoc />
     public void SetException(Exception exception)
     {
@@ -165,7 +153,7 @@ public void FlushTo(TextWriter console, IBuildSystemFormatter formatter, ILogger
 
     private string FormatHeader(Exception? exception)
     {
-        var duration = (CompletedAtUtc ?? DateTime.UtcNow) - _startTimeUtc;
+        var duration = DateTime.UtcNow - _startTimeUtc;
         var durationStr = duration.TotalSeconds >= 60
             ? $"{duration.TotalMinutes:F1}m"
             : $"{duration.TotalSeconds:F1}s";

src/ModularPipelines/Console/NoOpProgressController.cs

@@ -0,0 +1,19 @@
+using System.Diagnostics.CodeAnalysis;
+
+namespace ModularPipelines.Console;
+
+/// <summary>
+/// Progress controller for CI environments where no dynamic display exists.
+/// All operations are no-ops.
+/// </summary>
+[ExcludeFromCodeCoverage]
+internal sealed class NoOpProgressController : IProgressController
+{
+    public static NoOpProgressController Instance { get; } = new();
+
+    public bool IsInteractive => false;
+
+    public Task PauseAsync() => Task.CompletedTask;
+
+    public Task ResumeAsync() => Task.CompletedTask;
+}