Improved timeout handling.

Author: gfraiteur

src/PostSharp.Engineering.BuildTools/AppExtensions.cs

@@ -219,11 +219,15 @@ internal static void AddCommands( this CommandApp app, Product product )
                     {
                         tools.AddCommand<KillCommand>( "kill" )
                             .WithData( data )
-                            .WithDescription( "Kill all compiler processes" );
+                            .WithDescription( "Kill all compiler processes." );
 
-                        tools.AddCommand<DumpAndKillCommand>( "dump-and-kill" )
+                        tools.AddCommand<DumpCommand>( "dump" )
                             .WithData( data )
-                            .WithDescription( "Dump and kill a given process and all its descendants" );
+                            .WithDescription( "Dump a given process and all its descendants." );
+
+                        tools.AddCommand<WaitCommand>( "wait" )
+                            .WithData( data )
+                            .WithDescription( "Wait a given number of seconds. When used to test the behavior of the the --timeout argument." );
 
                         tools.AddBranch(
                             "csproj",

src/PostSharp.Engineering.BuildTools/BaseCommand.cs

@@ -24,8 +24,13 @@ public abstract class BaseCommand<T> : Command<T>
     {
         public sealed override int Execute( CommandContext context, T settings )
         {
+            // We use two CancellationTokenSources: one for timeout, the second for manual Ctrl+C, so that we can react differently
+            // according to the cancellation reason.
             CancellationTokenSource? timeoutCancellation = null;
 
+            var mainCancellation = new CancellationTokenSource();
+            Console.CancelKeyPress += ( _, _ ) => mainCancellation.Cancel();
+
             try
             {
                 var stopwatch = Stopwatch.StartNew();
@@ -35,7 +40,7 @@ public sealed override int Execute( CommandContext context, T settings )
                     Debugger.Launch();
                 }
 
-                if ( !BuildContext.TryCreate( context, settings, out var buildContext ) )
+                if ( !BuildContext.TryCreate( context, settings, mainCancellation.Token, out var buildContext ) )
                 {
                     return 1;
                 }
@@ -49,7 +54,7 @@ public sealed override int Execute( CommandContext context, T settings )
                 if ( settings.Timeout != null )
                 {
                     timeoutCancellation = new CancellationTokenSource( TimeSpan.FromMinutes( settings.Timeout.Value ) );
-                    timeoutCancellation.Token.Register( () => OnTimeout( buildContext, stopwatch ) );
+                    timeoutCancellation.Token.Register( () => OnTimeout( buildContext, stopwatch, mainCancellation ) );
                 }
 
                 MSBuildHelper.InitializeLocator();
@@ -149,18 +154,23 @@ public sealed override int Execute( CommandContext context, T settings )
                 // Execute the command itself.
                 var success = this.ExecuteCore( buildContext, settings );
 
+                if ( buildContext.CancellationToken.IsCancellationRequested )
+                {
+                    return (int) ExitCodes.Cancelled;
+                }
+
                 if ( !settings.NoLogo )
                 {
                     buildContext.Console.WriteMessage( $"Finished at {DateTime.Now} after {stopwatch.Elapsed}." );
                 }
 
-                return success ? 0 : 1;
+                return (int) (success ? ExitCodes.Success : ExitCodes.Error);
             }
             catch ( Exception ex )
             {
                 AnsiConsole.WriteException( ex );
 
-                return 10;
+                return (int) ExitCodes.Exception;
             }
             finally
             {
@@ -170,15 +180,42 @@ public sealed override int Execute( CommandContext context, T settings )
 
         protected abstract bool ExecuteCore( BuildContext context, T settings );
 
-        private static void OnTimeout( BuildContext buildContext, Stopwatch stopwatch )
+        private static void OnTimeout( BuildContext buildContext, Stopwatch stopwatch, CancellationTokenSource mainCancellation )
         {
-            // ReSharper disable AccessToModifiedClosure
+            var console = buildContext.Console;
+
+            if ( RuntimeInformation.IsOSPlatform( OSPlatform.Windows ) )
+            {
+                console.WriteError( $"The process timed out after {stopwatch.Elapsed}. Dumping and killing the process tree." );
+                var directory = Path.Combine( buildContext.RepoDirectory, buildContext.Product.DumpDirectory );
+
+                // List all child processes.
+                var processes = ProcessHelper.GetProcessTree( console, Process.GetCurrentProcess().Id );
+
+                console.WriteMessage( "Process tree:" );
 
-            buildContext.Console.WriteError( $"The process timed out after {stopwatch.Elapsed}. Dumping and killing the process tree." );
-            var directory = Path.Combine( buildContext.RepoDirectory, buildContext.Product.DumpDirectory );
-            ProcessHelper.DumpAndKillProcessTree( buildContext.Console, Process.GetCurrentProcess(), directory );
+                foreach ( var node in processes )
+                {
+                    var indent = new string( '-', (node.NestingLevel + 1) * 3 );
+                    console.WriteMessage( $"+{indent} {node.Process.Id} {ProcessHelper.GetCommandLine( node.Process )}" );
+                }
+
+                // Dump these processes.
+                ProcessHelper.DumpProcesses( console, processes.Select( p => p.Process ), directory );
+
+                // Signal the main cancellation source.
+                mainCancellation.Cancel();
+
+                // Kill all processes (except the current one) in reverse order.
+                ProcessHelper.KillProcesses( console, processes.Reverse().Select( x => x.Process ) );
+            }
+            else
+            {
+                console.WriteError( $"The process timed out after {stopwatch.Elapsed}. Exiting." );
+            }
 
-            // ReSharper restore AccessToModifiedClosure
+            console.WriteWarning( "Terminating the current process." );
+            Environment.FailFast( $"The process timed out after {stopwatch.Elapsed}." );
         }
     }
 }

src/PostSharp.Engineering.BuildTools/Build/BuildContext.cs

@@ -9,6 +9,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
+using System.Threading;
 
 namespace PostSharp.Engineering.BuildTools.Build
 {
@@ -72,7 +73,8 @@ private BuildContext(
             string branch,
             CommandContext commandContext,
             bool useProjectDirectoryAsWorkingDirectory,
-            CommonCommandSettings settings )
+            CommonCommandSettings settings,
+            CancellationToken cancellationToken )
         {
             this.Console = console;
             this.RepoDirectory = repoDirectory;
@@ -81,6 +83,7 @@ private BuildContext(
             this.CommandContext = commandContext;
             this.UseProjectDirectoryAsWorkingDirectory = useProjectDirectoryAsWorkingDirectory;
             this.Settings = settings;
+            this.CancellationToken = cancellationToken;
         }
 
         /// <summary>
@@ -89,6 +92,7 @@ private BuildContext(
         public static bool TryCreate(
             CommandContext commandContext,
             CommonCommandSettings settings,
+            CancellationToken cancellationToken,
             [NotNullWhen( true )] out BuildContext? buildContext )
         {
             buildContext = null;
@@ -114,7 +118,8 @@ public static bool TryCreate(
                 currentBranch,
                 commandContext,
                 useProjectDirectoryAsWorkingDirectory: false,
-                settings );
+                settings,
+                cancellationToken );
 
             return true;
         }
@@ -158,7 +163,8 @@ public BuildContext WithConsoleHelper( ConsoleHelper consoleHelper )
                 this.Branch,
                 this.CommandContext,
                 this.UseProjectDirectoryAsWorkingDirectory,
-                this.Settings );
+                this.Settings,
+                this.CancellationToken );
 
         public BuildContext WithUseProjectDirectoryAsWorkingDirectory( bool useProjectDirectoryAsWorkingDirectory )
             => new(
@@ -168,7 +174,8 @@ public BuildContext WithUseProjectDirectoryAsWorkingDirectory( bool useProjectDi
                 this.Branch,
                 this.CommandContext,
                 useProjectDirectoryAsWorkingDirectory,
-                this.Settings );
+                this.Settings,
+                this.CancellationToken );
 
         [PublicAPI]
 #pragma warning disable CA1822
@@ -180,6 +187,8 @@ public BuildContext WithUseProjectDirectoryAsWorkingDirectory( bool useProjectDi
         [PublicAPI]
         public CommonCommandSettings Settings { get; }
 
+        public CancellationToken CancellationToken { get; }
+
         internal TimeSpan BuildTimeout
         {
             get

src/PostSharp.Engineering.BuildTools/CodeStyle/BaseCodeStyleCommand.cs

@@ -15,21 +15,22 @@ internal abstract class BaseCodeStyleCommand<T> : BaseCommand<T>
         protected static string? GetCodeStyleRepo( BuildContext context, CodeStyleSettings settings )
         {
             var sharedRepo = Path.GetFullPath( Path.Combine( context.RepoDirectory, "..", "PostSharp.Engineering.CodeStyle" ) );
+            var console = context.Console;
 
             // Check if the repo exists.
             if ( !Directory.Exists( sharedRepo ) )
             {
                 if ( !settings.Create )
                 {
-                    context.Console.WriteError( $"The directory '{sharedRepo} does not exist. Use --create'" );
+                    console.WriteError( $"The directory '{sharedRepo} does not exist. Use --create'" );
 
                     return null;
                 }
                 else
                 {
                     var baseDir = Path.GetDirectoryName( sharedRepo )!;
 
-                    ToolInvocationHelper.InvokeTool( context.Console, "git", $"clone {settings.Url}", baseDir );
+                    ToolInvocationHelper.InvokeTool( console, "git", $"clone {settings.Url}", baseDir );
                 }
             }
 

src/PostSharp.Engineering.BuildTools/CommonCommandSettings.cs

@@ -118,7 +118,7 @@ protected set
 
         [Description( "Overrides the build timeout." )]
         [CommandOption( "--timeout" )]
-        public int? Timeout { get; set; }
+        public double? Timeout { get; set; }
 
         public ImmutableDictionary<string, string> Properties { get; protected set; } =
             ImmutableDictionary.Create<string, string>( StringComparer.OrdinalIgnoreCase );

src/PostSharp.Engineering.BuildTools/EngineeringApp.cs

@@ -27,5 +27,5 @@ public void Configure( Action<IConfigurator> configure )
         this._app.Configure( configure );
     }
 
-    public int Run( IEnumerable<string> args ) => this._app.Run( args );
+    public int Run( IEnumerable<string> args ) { return this._app.Run( args ); }
 }

src/PostSharp.Engineering.BuildTools/ExitCodes.cs

@@ -0,0 +1,12 @@
+// Copyright (c) SharpCrafters s.r.o. See the LICENSE.md file in the root directory of this repository root for details.
+
+namespace PostSharp.Engineering.BuildTools;
+
+public enum ExitCodes
+{
+    Success,
+    Error,
+    Exception = 100,
+    Cancelled = 200,
+    Timeout = 300
+}

src/PostSharp.Engineering.BuildTools/Tools/Processes/DumpAndKillCommand.cs

@@ -0,0 +1,36 @@
+// Copyright (c) SharpCrafters s.r.o. See the LICENSE.md file in the root directory of this repository root for details.
+
+using JetBrains.Annotations;
+using PostSharp.Engineering.BuildTools.Build;
+using PostSharp.Engineering.BuildTools.Utilities;
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+
+namespace PostSharp.Engineering.BuildTools.Tools.Processes;
+
+[UsedImplicitly]
+internal class DumpCommand : BaseCommand<DumpCommandSettings>
+{
+    protected override bool ExecuteCore( BuildContext context, DumpCommandSettings settings )
+    {
+        var console = context.Console;
+
+        // List all child processes.
+        var processes = ProcessHelper.GetProcessTree( console, Process.GetCurrentProcess().Id );
+
+        console.WriteMessage( "Process tree:" );
+
+        foreach ( var node in processes )
+        {
+            var indent = new string( '-', (node.NestingLevel + 1) * 3 );
+            console.WriteMessage( $"+{indent} {node.Process.Id} {ProcessHelper.GetCommandLine( node.Process )}" );
+        }
+
+        // Dump these processes.
+        ProcessHelper.DumpProcesses( console, processes.Select( p => p.Process ), Path.GetTempPath() );
+
+        return true;
+    }
+}

src/PostSharp.Engineering.BuildTools/Tools/Processes/DumpCommand.cs

@@ -6,7 +6,7 @@
 namespace PostSharp.Engineering.BuildTools.Tools.Processes;
 
 [UsedImplicitly]
-internal sealed class DumpAndKillCommandSettings : CommonCommandSettings
+internal sealed class DumpCommandSettings : CommonCommandSettings
 {
     [CommandArgument( 0, "<process-id>" )]
     public int ProcessId { get; init; }