[NativeAOT] Print OOM message before Abort() on Linux

src/coreclr/nativeaot/System.Private.CoreLib/src/System/RuntimeExceptionHelpers.cs

@@ -201,16 +201,23 @@ internal static void SerializeCrashInfo(RhFailFastReason reason, string? message
             int previousState = Interlocked.CompareExchange(ref s_crashInfoPresent, -1, 0);
             if (previousState == 0)
             {
-                CrashInfo crashInfo = new();
+                try
+                {
+                    CrashInfo crashInfo = new();
 
-                crashInfo.Open(reason, Thread.CurrentOSThreadId, message ?? GetStringForFailFastReason(reason));
-                if (exception != null)
+                    crashInfo.Open(reason, Thread.CurrentOSThreadId, message ?? GetStringForFailFastReason(reason));
+                    if (exception != null)
+                    {
+                        crashInfo.WriteException(exception);
+                    }
+                    crashInfo.Close();
+                    s_triageBufferAddress = crashInfo.TriageBufferAddress;
+                    s_triageBufferSize = crashInfo.TriageBufferSize;
+                }
+                catch
                 {
-                    crashInfo.WriteException(exception);
+                    // If crash info serialization fails (for example, due to OOM), proceed without it.
                 }
-                crashInfo.Close();
-                s_triageBufferAddress = crashInfo.TriageBufferAddress;
-                s_triageBufferSize = crashInfo.TriageBufferSize;
 
                 s_crashInfoPresent = 1;
             }
@@ -235,8 +242,20 @@ internal static unsafe void FailFast(string? message = null, Exception? exceptio
             ulong previousThreadId = Interlocked.CompareExchange(ref s_crashingThreadId, currentThreadId, 0);
             if (previousThreadId == 0)
             {
-                bool minimalFailFast = (exception == PreallocatedOutOfMemoryException.Instance);
-                if (!minimalFailFast)
+                bool minimalFailFast = exception == PreallocatedOutOfMemoryException.Instance;
+                if (minimalFailFast)
+                {
+                    // Minimal OOM fail-fast path: avoid heap allocations as much as possible, but still
+                    // report that OOM is the reason for the crash.
+                    try
+                    {
+                        // Try to print the same short message CoreCLR prints.
+                        Internal.Console.Error.Write("Out of memory.");
+                        Internal.Console.Error.WriteLine();
+                    }
+                    catch { }
+                }
+                else
                 {
                     Internal.Console.Error.Write(((exception == null) || (reason is RhFailFastReason.EnvironmentFailFast or RhFailFastReason.AssertionFailure)) ?
                         "Process terminated. " : "Unhandled exception. ");
@@ -266,8 +285,21 @@ internal static unsafe void FailFast(string? message = null, Exception? exceptio
 
                     if ((exception != null) && (reason is not RhFailFastReason.AssertionFailure))
                     {
-                        Internal.Console.Error.Write(exception.ToString());
-                        Internal.Console.Error.WriteLine();
+                        try
+                        {
+                            Internal.Console.Error.Write(exception.ToString());
+                            Internal.Console.Error.WriteLine();
+                        }
+                        catch
+                        {
+                            // If ToString() fails (for example, due to OOM), fall back to printing just the type name.
+                            try
+                            {
+                                Internal.Console.Error.Write(exception.GetType().FullName);
+                                Internal.Console.Error.WriteLine();
+                            }
+                            catch { }
+                        }
                     }
 
 #if TARGET_WINDOWS

src/tests/baseservices/exceptions/OutOfMemoryException/OutOfMemoryException.cs

@@ -0,0 +1,113 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// This test verifies that an out-of-memory condition produces a diagnostic
+// message on stderr before the process terminates.
+//
+// The test spawns itself as a subprocess with a small GC heap limit set via
+// DOTNET_GCHeapHardLimit so that the subprocess reliably runs out of memory.
+// The outer process then validates that the subprocess wrote the expected
+// OOM message to its standard error stream.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+class OutOfMemoryExceptionTest
+{
+    const int Pass = 100;
+    const int Fail = -1;
+    const int TimeoutMilliseconds = 60 * 1000;
+
+    const string AllocateSmallArg = "--allocate-small";
+    const string AllocateLargeArg = "--allocate-large";
+    // The standard unhandled-exception path ("Unhandled exception. System.OutOfMemoryException...")
+    // contains this token. The minimal OOM fail-fast path may only print a short "Out of memory." message.
+    // The test validates that some OOM diagnostic is printed rather than just "Aborted" with no context.
+    const string ExpectedOomToken = "OutOfMemoryException";
+    const string ExpectedMinimalOomToken = "Out of memory.";
+
+    static int Main(string[] args)
+    {
+        if (args.Length > 0 && args[0] == AllocateSmallArg)
+        {
+            // Pre-allocate a flat array for storage.
+            object[] storage = new object[8192];
+            int idx = 0;
+            // We expect ~2048 iterations in the first loop and ~64 iterations in the second.
+            try { while (idx < storage.Length) storage[idx++] = new byte[16 * 1024]; } catch (OutOfMemoryException) { }
+            try { while (idx < storage.Length) storage[idx++] = new byte[256]; } catch (OutOfMemoryException) { }
+            // < 280 bytes free.
+            // Use the smallest possible allocation to exhaust the last scraps.
+            while (idx < storage.Length) storage[idx++] = new object();
+        }
+
+        if (args.Length > 0 && args[0] == AllocateLargeArg)
+        {
+            // Subprocess mode: allocate 128 KB chunks until OOM is triggered.
+            // This leaves some free memory when OOM fires, exercising the code
+            // path where GetRuntimeException may allocate a new OutOfMemoryException.
+            var list = new List<byte[]>();
+            while (true) list.Add(new byte[128 * 1024]);
+        }
+
+        // Controller mode: launch subprocesses with a GC heap limit and verify their output.
+        int result = RunSubprocess(AllocateSmallArg, "small allocations");
+        if (result != Pass)
+            return result;
+
+        return RunSubprocess(AllocateLargeArg, "large allocations");
+    }
+
+    static int RunSubprocess(string allocateArg, string description)
+    {
+        Console.WriteLine($"Testing OOM with {description}...");
+
+        string fileName = Environment.ProcessPath;
+        string[] arguments = TestLibrary.Utilities.IsNativeAot
+            ? [allocateArg]
+            : [typeof(OutOfMemoryExceptionTest).Assembly.Location, allocateArg];
+
+        var psi = new ProcessStartInfo(fileName, arguments)
+        {
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+        };
+        // 32 MB GC heap limit (0x2000000): small enough to exhaust quickly but large enough for startup.
+        psi.Environment["DOTNET_GCHeapHardLimit"] = "0x2000000";
+        psi.Environment["DOTNET_DbgEnableMiniDump"] = "0";
+
+        ProcessTextOutput output;
+        try
+        {
+            output = Process.RunAndCaptureText(psi, TimeSpan.FromMilliseconds(TimeoutMilliseconds));
+        }
+        catch (TimeoutException)
+        {
+            Console.WriteLine($"Subprocess timed out after {TimeoutMilliseconds / 1000} seconds.");
+            return Fail;
+        }
+
+        Console.WriteLine($"Subprocess exit code: {output.ExitStatus.ExitCode}");
+        Console.WriteLine($"Subprocess stderr: {output.StandardError}");
+
+        if (output.ExitStatus.ExitCode == 0 || output.ExitStatus.ExitCode == Pass)
+        {
+            Console.WriteLine("Expected a non-success exit code from the OOM subprocess.");
+            return Fail;
+        }
+
+        string stderr = output.StandardError;
+
+        // Even in the small allocations case, the runtime might still have enough memory to construct
+        // an OutOfMemoryException and print the full diagnostic.
+        // Either token is acceptable, but at least one should be present to confirm that OOM was the reason for termination.
+        if (!(stderr.Contains(ExpectedOomToken) || stderr.Contains(ExpectedMinimalOomToken)))
+        {
+            Console.WriteLine($"Expected OOM diagnostic token not found in subprocess stderr.");
+            return Fail;
+        }
+
+        return Pass;
+    }
+}

src/tests/baseservices/exceptions/OutOfMemoryException/OutOfMemoryException.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <CLRTestPriority>0</CLRTestPriority>
+    <!-- This test spawns a subprocess; not supported on mobile, browser, or WASI platforms -->
+    <CLRTestTargetUnsupported Condition="'$(TargetsAppleMobile)' == 'true' or '$(TargetsAndroid)' == 'true' or '$(TargetsBrowser)' == 'true' or '$(TargetsWasi)' == 'true'">true</CLRTestTargetUnsupported>
+    <RequiresProcessIsolation>true</RequiresProcessIsolation>
+    <ReferenceXUnitWrapperGenerator>false</ReferenceXUnitWrapperGenerator>
+    <!-- Mono doesn't enforce DOTNET_GCHeapHardLimit as a GC heap limit -->
+    <DisableProjectBuild Condition="'$(RuntimeFlavor)' == 'mono'">true</DisableProjectBuild>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="OutOfMemoryException.cs" />
+    <ProjectReference Include="$(TestLibraryProjectPath)" />
+  </ItemGroup>
+</Project>