Slightly lower memory usage estimate for memory paralleization in NuGet job

Author: MihaZupan

Runner/Jobs/JitDiffJob.cs

@@ -360,9 +360,11 @@ await Task.WhenAll(
                 RuntimeHelpers.CopyAspNetSharedFrameworkToCoreRootAsync(this, "artifacts-main"),
                 RuntimeHelpers.CopyAspNetSharedFrameworkToCoreRootAsync(this, "artifacts-pr"));
 
+            int memoryAvailableGB = GetRemainingSystemMemoryGB();
+
             await Task.WhenAll(
-                DiffExtraProjectsAsync("artifacts-main", "clr-checked-main", DiffsMainDirectory),
-                DiffExtraProjectsAsync("artifacts-pr", "clr-checked-pr", DiffsPrDirectory));
+                DiffExtraProjectsAsync("artifacts-main", "clr-checked-main", DiffsMainDirectory, memoryAvailableGB),
+                DiffExtraProjectsAsync("artifacts-pr", "clr-checked-pr", DiffsPrDirectory, memoryAvailableGB));
         }
         finally
         {
@@ -379,7 +381,7 @@ await Task.WhenAll(
 
         return diffAnalyzeSummary;
 
-        async Task DiffExtraProjectsAsync(string coreRootFolder, string checkedClrFolder, string outputFolder)
+        async Task DiffExtraProjectsAsync(string coreRootFolder, string checkedClrFolder, string outputFolder, int memoryAvailableGB)
         {
             string projectsRoot = ExtraProjectsDirectory;
             string branch = coreRootFolder.Contains("main", StringComparison.Ordinal) ? "main" : "pr";
@@ -411,16 +413,27 @@ async Task DiffExtraProjectsAsync(string coreRootFolder, string checkedClrFolder
                 return;
             }
 
-            int coreRootCopies = Math.Min(Math.Min(Environment.ProcessorCount / 2, GetRemainingSystemMemoryGB() / 3), projectDirs.Count / 20);
-            coreRootCopies = Math.Max(coreRootCopies, 1);
+            int memoryParallelism = OnRamDisk ? (int)(memoryAvailableGB / 2.5) : memoryAvailableGB * 2;
+            int coreRootCopies = Math.Min(Math.Min(Environment.ProcessorCount, memoryParallelism), projectDirs.Count);
+            coreRootCopies = Math.Max(coreRootCopies / 2, 1); // / 2 because we run for main and pr in parallel
+            var countdown = new CountdownEvent(coreRootCopies);
 
             await Parallel.ForAsync(0, coreRootCopies, async (index, _) =>
             {
-                string newCoreRootFolder = $"{coreRootFolder}_{index}";
-                string newCheckedClrFolder = $"{checkedClrFolder}_{index}";
+                string newCoreRootFolder = coreRootFolder;
+                string newCheckedClrFolder = checkedClrFolder;
+
+                if (index > 0)
+                {
+                    newCoreRootFolder = $"{newCoreRootFolder}_{index}";
+                    newCheckedClrFolder = $"{newCheckedClrFolder}_{index}";
 
-                CopyDirectory(coreRootFolder, newCoreRootFolder);
-                CopyDirectory(checkedClrFolder, newCheckedClrFolder);
+                    CopyDirectory(coreRootFolder, newCoreRootFolder);
+                    CopyDirectory(checkedClrFolder, newCheckedClrFolder);
+                }
+
+                countdown.Signal();
+                countdown.Wait(JobTimeout);
 
                 while (true)
                 {
@@ -466,8 +479,11 @@ await Parallel.ForAsync(0, coreRootCopies, async (index, _) =>
                     }
                 }
 
-                try { Directory.Delete(newCoreRootFolder, recursive: true); } catch { }
-                try { Directory.Delete(newCheckedClrFolder, recursive: true); } catch { }
+                if (index > 0)
+                {
+                    try { Directory.Delete(newCoreRootFolder, recursive: true); } catch { }
+                    try { Directory.Delete(newCheckedClrFolder, recursive: true); } catch { }
+                }
             });
         }
 

Runner/Jobs/NuGetExtraAssembliesJob.cs

@@ -197,7 +197,7 @@ private async Task ProcessApprovedPackagesAsync(List<PackageInfo> approvedPackag
         string diffOutputDir = "nuget-diff-temp";
         Directory.CreateDirectory(diffOutputDir);
 
-        int memoryParallelism = OnRamDisk ? GetRemainingSystemMemoryGB() / 3 : GetRemainingSystemMemoryGB() * 2;
+        int memoryParallelism = OnRamDisk ? (int)(GetRemainingSystemMemoryGB() / 2.5) : GetRemainingSystemMemoryGB() * 2;
         int parallelism = Math.Min(Math.Min(Environment.ProcessorCount, memoryParallelism), approvedPackages.Count);
         parallelism = Math.Max(parallelism, 1);
         var packageQueue = new Queue<PackageInfo>(