Improve Orca memory troubleshooting for capped total memory

When an Orca job fails due to insufficient memory and hits a total memory limit, ARC now attempts to resolve the issue by reducing the number of CPU cores to increase the memory available per core.

- Added logic to calculate the maximum feasible CPU cores that fit within the total memory cap while meeting Orca's per-core requirements.
- Ensures at least one core is utilized if viable, rather than failing the troubleshooting step.
- Prevents recalculating/inflating total memory when it is already constrained by a cap.
- Adjusted the conservative memory buffer added during total memory estimation from 5 GB to 3 GB.

.

Author: calvinp0

arc/job/trsh.py

@@ -1028,14 +1028,18 @@ def trsh_ess_job(label: str,
             raise TrshError(f'DLPNO methods are incompatible with single-electron species {label} in Orca. '
                             f'This should have been caught by the Scheduler before job submission.')
         elif 'Memory' in job_status['keywords']:
-            # Increase memory allocation.
-            # job_status will be for example
-            # `Error  (ORCA_SCF): Not enough memory available! Please increase MaxCore to more than: 289 MB`.
+            # ORCA memory troubleshooting keeps the total job memory fixed and
+            # reduces cpu cores so %%maxcore increases on the rerun.
             if 'memory' not in ess_trsh_methods:
                 ess_trsh_methods.append('memory')
+            original_cpu_cores = cpu_cores
+            total_memory_mb = math.ceil(memory_gb * 1024)
             try:
                 # parse Orca's memory requirement in MB
                 estimated_mem_per_core = float(job_status['error'].split()[-2])
+                # round up to the next hundred
+                estimated_mem_per_core = int(np.ceil(estimated_mem_per_core / 100.0)) * 100
+                cpu_cores = math.floor(total_memory_mb / estimated_mem_per_core)
             except ValueError:
                 estimated_mem_per_core = estimate_orca_mem_cpu_requirement(num_heavy_atoms=num_heavy_atoms,
                                                                            server=server,

arc/job/trsh_test.py

@@ -677,7 +677,7 @@ def test_trsh_ess_job(self):
         # Test Orca
         # Orca: test 1
         # Test troubleshooting insufficient memory issue
-        # Automatically increase memory provided not exceeding maximum available memory
+        # Keep total memory fixed and reduce cpu cores so %%maxcore increases
         label = 'test'
         level_of_theory = {'method': 'dlpno-ccsd(T)'}
         server = 'server1'
@@ -697,8 +697,8 @@ def test_trsh_ess_job(self):
                                                                        job_type, software, fine, memory_gb,
                                                                        num_heavy_atoms, cpu_cores, ess_trsh_methods)
         self.assertIn('memory', ess_trsh_methods)
-        self.assertEqual(cpu_cores, 32)
-        self.assertAlmostEqual(memory, 327)
+        self.assertEqual(cpu_cores, 24)
+        self.assertAlmostEqual(memory, 250)
 
         # Orca: test 2
         # Test troubleshooting insufficient memory issue