tests: zombie kill-cap regression + align stubs with 6h grace

Add a two-pass regression test asserting the second zombie detection
for the same (species, job_type) is a no-op after one kill+resubmit.
Derive stub-job offsets from ZOMBIE_GRACE_SECONDS, which commit
9fdb8b9 raised to 6h without updating the tests.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: alongd

arc/job/zombie_test.py

@@ -14,7 +14,8 @@
 
 
 def _stub_job(job_adapter='molpro', job_type='sp', execution_type='queue',
-              initial_offset_seconds=7200, job_name='sp_a3177', job_id=12345,
+              initial_offset_seconds=zombie.ZOMBIE_GRACE_SECONDS + 3600,
+              job_name='sp_a3177', job_id=12345,
               server='server1', remote_path='/remote/no/such/path',
               local_path='/tmp/no/such/path',
               local_path_to_output_file='/tmp/no/such/output.out'):
@@ -36,7 +37,7 @@ def test_periodic_writers_set(self):
         )
 
     def test_grace_period_default(self):
-        self.assertEqual(zombie.ZOMBIE_GRACE_SECONDS, 3600)
+        self.assertEqual(zombie.ZOMBIE_GRACE_SECONDS, 21600)
 
 
 class TestIsZombie(unittest.TestCase):

arc/scheduler_test.py

@@ -22,6 +22,7 @@
 from arc.common import ARC_PATH, ARC_TESTING_PATH, almost_equal_coords_lists, initialize_job_types, read_yaml_file
 from arc.job.adapters.common import default_incore_adapters, ts_adapters_by_rmg_family, ts_adapters_for_unknown_unimolecular
 from arc.job.factory import job_factory
+from arc.job.zombie import ZOMBIE_GRACE_SECONDS
 from arc.level import Level
 from arc.level.protocol import CompositeProtocol
 from arc.plotter import save_conformers_file
@@ -2567,7 +2568,7 @@ def _make_sched(self):
         return sched
 
     def _stub_job(self, job_adapter='molpro', job_type='sp', execution_type='queue',
-                  initial_offset_seconds=7200, job_name='sp_a3177', job_id=12345):
+                  initial_offset_seconds=ZOMBIE_GRACE_SECONDS + 3600, job_name='sp_a3177', job_id=12345):
         job = SimpleNamespace(
             job_name=job_name, job_type=job_type, job_id=job_id,
             job_adapter=job_adapter, execution_type=execution_type,
@@ -2615,7 +2616,7 @@ def test_healthy_job_not_killed(self):
 
     def test_grace_period_blocks_zombie_check(self):
         sched = self._make_sched()
-        # Spawned 30 minutes ago — within the 1hr grace period.
+        # Spawned 30 minutes ago — within the grace period.
         job = self._stub_job(initial_offset_seconds=1800)
         self._install(sched, job)
         sched._run_a_job = lambda *a, **kw: self.fail("must not act inside grace window")
@@ -2663,6 +2664,31 @@ def test_cap_prevents_double_resubmit(self):
         self.assertFalse(job.deleted)
         self.assertIn(job.job_name, sched.running_jobs['H2'])
 
+    def test_second_zombie_pass_is_noop_after_one_kill(self):
+        """A second zombie detection for the same (species, job_type) after one
+        real kill-and-resubmit cycle must be a no-op: no second delete, no second
+        resubmission, and the 'leaving for manual intervention' branch is taken."""
+        sched = self._make_sched()
+        job = self._stub_job()
+        self._install(sched, job)
+        run_calls = []
+        sched._run_a_job = lambda job, label: run_calls.append((job.job_name, label))
+        with patch('arc.job.zombie.output_mtime', return_value=None):
+            sched.check_for_zombie_jobs('H2')
+            self.assertTrue(job.deleted)
+            self.assertEqual(run_calls, [(job.job_name, 'H2')])
+            self.assertEqual(sched._zombie_kills['H2'], {'sp'})
+            # The resubmitted job wedges too: the queue reports it running again.
+            job.deleted = False
+            sched.running_jobs['H2'].append(job.job_name)
+            with self.assertLogs(logger='arc', level=logging.WARNING) as cm:
+                sched.check_for_zombie_jobs('H2')
+        self.assertEqual(run_calls, [(job.job_name, 'H2')])
+        self.assertFalse(job.deleted)
+        self.assertEqual(sched._zombie_kills['H2'], {'sp'})
+        self.assertIn(job.job_name, sched.running_jobs['H2'])
+        self.assertIn('manual intervention', '\n'.join(cm.output))
+
     def test_cap_is_per_job_type(self):
         sched = self._make_sched()
         sched._zombie_kills['H2'] = {'sp'}  # sp already used.