Skip to content

Commit c993e6e

Browse files
committed
feat(evolution): run processing chain every 4h (raise throughput vs backlog growth)
The pipeline generates ~25 issues/day (research+issues+introspection) but the once-daily processing chain landed only ~1-2/night (analysis caps at 8, most triage out / don't land cleanly), so the open backlog grows ~20/day — the recurring 'again many unprocessed issues'. Run the PROCESSING chain every 4h instead of daily (generation stays daily): - analysis: 0 1,5,9,13,17,21 * * * (was 0 21) - implementation: 0 2,6,10,14,18,22 * * * (was 0 22, +1h after analysis) - integration: 0 3,7,11,15,19,23 * * * (was 0 23, +1h so CI settles) This is ~6x processing passes/day. Mechanically safe: reports are date-keyed (overwritten each run, latest wins) and the implementation freshness gate is date-based ('today's analysis = fresh'), so intra-day re-runs don't break the chain; the accepted-label + dedup prevent re-processing already-handled issues. Watchdog: STAGES now mirrors the FIRST daily slot (analysis=1, impl=2, integration=3) — 'a report for today must exist by then' (reports are date-keyed). Mirror test updated to read the first hour of a multi-slot cron; stage-report tests made slot-aware. 35 watchdog tests pass. NOTE: takes effect after the server re-registers cron (register_evolution_cron on next 'hermes update'/pull). Cost ~6x the processing agent runs/day — dial to every 6h/8h if needed. Complementary generation backlog-cap still recommended to fully converge (generation ~25/day > even 6x processing).
1 parent a60abf2 commit c993e6e

5 files changed

Lines changed: 32 additions & 16 deletions

File tree

cron/evolution/analysis.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: evolution-analysis
2-
schedule: "0 21 * * *" # Daily at 9 PM
2+
schedule: "0 1,5,9,13,17,21 * * *" # Every 4h (was daily 21:00). Raises processing throughput vs ~25 issues/day generation. 21:00 slot still follows introspection (20:00). Watchdog STAGES mirrors the FIRST slot (1).
33
enabled: true
44
mode: PRIVATE
55

cron/evolution/implementation.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: evolution-implementation
2-
schedule: "0 22 * * *" # Daily at 10 PM
2+
schedule: "0 2,6,10,14,18,22 * * *" # Every 4h, +1h after analysis (1,5,9,...). Watchdog STAGES mirrors the FIRST slot (2).
33
enabled: true
44
mode: PRIVATE
55

cron/evolution/integration.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: evolution-integration
2-
schedule: "0 23 * * *" # Daily 23:00 — after implementation (22:00), CI has settled
2+
schedule: "0 3,7,11,15,19,23 * * *" # Every 4h, +1h after implementation (2,6,10,...) so CI settles. Watchdog STAGES mirrors the FIRST slot (3).
33
enabled: true
44
mode: PRIVATE
55

scripts/evolution_watchdog.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,13 @@
4141
STAGES: Dict[str, Tuple[int, str]] = {
4242
"research": (9, "md"),
4343
"introspection": (20, "json"),
44-
"analysis": (21, "json"),
45-
"implementation": (22, "md"),
46-
"integration": (23, "json"),
44+
# analysis/implementation/integration run every 4h (processing throughput);
45+
# the slot here is the FIRST daily slot — the watchdog only needs "a report
46+
# for today exists by then", and reports are date-keyed (overwritten each
47+
# run). Mirrors cron/evolution/*.yaml first hour (locked by the mirror test).
48+
"analysis": (1, "json"),
49+
"implementation": (2, "md"),
50+
"integration": (3, "json"),
4751
}
4852

4953
GRACE_HOURS = 2

tests/scripts/test_evolution_watchdog.py

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,17 @@ def test_within_grace_still_expects_yesterday(self):
3939

4040

4141
class TestStageReports:
42-
def _make_reports(self, tmp_path, date="2026-06-10", skip=(), tiny=()):
42+
def _make_reports(self, tmp_path, date=None, skip=(), tiny=()):
43+
# Each stage's report is dated at ITS OWN expected slot date (slot-aware),
44+
# so the helper stays correct regardless of per-stage schedules.
4345
for stage, (slot, ext) in STAGES.items():
4446
if stage in skip:
4547
continue
4648
d = tmp_path / stage
4749
d.mkdir(exist_ok=True)
50+
dt = date or expected_report_date(NOW, slot)
4851
content = "x" * 10 if stage in tiny else "x" * 500
49-
(d / f"{date}.{ext}").write_text(content)
52+
(d / f"{dt}.{ext}").write_text(content)
5053

5154
def test_all_present_no_alerts(self, tmp_path):
5255
self._make_reports(tmp_path)
@@ -57,7 +60,8 @@ def test_missing_report_alerts(self, tmp_path):
5760
alerts = check_stage_reports(tmp_path, NOW)
5861
assert len(alerts) == 1
5962
assert "implementation" in alerts[0]
60-
assert "2026-06-10" in alerts[0]
63+
exp = expected_report_date(NOW, STAGES["implementation"][0])
64+
assert exp in alerts[0]
6165

6266
def test_trivially_small_report_alerts(self, tmp_path):
6367
self._make_reports(tmp_path, tiny=("analysis",))
@@ -80,9 +84,14 @@ def _jobs_file(self, tmp_path, name, *, status="ok", last_run="2026-06-10T22:01:
8084

8185
def test_missing_report_quiet_when_job_ran_clean(self, tmp_path):
8286
# implementation report missing, but its cron job ran ok at/after the
83-
# 22:00 slot for the expected date (2026-06-10) → idle clean cycle, no alert.
87+
# slot for the expected date → idle clean cycle, no alert. Slot-aware.
88+
slot = STAGES["implementation"][0]
89+
exp = expected_report_date(NOW, slot)
8490
self._make_reports(tmp_path, skip=("implementation",))
85-
jf = self._jobs_file(tmp_path, "evolution-implementation")
91+
jf = self._jobs_file(
92+
tmp_path, "evolution-implementation",
93+
last_run=f"{exp}T{slot:02d}:01:00",
94+
)
8695
assert check_stage_reports(tmp_path, NOW, jf) == []
8796

8897
def test_missing_report_alerts_when_job_errored(self, tmp_path):
@@ -301,11 +310,14 @@ def test_extension_matches_output_file(self):
301310
def test_slot_hour_matches_schedule(self):
302311
for stage, (slot, _ext) in STAGES.items():
303312
spec = (self.CRON_DIR / f"{stage}.yaml").read_text()
304-
m = re.search(r'^schedule:\s*"(\d+)\s+(\d+)\s', spec, re.M)
305-
assert m, f"{stage}.yaml has no parsable daily schedule"
306-
assert int(m.group(2)) == slot, (
307-
f"watchdog STAGES says '{stage}' runs at {slot:02d}:00, "
308-
f"but {stage}.yaml schedules hour {m.group(2)}"
313+
# Hour field may be a single hour ("21") or a multi-slot list
314+
# ("1,5,9,13,17,21"); STAGES mirrors the FIRST slot.
315+
m = re.search(r'^schedule:\s*"(\d+)\s+([\d,]+)\s', spec, re.M)
316+
assert m, f"{stage}.yaml has no parsable schedule"
317+
first_hour = int(m.group(2).split(",")[0])
318+
assert first_hour == slot, (
319+
f"watchdog STAGES says '{stage}' first slot is {slot:02d}:00, "
320+
f"but {stage}.yaml's first scheduled hour is {first_hour}"
309321
)
310322

311323

0 commit comments

Comments
 (0)