From 90c851cd5af9991f20f4f06b7629e8136384269c Mon Sep 17 00:00:00 2001 From: baskduf Date: Sat, 6 Jun 2026 23:35:44 +0900 Subject: [PATCH] Validate make and just command references --- ROADMAP.md | 6 +- ...ink-failure-memory-to-regression-checks.md | 16 +- .../005-make-just-command-validation.yaml | 78 +++++ scripts/check_effectiveness_plan.py | 156 +++++++++- scripts/check_failure_memory.py | 124 +++++++- .../scripts/check_effectiveness_plan.py | 156 +++++++++- .../generic/scripts/check_failure_memory.py | 124 +++++++- tests/test_check_effectiveness_plan.py | 282 ++++++++++++++++++ tests/test_check_failure_memory.py | 150 ++++++++++ 9 files changed, 1075 insertions(+), 17 deletions(-) create mode 100644 docs/examples/task-outcomes/005-make-just-command-validation.yaml diff --git a/ROADMAP.md b/ROADMAP.md index f0c9100..a88f722 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -106,9 +106,9 @@ guidance, Next.js App Router notes, failure-memory verification, decision-memory warnings, deterministic behavior gate placement, and trigger-based task outcome evidence. Useful next additions include: -- command existence validation for `make`, `just`, Maven, Gradle, Go, and other - profile-relevant task runners referenced by failure-memory or effectiveness - records +- command existence validation beyond package scripts, root `make` targets, and + root `just` recipes for Maven, Gradle, Go, and other profile-relevant task + runners referenced by failure-memory or effectiveness records - more fixture-backed examples for provider-specific request shape, response envelopes, redaction, zero-result behavior, and provider errors - clearer ADR and failure-record boundary examples for small changes so diff --git a/docs/decisions/0004-link-failure-memory-to-regression-checks.md b/docs/decisions/0004-link-failure-memory-to-regression-checks.md index 1808c1d..764d789 100644 --- a/docs/decisions/0004-link-failure-memory-to-regression-checks.md +++ b/docs/decisions/0004-link-failure-memory-to-regression-checks.md @@ -78,17 +78,21 @@ post-push, or unknown. JavaScript package scripts and avoids passing a root command only because a nested workspace package has the same script, but it still does not prove that the script asserts the specific failure axis. +- Root `make target` and `just recipe` checks are verified against checked-in + root Makefile variants and justfile variants. This closes the same + fake-command gap for common task-runner commands, but it still does not parse + included makefiles, option-heavy invocations such as `make -C app check`, or + prove that the target asserts the specific failure axis. - Other command-shaped checks are still recognized mostly by shape. The checker - does not yet verify that `make`, `just`, Python module commands, Gradle, - Maven, Go, Rust, .NET, or other task-runner commands exist in the target - configuration. + does not yet verify that Python module commands, Gradle, Maven, Go, Rust, + .NET, or other task-runner commands exist in the target configuration. - Monorepo and workspace-specific commands need explicit target adaptation when the intended command is not runnable from the repository root. - Detection-link validation is regex-based. It blocks known non-committal phrases, but future wording may require additional test cases. -- Generic command coverage is still biased toward common JavaScript and Python - commands. Add explicit coverage before relying on this gate for Go, Rust, - Java, .NET, or Gradle-heavy targets. +- Generic command coverage is still biased toward common JavaScript, Make, + Just, and Python-shaped commands. Add explicit coverage before relying on + this gate for Go, Rust, Java, .NET, or Gradle-heavy targets. - Target repositories with pre-existing non-kit `docs/failures/*.md` schemas may need adoption-specific adaptation instead of blindly applying the generic checker. diff --git a/docs/examples/task-outcomes/005-make-just-command-validation.yaml b/docs/examples/task-outcomes/005-make-just-command-validation.yaml new file mode 100644 index 0000000..7bdc242 --- /dev/null +++ b/docs/examples/task-outcomes/005-make-just-command-validation.yaml @@ -0,0 +1,78 @@ +schema_version: 1 + +target: + repository: baskduf/harness-starter-kit + repository_ref: codex/make-just-command-validation working tree based on e98b6f40f134bcacbab0bb9cb12178f1e071bb63 + stack_or_framework: Python and Markdown harness kit + date: 2026-06-06 + agent_or_model: Codex + reviewer: Codex primary agent plus read-only subagent reviewer + +task: + id: make-just-command-validation + run_id: harness-starter-kit-005 + prompt_summary: Extend command existence validation for root make targets and just recipes. + prompt_ref: current Codex thread request to proceed with option 1 and review findings until resolved + prompt_hash: not recorded + comparable_task_group: harness-maintenance + condition: harnessed-only + expected_boundary: + - ROADMAP.md + - docs/decisions/0004-link-failure-memory-to-regression-checks.md + - docs/examples/task-outcomes/** + - scripts/check_effectiveness_plan.py + - scripts/check_failure_memory.py + - templates/generic/scripts/check_effectiveness_plan.py + - templates/generic/scripts/check_failure_memory.py + - tests/test_check_effectiveness_plan.py + - tests/test_check_failure_memory.py + - tests/test_repository_hygiene.py + known_failure_mode: Failure memory or task outcome evidence can cite fake make or just commands that look concrete but are not declared in the target repository. + +harness_context: + harness_doctor_score: previously 98/100, not treated as effectiveness proof + harness_source: + kit_url: https://github.com/baskduf/harness-starter-kit + kit_commit: e98b6f40f134bcacbab0bb9cb12178f1e071bb63 + source_tracking_ref: none; this repository is the kit source + relevant_instructions: + - AGENTS.md + - docs/decisions/0004-link-failure-memory-to-regression-checks.md + - ROADMAP.md + relevant_constraints: + - python3 -m unittest tests.test_check_failure_memory tests.test_check_effectiveness_plan tests.test_repository_hygiene + - python3 -m py_compile scripts/check_failure_memory.py scripts/check_effectiveness_plan.py templates/generic/scripts/check_failure_memory.py templates/generic/scripts/check_effectiveness_plan.py + - python3 scripts/check_effectiveness_plan.py + - python3 scripts/check_failure_memory.py + relevant_memory_records: + - docs/decisions/0004-link-failure-memory-to-regression-checks.md + - docs/decisions/0006-trigger-task-outcome-evidence-for-substantial-harness-work.md + - docs/failures/0005-failure-memory-was-not-linked-to-regression-checks.md + - docs/failures/0007-dogfood-first-pass-failures-lacked-memory-decision.md + +outcome: + files_changed: + - ROADMAP.md + - docs/decisions/0004-link-failure-memory-to-regression-checks.md + - docs/examples/task-outcomes/005-make-just-command-validation.yaml + - scripts/check_effectiveness_plan.py + - scripts/check_failure_memory.py + - templates/generic/scripts/check_effectiveness_plan.py + - templates/generic/scripts/check_failure_memory.py + - tests/test_check_effectiveness_plan.py + - tests/test_check_failure_memory.py + wrong_file_edits: 0 + repeated_known_mistake: false + verification_command: python3 -m unittest tests.test_check_failure_memory tests.test_check_effectiveness_plan tests.test_repository_hygiene && python3 -m py_compile scripts/check_failure_memory.py scripts/check_effectiveness_plan.py templates/generic/scripts/check_failure_memory.py templates/generic/scripts/check_effectiveness_plan.py + first_pass_verification: + result: failed_then_passed + drift_violations_detected: [] + human_rework_minutes: 0 + reverted_files: [] + notes: The validation is intentionally scoped to root Makefile and justfile variants; option-heavy or included-file task runners remain follow-up scope. Primary-agent review fixed a test assertion indentation issue; subagent review found and the final loop fixed task-outcome false-inclusion validation, GNU makefile precedence, make variable assignments including dotted values, just default-parameter parsing, and an inline-code backtick regex miss before final validation. + +follow_up: + harness_change_needed: false + decision_or_failure_record: docs/decisions/0004-link-failure-memory-to-regression-checks.md; no failure record because this closed a known validation gap rather than fixing a recurring failed check from this run. + include_in_effectiveness_report: false + include_in_comparable_product_task_count: false diff --git a/scripts/check_effectiveness_plan.py b/scripts/check_effectiveness_plan.py index 2ebed19..aa0f72c 100644 --- a/scripts/check_effectiveness_plan.py +++ b/scripts/check_effectiveness_plan.py @@ -97,7 +97,11 @@ re.compile(r"\b(?:tests?|specs?|fixtures?|scripts?)/[^\s,.;)]+"), re.compile(r"`?\.github/workflows/[^\s,.;)`]+`?"), re.compile(r"\b(?:npm|pnpm|yarn|bun)\s+run\s+[\w:./-]+"), - re.compile(r"\b(?:make|just)\s+[\w:./-]+"), + re.compile( + r"\bmake(?:\s+[\w.-]+=[^\s,;)`\]}]+)*\s+(?!-)[\w:./-]+" + r"(?=$|[\s,.;)`\]}])" + ), + re.compile(r"\bjust\s+(?!-)[\w:./-]+"), re.compile(r"\bpython3?\s+(?:-m\s+[\w.:-]+|scripts?/[^\s,.;)]+)"), re.compile(r"\bpytest\s+(?:-[\w-]+|tests?/[^\s,.;)]+|[\w/.-]+)"), re.compile(r"\b(?:vitest|jest|ruff|mypy|eslint)\s+[\w/.:@-]+"), @@ -116,6 +120,13 @@ PACKAGE_SCRIPT_COMMAND_RE = re.compile( r"\b(?Pnpm|pnpm|yarn|bun)\s+run\s+(?P