scaffold_repo_eval.py

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
from pathlib import Path

README_TEMPLATE = """# Repo Agent Eval Kit

This directory was scaffolded by the `harness-engineering` skill.

## What to do next

1. Replace placeholders in `manifest.json`.
2. Create 8 to 20 real tasks under `tasks/`.
3. For each task, keep `spec.md` visible and `reviewer-notes.md` private when scoring.
4. Reuse the same task set to compare harness changes over time.

## Suggested surfaces

- frontend
- backend
- tests
- migration
- review

## Scoring dimensions

- correctness
- completeness
- code quality
- verification quality
"""

SPEC_TEMPLATE = """# Task Spec

Describe the task without leaking the final implementation.

## User-visible goal

...

## Constraints

- ...

## Allowed commands

- ...

## Expected verification

- ...
"""

REVIEWER_TEMPLATE = """# Reviewer Notes

Keep this file out of the evaluated prompt.

## Ground truth

- ...

## Common failure modes

- ...

## Scoring notes

- correctness:
- completeness:
- code quality:
- verification quality:
"""


def build_manifest(repo_name: str) -> dict:
    return {
        "version": 1,
        "repo_name": repo_name,
        "owner": "replace-me",
        "generated_by": "harness-engineering",
        "scoring": {
            "correctness": 0.4,
            "completeness": 0.25,
            "code_quality": 0.2,
            "verification_quality": 0.15,
        },
        "tasks": [
            {
                "id": "example-task-001",
                "title": "Replace with a real task title",
                "surface": "frontend",
                "difficulty": "medium",
                "source_type": "pull_request",
                "source_ref": "replace-with-link-or-id",
                "prompt_file": "tasks/example-task-001/spec.md",
                "verification": [
                    {
                        "type": "command",
                        "command": "replace-with-real-command"
                    }
                ],
                "hidden_reference": "tasks/example-task-001/reviewer-notes.md",
            }
        ],
    }


def write_file(path: Path, content: str, force: bool) -> None:
    if path.exists() and not force:
        return
    path.write_text(content, encoding="utf-8")


def main() -> int:
    parser = argparse.ArgumentParser(description="Scaffold a repo-specific agent eval starter kit.")
    parser.add_argument("output_dir", help="Directory to create or update.")
    parser.add_argument("--repo-name", required=True, help="Repository name to place in the manifest.")
    parser.add_argument("--force", action="store_true", help="Overwrite existing scaffold files.")
    args = parser.parse_args()

    out = Path(args.output_dir).resolve()
    tasks_dir = out / "tasks" / "example-task-001"
    tasks_dir.mkdir(parents=True, exist_ok=True)

    manifest = build_manifest(args.repo_name)
    manifest_path = out / "manifest.json"
    readme_path = out / "README.md"
    spec_path = tasks_dir / "spec.md"
    reviewer_path = tasks_dir / "reviewer-notes.md"

    if not manifest_path.exists() or args.force:
        manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

    write_file(readme_path, README_TEMPLATE, args.force)
    write_file(spec_path, SPEC_TEMPLATE, args.force)
    write_file(reviewer_path, REVIEWER_TEMPLATE, args.force)

    print(f"Scaffolded repo eval kit at {out}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())