diff --git a/.github/workflows/self-check.yml b/.github/workflows/self-check.yml index 00f9694..3ab4354 100644 --- a/.github/workflows/self-check.yml +++ b/.github/workflows/self-check.yml @@ -13,7 +13,8 @@ jobs: - uses: actions/checkout@v4 - uses: ./ with: + mode: verify instruction-file: tests/fixtures/sample-claude.md output-dir: src env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index feeffb7..d9792f6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ coverage/ # Environment variables .env +# Phase 0 data collection cache and output +.cache/ +data/ + # Scripts (local utilities, not shipped) scripts/ @@ -27,3 +31,6 @@ test-treesitter*.mjs .github/ruleprobe-e2e-verification-guide.md docs/ruleprobe-build-guide.md .ruleprobe-semantic/ +.codex +.codex-tmp/ +.ruleprobe-semantic/ diff --git a/.npmignore b/.npmignore index 2b7b9e9..66817fa 100644 --- a/.npmignore +++ b/.npmignore @@ -6,6 +6,10 @@ scraped-instructions/ # Development files .vscode/ +.claude/ +.codex/ +.codex +.codex-tmp/ .github/ docs/ SECURITY.md @@ -15,6 +19,9 @@ tsconfig.json .env .env.* +# RuleProbe semantic cache +.ruleprobe-semantic/ + # Git .git/ .gitignore diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..84e45a8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,129 @@ +# AGENTS.md + +Instructions for AI coding agents working on the RuleProbe codebase. + +RuleProbe verifies whether agents follow instruction files. This file is the instruction file for agents working on RuleProbe itself. It is parsed by RuleProbe in the self-check workflow, so every rule below is written to be machine-verifiable. + +## Project + +- Repository: https://github.com/moonrunnerkc/ruleprobe +- Package: https://www.npmjs.com/package/ruleprobe +- Language: TypeScript (strict) +- Runtime: Node.js >= 18 +- License: MIT + +## Build and Test + +- Use `npm` as the package manager. Do not switch to pnpm, yarn, or bun. +- Use `vitest` as the test runner. Do not introduce jest or mocha. +- Run `npm test` before declaring any change complete. +- Run `npm run build` to verify the TypeScript compile is clean. +- A `package-lock.json` must exist at the repo root. +- Pinned dependency versions are required in `package.json` (no `^` or `~` ranges). + +## Code Style + +- Use TypeScript strict mode. Never disable strict checks. +- Never use `any`. Use `unknown` and narrow, or define a precise type. +- Always use named exports. Never use default exports. +- Use camelCase for variables and functions. +- Use PascalCase for types, interfaces, and classes. +- Use kebab-case for filenames. +- Prefer `const` over `let`. +- Prefer `interface` over `type` for object shapes. +- Prefer `async/await` over `.then()` chains. +- Never use `console.log` in production code. Use the structured logger. +- Never use `eval`. +- No magic numbers without a named constant or inline comment justifying the value. +- No em dashes anywhere in source, comments, docs, or commit messages. Use commas, colons, semicolons, parentheses, or separate sentences. +- Files must stay under 300 lines. If a file approaches the limit, decompose it. +- Add full JSDoc to every exported function, class, and type. +- Avoid nested ternaries. Use early returns to flatten control flow. + +## Architecture Boundaries + +- Parser code lives under `src/parser/`. Do not call verifier code from the parser. +- Verifier engines live under `src/verifiers/`. Each engine exports a single entrypoint that returns `VerificationResult[]`. +- The semantic tier lives under `src/semantic/`. Source code must never leave the user's machine. Only numeric AST vectors, opaque sub-tree hashes, boolean flags, and rule text may be sent to an LLM. +- The CLI lives under `src/cli/`. Commands compose pipeline functions; they do not contain pipeline logic. +- Shared types live under `src/types/`. Do not duplicate type definitions across modules. + +## Verifier Engines + +There are eight verifier engines: `ast`, `filesystem`, `regex`, `treesitter`, `preference`, `tooling`, `config-file`, `git-history`. When adding a check, use an existing engine. Adding a new engine requires a written justification in the PR description. + +- AST checks must use `ts-morph`. Do not parse TypeScript with regex. +- Tree-sitter WASM loading must fail gracefully. If a grammar fails to load, log a warning and skip the check. Never block other verifiers. +- Type-aware AST checks (implicit any, unused exports, unresolved imports) require a `tsconfig.json` and the `--project` flag. Skip cleanly when absent. +- Semantic tier failures must never prevent deterministic results from returning. + +## Parser Rules + +- The parser supports 7 instruction file formats: `CLAUDE.md`, `AGENTS.md`, `.cursorrules`, `copilot-instructions.md`, `GEMINI.md`, `.windsurfrules`, `.rules`. Parser changes must not break extraction for any of them. +- Lines that cannot be mapped to a deterministic check go into the `unparseable` array. Do not invent rules to inflate the parse rate. +- LLM-extracted rules must be tagged `extractionMethod: 'llm'` with `confidence: 'medium'`. +- Rubric-decomposed rules must be tagged `confidence: 'low'`. + +## Testing + +- Every new function requires at least one test. +- Test files live under `tests/` and mirror the `src/` directory structure. +- Test names describe behavior, not implementation. +- Tests must validate real behavior, not wiring. Reading the implementation should not be required to understand what a test verifies. +- No mocks except at external API boundaries (Anthropic API, OpenAI API, GitHub API, filesystem boundaries when testing error paths). +- Use `describe` and `it` blocks. Do not use `test()` directly. +- Never use `console.log` in tests. +- New matchers require: the matcher implementation, a test file with real-world instruction examples, and an entry in `docs/matchers.md`. + +## Security + +- Never execute scanned code. +- Never modify files in the scanned directory. +- All user-supplied paths must be resolved and bounded to the working directory. +- Symlinks resolving outside the project root must be skipped unless `--allow-symlinks` is passed. +- Never write API keys to disk or include them in reports. +- Network calls are allowed only when the user opts in: `--llm-extract`, `--rubric-decompose`, `--semantic`, or `ruleprobe run`. + +## Imports + +- No path aliases. Use relative imports. +- No barrel imports from deep internal modules. Import directly from the file that defines the symbol. +- No wildcard imports. +- Do not import lodash. Use native JavaScript methods. + +## Error Handling + +- Never use empty catch blocks. +- Never swallow errors silently. Log or rethrow. +- Catch clauses must declare the caught type as `unknown` and narrow. +- Error messages must include what failed and what to do about it. + +## Git Workflow + +- Use conventional commit messages: `feat:`, `fix:`, `docs:`, `refactor:`, `test:`, `chore:`. +- Branch names use kebab-case: `feat/new-matcher`, `fix/parser-bug`. +- Pull requests must pass the self-check workflow before merge. +- Do not commit `.env` files or any file containing secrets. + +## Configuration Files + +- ESLint config lives at `.eslintrc.json` or `eslint.config.js`. +- Prettier config lives at `.prettierrc` or `.prettierrc.json`. +- TypeScript config lives at `tsconfig.json`. +- Vitest config lives at `vitest.config.ts`. +- Do not add competing tools (Biome alongside ESLint, Rome, etc.). + +## Documentation + +- Update `docs/matchers.md` when adding or modifying a matcher. +- Update `docs/cli-reference.md` when adding or changing a CLI command or flag. +- Update `docs/api-reference.md` when changing the public API surface. +- Update the relevant release notes file under `docs/` for any user-facing change. + +## What Not To Do + +- Do not introduce a new agent SDK adapter without a corresponding integration test. +- Do not weaken the security boundary to make a check easier to implement. +- Do not push deterministic logic into the semantic tier because it is easier to write. +- Do not add features that require API keys for the default deterministic path. +- Do not add dependencies without a clear justification. Each dependency is a maintenance cost. diff --git a/CHANGELOG.md b/CHANGELOG.md index f575b1e..fd7266d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,49 @@ All notable changes to this project will be documented in this file. +## [4.5.0] - 2026-05-08 + +### Breaking Changes + +- **Default action mode changed.** The primary workflow is now `lint-config`, `drift`, and `extract`, not `verify`. The GitHub Action defaults to drift detection mode. +- **`compare` command removed.** Agent comparison is no longer a primary use case. Use drift detection instead. +- **`tasks` and `task` commands removed.** Task template listing and printing removed. +- **`run` command removed.** Agent invocation via the Claude Agent SDK removed. The `@anthropic-ai/claude-agent-sdk` is no longer a dependency. +- **Runner module removed from public API.** `buildAgentConfig`, `invokeAgent`, `isAgentSdkAvailable`, `hasAgentOutput`, `watchForCompletion`, `countCodeFiles`, `AgentInvocationConfig`, `RunOptions`, `InvocationResult`, `WatchOptions`, `WatchResult` are no longer exported. +- **`formatComparisonMarkdown` removed.** The comparison report formatter is no longer exported from `reporter/index`. +- **`verify` command deprecated.** Still works, but the primary workflow is now translate, detect drift, and extract. +- **67 unmappable matchers removed.** Categories removed: `test-requirement`, `dependency`, `preference`, `file-structure`, `tooling`, `testing`, `workflow`. Verifier types removed: `treesitter`, `preference`, `tooling`, `config-file`, `git-history`. The remaining 34 matchers all map to ESLint rules. +- **`RuleCategory` union narrowed.** Removed: `test-requirement`, `dependency`, `preference`, `file-structure`, `tooling`, `testing`, `workflow`. Remaining: `naming`, `forbidden-pattern`, `structure`, `import-pattern`, `error-handling`, `type-safety`, `code-style`, `agent-behavior`. +- **`VerifierType` union narrowed.** Removed: `treesitter`, `preference`, `tooling`, `config-file`, `git-history`. Remaining: `ast`, `regex`, `filesystem`. + +### New Features + +- **`lint-config` command.** Translates an instruction file into a flat or legacy ESLint config. Unmappable rules appear as comments in the output. +- **`drift` command.** Compares an instruction file against an existing ESLint config. Reports rules present in one but missing from the other, severity mismatches, and config argument differences. +- **`extract` command.** Parses an ESLint config and emits a markdown rules section suitable for pasting into an instruction file. + +### Removed + +- `compare` command and `formatComparisonMarkdown` export. +- `tasks` and `task` commands and `src/runner/task-templates/` directory. +- `run` command and `src/runner/agent-configs.ts`, `src/runner/agent-invoker.ts`, `src/runner/watch-mode.ts`. +- Matcher files: `rule-patterns-preference.ts`, `rule-patterns-file-structure.ts`, `rule-patterns-tooling.ts`, `rule-patterns-testing.ts`, `rule-patterns-config-file.ts`, `rule-patterns-git-history.ts`. +- Individual unmappable matchers from remaining files: `test-files-exist`, `test-named-pattern`, `structure-strict-mode`, `error-async-try-catch`, `structure-typescript-required`, `error-log-contextual`, `import-no-unresolved`, `naming-python-snake-case`, `naming-python-class`, `naming-go-conventions`, `style-python-function-length`, `style-go-function-length`, `style-concise-conditionals`, `naming-kebab-case-directories`, `structure-no-barrel-files`, `test-no-settimeout`, `test-no-only`, `test-no-skip`, `import-banned-package`, `structure-readme-exists`, `structure-changelog-exists`, `structure-formatter-config`, `dependency-pinned-versions`. + +### Stats + +| Metric | v4.0.0 | v4.5.0 | +|--------|--------|--------| +| Rule matchers | ~103 | 34 | +| Rule categories | 14 | 7 (+ `agent-behavior`) | +| Verifier engines | 8 | 3 | +| CLI commands | 9 | 6 | +| Public API exports | ~40 | ~25 | + +## [4.0.0] - 2026-04-28 + +Major release consolidating the three-repo architecture. See [docs/release-v4.0.0.md](docs/release-v4.0.0.md) for full details. + ## [1.0.0] - 2026-04-07 14 commits, 100 files changed, +9,017 lines since v0.1.0. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..2c23c08 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,103 @@ +# CLAUDE.md + +Instructions for Claude (Claude Code, Claude SDK, claude.ai) when working on the RuleProbe codebase. + +The full engineering rules live in [AGENTS.md](./AGENTS.md). Read that file first. Everything below is Claude-specific workflow guidance that supplements it. When the two conflict, AGENTS.md wins. + +## Project + +- Repository: https://github.com/moonrunnerkc/ruleprobe +- Package: https://www.npmjs.com/package/ruleprobe +- Language: TypeScript (strict) +- Runtime: Node.js >= 18 + +## Working Style + +- Make targeted edits over full rewrites when a specific issue is flagged. +- Never pad documentation, README, or release notes with marginal results. Only genuinely interesting findings belong. +- Run verification before treating any feature as done: `npm test` and `npm run build`. +- Push back when a claim is not backed by real data. Do not invent metrics, repo counts, or compliance scores. +- Use `git status` and `git diff` to confirm what you actually changed before reporting completion. + +## Engineering Standards + +These are enforced by the self-check workflow (RuleProbe verifies its own codebase). Following them keeps the build green. + +- Always use TypeScript strict mode. +- Never use `any`. Use `unknown` and narrow. +- Always use named exports. Never use default exports. +- Use kebab-case for filenames. +- Use camelCase for variables and functions. +- Use PascalCase for types, interfaces, and classes. +- Prefer `const` over `let`. +- Prefer `interface` over `type` for object shapes. +- Files must stay under 300 lines. +- Add full JSDoc to every exported symbol. +- Never use em dashes anywhere. +- No magic numbers without a named constant. + +## Tooling + +- Use `npm` as the package manager. +- Use `vitest` as the test runner. +- Use `eslint` for linting and `prettier` for formatting. +- Do not introduce competing tools (no biome, no jest, no pnpm). + +## Pipeline Boundaries + +When editing pipeline code, respect the three-stage boundary: parse, verify, report. Each stage is independently testable and lives under its own directory in `src/`. + +- Parser code lives under `src/parser/`. The parser must not call verifier code. +- Verifier engines live under `src/verifiers/`. Each engine returns `VerificationResult[]`. +- Report generation lives under `src/report/`. Reports must not run verifications. +- The semantic tier under `src/semantic/` must never send source code, file paths, variable names, or comments to any LLM. + +## Tree-sitter and Type-aware Checks + +- Tree-sitter WASM grammars must fail gracefully. If a grammar fails to load, log a warning and continue. Never block other verifiers. +- Type-aware AST checks require `--project` and a valid `tsconfig.json`. Skip cleanly when absent. +- Semantic tier failures must never prevent deterministic results from returning. + +## Testing + +- Every new function requires at least one test. +- Test files live under `tests/` and mirror the `src/` structure. +- Tests must validate real behavior, not wiring. +- No mocks except at external API boundaries. +- Use `describe` and `it` blocks. +- Never use `console.log` in tests. +- New matchers require a test file with real-world instruction examples. + +## Security + +- Never execute scanned code. +- Never modify files in the scanned directory. +- Bound all paths to the working directory. +- Skip symlinks resolving outside the project root unless `--allow-symlinks` is passed. +- Never write API keys to disk. + +## Claude Code Workflow + +- When invoked through Claude Code, prefer running tests and builds in the integrated terminal so output is captured in the session. +- For multi-file changes, list the files you intend to touch before editing. +- For changes that affect the parser, run `ruleprobe parse` against `tests/fixtures/` instruction files to confirm no regression in extraction. +- For changes that affect a verifier, run `npm test -- ` before running the full suite. +- For changes that touch the semantic tier, manually verify that no source code, file paths, variable names, or comments appear in any outbound payload. + +## Commit Messages + +Use conventional commit format: `feat:`, `fix:`, `docs:`, `refactor:`, `test:`, `chore:`. + +Examples: +- `feat: add tree-sitter naming check for Rust` +- `fix: parser drops rules with backtick-wrapped patterns` +- `docs: update matchers.md with new file-structure entries` + +## What Not To Do + +- Do not refactor code outside the scope of the requested change. +- Do not weaken the security boundary to simplify an implementation. +- Do not push deterministic logic into the semantic tier. +- Do not add dependencies without justification. +- Do not write release notes that make claims unsupported by the actual diff. +- Do not present completion as fact without running tests and build. diff --git a/README.md b/README.md index aa259a3..47915a1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

RuleProbe

- Verify whether AI coding agents actually follow the instruction files they're given. + Translate instruction files into ESLint configs, detect drift, and extract rules.

npm version @@ -15,11 +15,15 @@ ## Why -Every AI coding agent reads an instruction file. None of them prove they followed it. +AI coding agents read your `CLAUDE.md`, `AGENTS.md`, or `.cursorrules`, but nothing checks whether your ESLint config actually enforces the same rules. Drift accumulates: the instruction file says "use camelCase" but ESLint never checks it, or ESLint enforces rules the instruction file never mentioned. -You write `CLAUDE.md` or `AGENTS.md` with specific rules: camelCase variables, no `any` types, named exports only, test files for every source file. The agent says "Done." But did it actually follow them? Your code review catches some violations, misses others, and doesn't scale. +RuleProbe closes that gap. It reads your instruction file, extracts machine-verifiable rules, and bridges them to your tooling in three ways: -RuleProbe reads the same instruction file, extracts the machine-verifiable rules, and checks agent output against each one. Compliance scores with file paths and line numbers as evidence. Deterministic and reproducible by default. Optional semantic analysis for pattern-matching and consistency rules that require codebase-aware judgment. +**Translate.** Generate an ESLint config from your instruction file. `ruleprobe lint-config CLAUDE.md` produces a flat or legacy config you can drop into your project immediately. + +**Detect drift.** Compare your instruction file against an existing ESLint config. `ruleprobe drift CLAUDE.md .eslintrc.json` reports rules present in one but missing from the other, severity mismatches, and config argument differences. + +**Extract.** Pull a rules section out of an ESLint config for pasting into an instruction file. `ruleprobe extract .eslintrc.json` converts ESLint rules back into human-readable instruction prose. ## Quick Start @@ -33,41 +37,53 @@ Or run it directly: npx ruleprobe --help ``` -**Parse an instruction file** to see what rules RuleProbe can extract: +**Translate an instruction file to ESLint config:** ```bash -ruleprobe parse CLAUDE.md -ruleprobe parse AGENTS.md --show-unparseable +ruleprobe lint-config CLAUDE.md +ruleprobe lint-config AGENTS.md --format legacy --output .eslintrc.json ``` -**Verify agent output** against those rules: +**Detect drift between instructions and ESLint:** ```bash -ruleprobe verify CLAUDE.md ./agent-output --format text -ruleprobe verify AGENTS.md ./src --format summary --threshold 0.9 +ruleprobe drift CLAUDE.md .eslintrc.json +ruleprobe drift CLAUDE.md .eslintrc.json --format markdown +``` + +**Extract rules from an ESLint config:** + +```bash +ruleprobe extract .eslintrc.json +ruleprobe extract .eslintrc.json --output rules-section.md ``` -**Analyze a whole project** across all instruction files: +**Parse an instruction file** to see what rules RuleProbe can extract: ```bash -ruleprobe analyze ./my-project +ruleprobe parse CLAUDE.md +ruleprobe parse AGENTS.md --show-unparseable ``` -Every failure includes the file, line number, and what was found. Preference rules return compliance ratios instead of binary pass/fail. +**Verify agent output** against extracted rules (legacy mode). Valid formats: `text`, `json`, `markdown`, `rdjson`, `summary`, `detailed`, `ci`. + +```bash +ruleprobe verify CLAUDE.md ./agent-output --format text +``` ## What It Does -**Parse.** Reads 7 instruction file formats (CLAUDE.md, AGENTS.md, .cursorrules, copilot-instructions.md, GEMINI.md, .windsurfrules, .rules) and extracts rules that can be checked mechanically. Each rule gets a qualifier (`always`, `prefer`, `when-possible`, `avoid-unless`, `try-to`, `never`) detected from the instruction text, and the markdown section it came from. Subjective instructions like "write clean code" are reported as unparseable so you know what was skipped. +**Translate.** Reads 7 instruction file formats (CLAUDE.md, AGENTS.md, .cursorrules, copilot-instructions.md, GEMINI.md, .windsurfrules, .rules) and emits an ESLint config. Flat config is the default; pass `--format legacy` for `.eslintrc.json` output. Each extractable rule maps to an ESLint rule with appropriate severity and options. Rules that have no ESLint equivalent appear as comments in the output so you know what wasn't covered. -**Verify.** Runs each extracted rule against a directory of agent-generated code. Eight verifier engines: AST (ts-morph), filesystem, regex, tree-sitter (TypeScript, JavaScript, Python, Go), preference (compliance ratios for "prefer X over Y" patterns), tooling (package.json/lockfile/config checks), config-file (linter/formatter/build tool configs), and git-history (commit message and workflow checks). No LLM evaluation by default; results are deterministic. +**Detect drift.** Compares parsed rules against an existing ESLint config. Reports rules in the instruction file but missing from ESLint (you're not enforcing what you said), rules in ESLint but not in the instructions (you're enforcing what you never stated), severity mismatches, and argument differences. Use `--format markdown` for PR-ready output. -**Analyze.** Discovers all instruction files in a project, parses each, and cross-references them. Detects conflicts (same topic, contradictory rules across files) and redundancies (same rule in multiple files). Returns a coverage map showing which categories each file addresses. Pass `--semantic` with an Anthropic API key to add structural pattern analysis. +**Extract.** Parses an ESLint config and generates a markdown rules section you can paste into an instruction file. Stylistic rules (semicolons, quotes) are reported but skipped from the output by default since they don't carry meaningful instruction content. -**LLM Extract (opt-in).** Pass `--llm-extract` to send unparseable lines through an OpenAI-compatible API. LLM-extracted rules are labeled with `extractionMethod: 'llm'` and `confidence: 'medium'`. Requires `OPENAI_API_KEY`. +**Parse.** Extracts machine-verifiable rules from instruction files with qualifiers (`always`, `prefer`, `when-possible`, `avoid-unless`, `try-to`, `never`) and section context. Subjective lines like "write clean code" are reported as unparseable. -**Compare.** Point RuleProbe at outputs from two or more agents and get a side-by-side comparison table showing which rules each one followed. +**Verify.** Runs extracted rules against a directory of code. Deterministic by default, optional semantic analysis available. The original mode, still supported but no longer the primary focus. -**GitHub Action.** Composite action for any repo. Runs `ruleprobe verify` on every PR, posts results as a comment, and optionally outputs reviewdog rdjson format for inline annotations. +**Analyze.** Discovers all instruction files in a project, parses each, and cross-references them for conflicts and redundancies. Pass `--semantic` for structural pattern analysis. ## Configuration @@ -109,24 +125,21 @@ export default defineConfig({ `defineConfig()` is a no-op passthrough that provides type checking in TypeScript configs. JSON configs work without it. -Custom rules use the same verifier types (`ast`, `regex`, `filesystem`, `treesitter`, `preference`, `tooling`, `config-file`, `git-history`) and pattern types as extracted rules. Any pattern type listed in the Supported Rule Types table works as a custom rule pattern. +Custom rules use the same verifier types (`ast`, `regex`, `filesystem`) and pattern types as extracted rules. ## CLI Reference -Seven commands: `parse`, `verify`, `analyze`, `compare`, `tasks`, `task`, `run`. Quick examples: +Six commands: `parse`, `verify`, `analyze`, `lint-config`, `drift`, `extract`. ```bash ruleprobe parse CLAUDE.md --show-unparseable -ruleprobe verify AGENTS.md ./src --format summary --threshold 0.9 +ruleprobe verify AGENTS.md ./src --format detailed --threshold 0.9 +ruleprobe lint-config CLAUDE.md --format flat --output eslint.config.js +ruleprobe drift CLAUDE.md .eslintrc.json --format markdown +ruleprobe extract .eslintrc.json --output rules-section.md ruleprobe analyze ./my-project --format json -ruleprobe compare AGENTS.md ./claude-output ./copilot-output --agents claude,copilot -ruleprobe tasks -ruleprobe task rest-endpoint -ruleprobe run CLAUDE.md --task rest-endpoint --agent claude-code --format text ``` -The `analyze` command supports semantic analysis flags (`--semantic`, `--anthropic-key`, `--cost-report`, `--semantic-log`). - Full command reference with all options: [docs/cli-reference.md](docs/cli-reference.md) ## GitHub Action @@ -134,10 +147,10 @@ Full command reference with all options: [docs/cli-reference.md](docs/cli-refere Drop this into `.github/workflows/ruleprobe.yml`: ```yaml -name: RuleProbe +name: RuleProbe Drift on: [pull_request] jobs: - check-rules: + drift-check: runs-on: ubuntu-latest permissions: contents: read @@ -146,15 +159,14 @@ jobs: - uses: actions/checkout@v4 - uses: moonrunnerkc/ruleprobe@v4 with: - instruction-file: AGENTS.md - output-dir: src + instruction-file: CLAUDE.md env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` -No API keys needed, deterministic results, runs in seconds. +No API keys needed, deterministic results, runs in seconds. The action only runs drift detection when instruction files or ESLint config files are changed in the PR, skipping otherwise. -> **Note:** `@v4` tracks the latest v4.x release. Pin to a specific tag (e.g., `@v4.0.0`) for reproducible builds. +> **Note:** `@v4` tracks the latest v4.x release. Pin to a specific tag (e.g., `@v4.5.0`) for reproducible builds.

Full action options @@ -162,36 +174,30 @@ No API keys needed, deterministic results, runs in seconds. ```yaml - uses: moonrunnerkc/ruleprobe@v4 with: - instruction-file: AGENTS.md - output-dir: src - agent: ci - model: unknown - format: text - severity: all - fail-on-violation: "true" - post-comment: "true" - reviewdog-format: "false" + mode: drift + instruction-file: CLAUDE.md + eslint-file: .eslintrc.json + regenerate-on-drift: "false" + comment-on-pr: "true" + fail-on-drift: "false" ``` | Input | Default | Description | |-------|---------|-------------| +| `mode` | `drift` | Run mode: `drift` (default) or `verify` (legacy) | | `instruction-file` | (required) | Path to instruction file | -| `output-dir` | `src` | Directory containing code to verify | -| `agent` | `ci` | Agent identifier for report metadata | -| `model` | `unknown` | Model identifier for report metadata | -| `format` | `text` | Report format: text, json, or markdown | -| `severity` | `all` | Filter: error, warning, or all | -| `fail-on-violation` | `true` | Fail the check on any violation | -| `post-comment` | `true` | Post results as a PR comment | -| `reviewdog-format` | `false` | Also output rdjson for reviewdog | +| `eslint-file` | (auto-detected) | Path to ESLint config file | +| `regenerate-on-drift` | `false` | Open a follow-up PR with regenerated config when drift is detected | +| `comment-on-pr` | `true` | Post drift results as a PR comment | +| `fail-on-drift` | `false` | Fail the action if drift is detected | -Outputs: `score`, `passed`, `failed`, `total` (available to downstream steps). +Drift mode outputs: `drift-count`, `has-drift`.
## Programmatic API -Core pipeline functions, project analysis, config, LLM extraction, and agent invocation are all exported: +Core pipeline functions are exported for programmatic use: ```typescript import { parseInstructionFile, verifyOutput, generateReport, formatReport } from 'ruleprobe'; @@ -199,7 +205,7 @@ import { parseInstructionFile, verifyOutput, generateReport, formatReport } from const ruleSet = parseInstructionFile('CLAUDE.md'); const results = await verifyOutput(ruleSet, './agent-output'); const report = generateReport( - { agent: 'claude-code', model: 'opus-4', taskTemplateId: 'rest-endpoint', + { agent: 'claude-code', model: 'opus-4', taskTemplateId: 'manual', outputDir: './agent-output', timestamp: new Date().toISOString(), durationSeconds: null }, ruleSet, results, @@ -207,203 +213,79 @@ const report = generateReport( console.log(formatReport(report, 'summary')); ``` -Full API reference with all exported functions and types: [docs/api-reference.md](docs/api-reference.md) +Full API reference: [docs/api-reference.md](docs/api-reference.md) ## How It Works ``` - Instruction File --> Rule Parser --> RuleSet --+ - +--> Verifier --> Adherence Report - Agent Output ----------+ +Instruction File --> Rule Parser --> RuleSet --+ + +--> Verifier --> Adherence Report + Agent Output ---------+ + +Instruction File --> Rule Parser --> RuleSet --> Mapper --> ESLint Config + +Instruction File --> Rule Parser --> RuleSet --+ +ESLint Config --> Parser --> Parsed --+--> Drift Report + +ESLint Config --> Extractor --> Markdown Rules Section ``` -The parser reads your instruction file and identifies lines that map to deterministic checks. Each rule gets a category, a verifier type, a pattern, and a qualifier (how strictly the instruction is worded). Eight verifier engines handle different rule types: +The parser reads your instruction file and identifies lines that map to deterministic checks. Each rule gets a category, a verifier type, a pattern, and a qualifier (how strictly the instruction is worded). Three verifier engines handle different rule types: | Engine | What it checks | |--------|---------------| | AST (ts-morph) | Code structure, naming, type safety, imports for TypeScript/JavaScript | | Filesystem | File existence, naming conventions, directory structure | -| Regex | Content patterns, forbidden strings, test conventions | -| Tree-sitter | Naming and function-length checks for TypeScript, JavaScript, Python, Go | -| Preference | Compliance ratios for "prefer X over Y" patterns (8 built-in pairs) | -| Tooling | Package manager, test runner, linter/formatter presence in package.json and lockfiles | -| Config-file | Linter, formatter, and build tool configuration file contents | -| Git-history | Commit message conventions, branch naming, workflow patterns | +| Regex | Content patterns, forbidden strings | -The report collects compliance scores with evidence for every rule. +The mapper translates extractable rules into ESLint rule entries. The drift detector compares parsed rules against an existing ESLint config. The extractor reverses the mapping: ESLint rules become human-readable instruction prose. ## Supported Rule Types -102 built-in matchers across 14 categories: - -| Category | Count | Verifier(s) | Examples | -|----------|------:|-------------|----------| -| naming | 9 | AST, Filesystem, Tree-sitter | camelCase variables, PascalCase types, kebab-case files | -| forbidden-pattern | 5 | AST, Regex | no `any`, no `console.log`, no `eval` | -| structure | 9 | AST, Filesystem | strict mode, named exports, JSDoc, max file length | -| test-requirement | 5 | AST, Filesystem, Regex | test file existence, test naming conventions | -| import-pattern | 5 | AST, Regex | no path aliases, no barrel imports, no wildcard imports | -| error-handling | 4 | AST | no empty catch, no swallowed errors, typed catches, error boundaries | -| type-safety | 6 | AST, Regex | no type assertions, no non-null assertions, no enums | -| code-style | 12 | AST, Regex, Tree-sitter | early returns, no magic numbers, no nested ternaries | -| dependency | 2 | Filesystem | pinned dependency versions, lockfile presence | -| preference | 8 | Preference | const over let, named over default exports, interface over type, async/await over .then() | -| file-structure | 5 | Filesystem | tests directory, components directory, .env file, module index files | -| tooling | 14 | Tooling | pnpm/yarn/bun, vitest/jest/pytest, eslint/prettier/biome, bundler configs | -| testing | 3 | Filesystem, Regex | test colocation, describe/it blocks, no console in tests | -| workflow | 15 | Config-file, Git-history | commit conventions, CI configs, linter/formatter settings, build tool configs | - -Full table with example instructions and check details: [docs/matchers.md](docs/matchers.md) - -### Compliance scoring +34 mappable matchers across 7 categories produce ESLint config entries: -Every rule result includes a `compliance` field (0 to 1): +| Category | Count | Verifier | Examples | +|----------|------:|----------|----------| +| naming | 5 | AST, Filesystem | camelCase variables, PascalCase types, kebab-case files, UPPER_CASE constants | +| forbidden-pattern | 5 | AST, Regex | no `any`, no `console.log`, no `var`, max line length, no `TODO` comments | +| structure | 4 | AST, Filesystem | named exports, JSDoc required, max file length, no unused exports | +| import-pattern | 4 | AST | no path aliases, no deep relative imports, no namespace imports, no wildcard exports | +| error-handling | 2 | AST | no empty catch, throw Error only | +| type-safety | 5 | AST, Regex | no enums, no type assertions, no non-null assertions, no implicit any, no ts directives | +| code-style | 9 | AST, Regex | prefer const, no else after return, no nested ternary, no magic numbers, semicolons, quotes, max function length, max params, no TODO comments | -- **Deterministic checks** (file exists, no `any` types): compliance is 0 or 1 -- **Preference checks** (prefer const over let): compliance is the ratio (0.85 = 85% const usage) -- **Coverage checks** (test colocation): compliance is the percentage of source files with tests -- **Tooling checks**: compliance is 1 if present, 0.5 if present with a competitor, 0 if absent - -The `--threshold` option (default 0.8) controls what compliance level counts as passing. - -## Semantic Analysis - -The deterministic engine handles rules with clear patterns. Rules like "follow existing patterns," "maintain consistency," or qualified rules ("when possible," "avoid unless") require codebase-aware judgment. The semantic tier handles these. - -**How it works:** RuleProbe extracts raw AST vectors locally (node type counts, sub-tree hashes, nesting depths). No source code, variable names, comments, or file paths ever leave your machine. The vectors are analyzed locally using structural fingerprinting and similarity scoring. An LLM is consulted only when vector similarity is ambiguous, and it receives only numeric data with rule text, never code. LLM calls go directly to Anthropic's API with your own key. - -```bash -ruleprobe analyze ./my-project --semantic -ruleprobe analyze ./my-project --semantic --cost-report -``` +Rules that can't map to ESLint (test file requirements, project config, git conventions) are reported as unmappable so you can enforce them through other tooling. -| Flag | Description | -|------|-------------| -| `--semantic` | Enable semantic analysis (requires `ANTHROPIC_API_KEY`) | -| `--anthropic-key ` | Anthropic API key (also: `ANTHROPIC_API_KEY` env var or `.ruleprobe/config.json`) | -| `--max-llm-calls ` | Cap LLM calls per analysis (default: 20) | -| `--no-cache` | Disable profile caching | -| `--semantic-log` | Print what was sent/received to stdout | -| `--cost-report` | Show token cost breakdown | - -Without `--semantic`, the analyze command runs deterministic analysis only. If no Anthropic API key is available, semantic analysis is skipped gracefully and deterministic results are still returned. - -## Authentication - -Most of RuleProbe works offline with no API keys. Opt-in features that use external APIs: - -| Feature | Flag(s) | Required env var | When you need it | -|---------|---------|-----------------|------------------| -| LLM rule extraction | `--llm-extract` | `OPENAI_API_KEY` | Extracting rules from unparseable instruction lines | -| Rubric decomposition | `--rubric-decompose` | `OPENAI_API_KEY` | Breaking subjective rules into concrete checks | -| Semantic analysis | `--semantic` | `ANTHROPIC_API_KEY` | Structural pattern and consistency checks | -| Agent invocation (SDK mode) | `ruleprobe run --agent claude-code` | `ANTHROPIC_API_KEY` | Invoking Claude to generate code, then verifying | -| GitHub Action | `uses: moonrunnerkc/ruleprobe@v4` | `GITHUB_TOKEN` | CI, PR comments | - -`parse`, `verify`, `compare`, `tasks`, and `task` work entirely offline. `analyze` works offline for deterministic analysis; `--semantic` requires `ANTHROPIC_API_KEY` for LLM calls. - -## Tree-sitter Support - -TypeScript, JavaScript, Python, and Go get naming and function-length checks via tree-sitter WASM grammars. The grammar packages (`tree-sitter-typescript`, `tree-sitter-javascript`, `tree-sitter-python`, `tree-sitter-go`, `web-tree-sitter`) ship as regular dependencies; no extra install step is required. WASM binaries are loaded at runtime from the installed packages. If loading fails (unsupported platform, missing native build), tree-sitter checks are skipped and other verifiers still run. +Full matcher details: [docs/matchers.md](docs/matchers.md) ## Security -RuleProbe never executes scanned code, never makes network calls (unless you opt in with `--llm-extract`, `--rubric-decompose`, `--semantic`, or `ruleprobe run`), and never modifies files in the scanned directory. User-supplied paths are resolved and bounded to the working directory; symlinks outside the project are skipped unless you pass `--allow-symlinks`. All dependencies are pinned to exact versions. +RuleProbe never executes scanned code, never makes network calls (unless you opt in with `--llm-extract`, `--rubric-decompose`, or `--semantic`), and never modifies files in the scanned directory. User-supplied paths are resolved and bounded to the working directory; symlinks outside the project are skipped unless you pass `--allow-symlinks`. All dependencies are pinned to exact versions. When `--semantic` is enabled, all analysis runs locally. The only network calls are to the Anthropic API for LLM judgment when vector similarity is ambiguous (using your own `ANTHROPIC_API_KEY`). Only numeric AST vectors, opaque sub-tree hashes, boolean flags, and rule text are sent to the LLM. No source code, variable names, comments, import paths, or file paths leave the machine. See [SECURITY.md](SECURITY.md) for the full model. ## Limitations -- **TypeScript gets the deepest coverage.** ts-morph gives full AST analysis for TypeScript and JavaScript across all 14 categories. Python, Go, TypeScript, and JavaScript get naming and function-length checks via tree-sitter. No Rust, Java, or C# AST support yet. -- **Subjective rules stay subjective.** "Write clean code" has no deterministic check. `--rubric-decompose` uses an LLM to break subjective instructions into weighted concrete checks, tagged with `confidence: 'low'`. Lines with no measurable proxy stay in the unparseable array. Requires `OPENAI_API_KEY`. -- **Agent invocation covers Claude SDK and watch mode only.** The `run` command invokes agents via the Claude Agent SDK or watches a directory for output. Copilot, Cursor, and other agent SDKs are not integrated; use `--watch` mode for those. -- **Type-aware checks require --project.** Three checks (implicit any, unused exports, unresolved imports) need a `tsconfig.json`. Without `--project`, ts-morph parses files in isolation and these checks are skipped. -- **102 matchers, not infinite.** The parser skips lines it can't confidently map to a check. Use `--show-unparseable` to see what was missed, and `--llm-extract` or `--rubric-decompose` to handle the remainder. The semantic tier (`--semantic`) covers pattern-matching and consistency rules that deterministic matchers cannot. -- **Preference pairs are TypeScript-focused.** The 8 built-in prefer-pairs (const vs let, named vs default exports, etc.) use ts-morph AST queries. Adding pairs for other languages requires new counting functions. - -## Troubleshooting - -**`sh: ruleprobe: not found` after global install** -The npm bin directory may not be in `PATH`. Run `npm bin -g` to find it and add it to your shell profile, or use `npx ruleprobe` instead. - -**`Error: OPENAI_API_KEY not set`** -`--llm-extract` and `--rubric-decompose` require an OpenAI-compatible API key. Export it before running: `export OPENAI_API_KEY=sk-...`. The key is never written to disk or included in reports. - -**Tree-sitter checks skipped** -The WASM grammars load from installed tree-sitter grammar packages. If packages are missing (e.g., after a partial install) or the platform doesn't support WASM, tree-sitter checks silently fall back and other verifiers still run. Re-run `npm install` to restore them. - -**`ruleprobe verify` exits 2 with "path outside project root"** -A file or symlink in the output directory resolves outside the project root. Pass `--allow-symlinks` to follow symlinks across boundaries, or move the symlink targets inside the project. - -**Fewer rules extracted than expected** -Run `ruleprobe parse --show-unparseable` to see which lines were skipped and why. Add `--llm-extract` to attempt extraction on skipped lines. - -**Semantic analysis skipped / missing API key** -Verify your Anthropic API key is set via `--anthropic-key`, `ANTHROPIC_API_KEY` env var, or `.ruleprobe/config.json`. Deterministic analysis always runs regardless of semantic tier status. - -## What's New in v4.0.0 +- **TypeScript gets the deepest coverage.** ts-morph gives full AST analysis for TypeScript and JavaScript across all categories. Other languages get regex-based checks only. +- **Subjective rules stay subjective.** "Write clean code" has no deterministic check. `--rubric-decompose` uses an LLM to break subjective instructions into weighted concrete checks, tagged with `confidence: 'low'`. Requires `OPENAI_API_KEY`. +- **Not all rules map to ESLint.** Test file requirements, project config conventions, git workflow rules, and preference pairs don't have ESLint equivalents. These are reported as unmappable so you can enforce them through other tooling. +- **Monorepo support is limited.** Drift detection and lint-config scan from the repository root and use the first ESLint config found. Monorepos with per-package instruction files or configs need to specify paths explicitly. -v4.0.0 consolidates the three-repo architecture into a single repo and open-sources the semantic analysis engine under MIT. +## What's New in v4.5.0 -Key changes: -- **Single repo**: the semantic engine (40 files, ~3,600 lines) and Anthropic caller moved into `src/semantic/engine/` and `src/semantic/anthropic-caller.ts`. No separate API service or private repos. -- **Local analysis**: semantic analysis runs entirely on the user's machine. No HTTP server, no license keys. LLM calls go directly to Anthropic with the user's own `ANTHROPIC_API_KEY`. -- **CLI change**: `--license-key` removed, replaced by `ANTHROPIC_API_KEY` env var (same pattern as `--llm-extract` with `OPENAI_API_KEY`). New `--anthropic-key` flag for explicit key passing. -- **Open source**: the full ASPE engine (fingerprinting, vector similarity, qualifier resolution, LLM escalation) is now MIT-licensed and visible to contributors. -- **1,085+ tests** across 86+ test files (was 864 across 68 in v3.0.0, plus 221 engine tests migrated). - -Full release notes: [docs/release-v4.0.0.md](docs/release-v4.0.0.md) - -## What's New in v3.0.0 - -v3.0.0 adds the **semantic analysis tier** (ASPE), fixes 12 root-cause bugs found during E2E validation, and delivers a batch AST verifier that drops parse complexity from O(rules * files) to O(files). +v4.5.0 pivots RuleProbe from "verify adherence" to "translate instruction files into ESLint configs." The core value proposition is now: translate, detect drift, extract. Key changes: -- **Semantic client** (`src/semantic/`): single-pass tree-sitter extraction, HTTP client, audit logging, license/config resolution. No source code leaves the machine. -- **Batch AST verifier**: parse each file once across all AST rules. Critical for large repos (PostHog: 7,000+ files). -- **Tree-sitter WASM stability**: parser caching prevents function table exhaustion on large codebases. -- **12 bug fixes**: tree-sitter crashes, O(rules*files) performance, matcher wiring, format routing, enum comparison, header mismatches, JSON corruption. All root-cause resolutions. -- **Calibration data**: measured on excalidraw (626 files) and PostHog (7,160 files). Fast-path threshold 0.85 confirmed. Jaccard/cosine weights 0.4/0.6 confirmed. -- **864 tests** across 68 files (up from 572 across 52 in v2.0.0). - -Full release notes and migration guide: [docs/release-v3.0.0.md](docs/release-v3.0.0.md) - -## Benchmarks - -**Corpus analysis: 580 instruction files from 568 repos.** RuleProbe parsed real CLAUDE.md, AGENTS.md, .cursorrules, .windsurfrules, GEMINI.md, and copilot-instructions.md files scraped from public GitHub repos with 10+ stars, including Sentry (43k stars), PingCAP/TiDB (40k), Lerna (36k), Dragonfly (30k), Kubernetes/kops (17k), RabbitMQ (14k), Google APIs (14k), Redpanda (12k), Cloudflare, Grafana, Microsoft, and others. 309 rules extracted from 150 files that contained verifiable instructions. The other 430 files (74%) had zero extractable rules; many were single-line redirects (e.g. Dragonfly's .cursorrules: "READ AGENTS.md"; Umi's .cursorrules: "RULE.md"). - -The extraction rate is 3.8% (309 rules from 8,222 total instruction lines). That sounds low until you look at what instruction files actually contain. 96% of the lines are markdown headers, code examples, project descriptions, build commands, agent behavior directives, and contextual prose. The parser isn't failing on those; it's correctly identifying them as not-rules. Only 26 files (4.5%) had parse rates above 20%. - -Raw data: [scraped-instructions/per-file-results.json](scraped-instructions/per-file-results.json) (580 entries), [scraped-instructions/all-extracted.json](scraped-instructions/all-extracted.json) (309 rules), [scraped-instructions/analysis.json](scraped-instructions/analysis.json) (summary stats). - -**E2E verification: excalidraw.** RuleProbe ran the full deterministic + semantic pipeline against excalidraw (~95k stars). The parser found 9 verifiable rules across CLAUDE.md and copilot-instructions.md. Deterministic analysis scored 66.1% compliance. Semantic analysis (structural fingerprinting of 626 source files) produced 9 verdicts, all resolved via fast-path vector similarity with zero LLM calls and zero token cost: - -| Rule | Compliance | Method | -|------|-----------|--------| -| Prefer functional components | 0.976 | structural-fast-path | -| PascalCase type naming | 0.976 | structural-fast-path | -| Async try/catch usage | 0.983 | structural-fast-path | -| Contextual error logging | 0.979 | structural-fast-path | -| Yarn as package manager | 0.50 | no matching topic | -| TypeScript required | 0.50 | no matching topic | -| Optional chaining preference | 0.50 | no matching topic | -| camelCase variables | 0.50 | no matching topic | -| UPPER\_CASE constants | 0.50 | no matching topic | - -Rules matching established code pattern topics (component-structure, error-handling) scored 0.97+. Rules about tooling or naming that don't map to structural AST patterns got a neutral 0.50. Privacy test confirmed: all 626 file IDs are opaque sequential integers; no source code, file paths, or variable names in any payload. - -Full report: [docs/verification/e2e-verification-report.md](docs/verification/e2e-verification-report.md) - -## Further Reading - -- [docs/cli-reference.md](docs/cli-reference.md) - Complete CLI command reference -- [docs/api-reference.md](docs/api-reference.md) - Programmatic API with types -- [docs/matchers.md](docs/matchers.md) - All 102 matchers with example instructions -- [docs/release-v4.0.0.md](docs/release-v4.0.0.md) - v4.0.0 release notes (single-repo consolidation) -- [docs/release-v3.0.0.md](docs/release-v3.0.0.md) - v3.0.0 release notes and migration guide -- [docs/release-v2.0.0.md](docs/release-v2.0.0.md) - v2.0.0 release notes -- [docs/case-study-v0.1.0.md](docs/case-study-v0.1.0.md) - Agent comparison case study -- [docs/verification/e2e-verification-report.md](docs/verification/e2e-verification-report.md) - E2E verification evidence +- **New `lint-config` command**: translates an instruction file into a flat or legacy ESLint config. +- **New `drift` command**: compares an instruction file against an existing ESLint config and reports mismatches. +- **New `extract` command**: parses an ESLint config and emits a markdown rules section for instruction files. +- **Removed `compare` command**: agent comparison is no longer a primary use case. +- **Removed `tasks` and `task` commands**: task template listing and printing removed. +- **Removed `run` command**: agent invocation via the Claude Agent SDK removed. +- **Removed runner module**: `buildAgentConfig`, `invokeAgent`, `watchForCompletion`, `countCodeFiles` are no longer exported. +- **Deprecated `verify` command**: still works, but the primary workflow is now lint-config, drift, and extract. +- **Matcher audit**: 34 ESLint-mappable matchers remain. 67 unmappable matchers (test file requirements, project config, git workflow, preference pairs, Python/Go, tree-sitter, barrel files, directory naming) removed. The remaining matchers all produce valid ESLint rule entries. +- **Category cleanup**: `test-requirement`, `dependency`, `preference`, `file-structure`, `tooling`, `testing`, and `workflow` categories removed. The remaining 7 categories (`naming`, `forbidden-pattern`, `structure`, `import-pattern`, `error-handling`, `type-safety`, `code-style`) all map to ESLint rules. ## Contributing @@ -417,4 +299,4 @@ Issues and pull requests welcome at [github.com/moonrunnerkc/ruleprobe](https:// ## License -[MIT](LICENSE) +[MIT](LICENSE) \ No newline at end of file diff --git a/action.yml b/action.yml index 5f9c2e8..57d70a6 100644 --- a/action.yml +++ b/action.yml @@ -1,26 +1,48 @@ -name: "RuleProbe: Verify Instruction Adherence" +name: "RuleProbe: Detect Config Drift" description: > - Verify whether AI coding agent output follows project instruction file rules. - Deterministic checks, no LLM calls, no API keys beyond GITHUB_TOKEN. + Detect drift between instruction files (CLAUDE.md, AGENTS.md, .cursorrules) + and ESLint config on every PR. Posts results as a PR comment and optionally + opens a follow-up PR with the regenerated config. Deterministic checks, + no LLM calls, no API keys beyond GITHUB_TOKEN. branding: icon: check-circle color: blue inputs: + mode: + description: "Run mode: drift (default) or verify (legacy)" + required: false + default: "drift" instruction-file: description: "Path to instruction file (CLAUDE.md, AGENTS.md, .cursorrules, etc)" required: true + eslint-file: + description: "Path to ESLint config (auto-detected if not specified)" + required: false + regenerate-on-drift: + description: "Open a follow-up PR with regenerated config when drift is detected" + required: false + default: "false" + comment-on-pr: + description: "Post drift results as a PR comment" + required: false + default: "true" + fail-on-drift: + description: "Fail the action if drift is detected" + required: false + default: "false" + # Legacy verify-mode inputs output-dir: - description: "Directory containing code to verify" - required: true + description: "Directory containing code to verify (verify mode only)" + required: false default: "src" agent: - description: "Agent identifier for the report" + description: "Agent identifier for the report (verify mode only)" required: false default: "ci" model: - description: "Model identifier for the report" + description: "Model identifier for the report (verify mode only)" required: false default: "unknown" format: @@ -28,34 +50,41 @@ inputs: required: false default: "text" severity: - description: "Minimum severity to report: error, warning, or all" + description: "Minimum severity to report (verify mode only)" required: false default: "all" fail-on-violation: - description: "Fail the action if any rule violations are found" - required: false - default: "true" - post-comment: - description: "Post results as a PR comment" + description: "Fail the action if any violations are found (verify mode only)" required: false default: "true" reviewdog-format: - description: "Also output in reviewdog rdjson format for annotation integration" + description: "Also output in reviewdog rdjson format (verify mode only)" required: false default: "false" + post-comment: + description: "Post results as a PR comment (verify mode only)" + required: false + default: "true" outputs: + drift-count: + description: "Number of drift issues detected" + value: ${{ steps.drift.outputs.drift-count }} + has-drift: + description: "Whether drift was detected" + value: ${{ steps.drift.outputs.has-drift }} + # Legacy verify outputs score: - description: "Adherence score as a percentage" + description: "Adherence score as a percentage (verify mode only)" value: ${{ steps.verify.outputs.score }} passed: - description: "Number of rules that passed" + description: "Number of rules that passed (verify mode only)" value: ${{ steps.verify.outputs.passed }} failed: - description: "Number of rules that failed" + description: "Number of rules that failed (verify mode only)" value: ${{ steps.verify.outputs.failed }} total: - description: "Total number of rules checked" + description: "Total number of rules checked (verify mode only)" value: ${{ steps.verify.outputs.total }} runs: @@ -73,7 +102,29 @@ runs: npm ci --ignore-scripts npm run build + - name: Run drift detection + if: inputs.mode != 'verify' + id: drift + shell: bash + env: + INPUT_MODE: ${{ inputs.mode }} + INPUT_INSTRUCTION_FILE: ${{ inputs.instruction-file }} + INPUT_ESLINT_FILE: ${{ inputs.eslint-file }} + INPUT_REGENERATE_ON_DRIFT: ${{ inputs.regenerate-on-drift }} + INPUT_COMMENT_ON_PR: ${{ inputs.comment-on-pr }} + INPUT_FAIL_ON_DRIFT: ${{ inputs.fail-on-drift }} + GITHUB_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_EVENT_PATH: ${{ github.event_path }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_WORKSPACE: ${{ github.workspace }} + GITHUB_OUTPUT: ${{ github.output_file }} + RULEPROBE_BIN: ${{ github.action_path }}/dist/cli.js + run: node "${{ github.action_path }}/dist/action/main.js" + - name: Run ruleprobe verify + if: inputs.mode == 'verify' id: verify shell: bash env: @@ -88,11 +139,11 @@ runs: RULEPROBE_BIN: ${{ github.action_path }}/dist/cli.js run: bash "${{ github.action_path }}/action-scripts/run-verify.sh" - - name: Post PR comment - if: inputs.post-comment == 'true' && github.event_name == 'pull_request' + - name: Post PR comment (verify mode) + if: inputs.mode == 'verify' && inputs.post-comment == 'true' && github.event_name == 'pull_request' shell: bash env: GITHUB_TOKEN: ${{ github.token }} INPUT_FAIL_ON_VIOLATION: ${{ inputs.fail-on-violation }} RULEPROBE_REPORT_FILE: ${{ github.workspace }}/.ruleprobe-report.txt - run: bash "${{ github.action_path }}/action-scripts/post-comment.sh" + run: bash "${{ github.action_path }}/action-scripts/post-comment.sh" \ No newline at end of file diff --git a/docs/cli-reference.md b/docs/cli-reference.md index cc902ad..3232291 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -85,65 +85,49 @@ Checks for `CLAUDE.md`, `AGENTS.md`, `.cursorrules`, `.github/copilot-instructio --- -## `ruleprobe compare ` +## `ruleprobe lint-config ` -Run verification against multiple agent outputs and produce a comparison. +Parse an instruction file and emit an ESLint config. Flat config is the default; use `--format legacy` for `.eslintrc.json` output. ```bash -ruleprobe compare AGENTS.md ./claude-output ./copilot-output --agents claude,copilot --format markdown +ruleprobe lint-config CLAUDE.md +ruleprobe lint-config CLAUDE.md --format legacy --output .eslintrc.json +ruleprobe lint-config AGENTS.md --format flat --output eslint.config.js ``` | Option | Default | Description | |--------|---------|-------------| -| `--agents ` | none | Comma-separated labels for each directory | -| `--format ` | `markdown` | Report format: `text`, `json`, or `markdown` | -| `--output ` | stdout | Write report to file | -| `--allow-symlinks` | `false` | Follow symlinks outside the working directory | -| `--config ` | auto-discovered | Path to config file | +| `--format ` | `flat` | Output format: `flat` (ESLint flat config) or `legacy` (`.eslintrc.json`) | +| `--output ` | stdout | Write config to file | --- -## `ruleprobe tasks` +## `ruleprobe drift ` -List available task templates and their descriptions. +Detect drift between an instruction file and an ESLint config. Reports rules present in only one side, severity mismatches, and argument differences. ```bash -ruleprobe tasks +ruleprobe drift CLAUDE.md .eslintrc.json +ruleprobe drift CLAUDE.md eslint.config.js --format markdown +ruleprobe drift AGENTS.md .eslintrc.json --format json --output drift-report.json ``` ---- - -## `ruleprobe task ` - -Output the full task prompt for a given template. Three templates ship: `rest-endpoint`, `utility-module`, `react-component`. - -```bash -ruleprobe task rest-endpoint -``` +| Option | Default | Description | +|--------|---------|-------------| +| `--format ` | `text` | Output format: `text`, `json`, or `markdown` | +| `--output ` | stdout | Write report to file | --- -## `ruleprobe run ` +## `ruleprobe extract ` -Invoke an AI agent on a task template, then verify its output. Requires `@anthropic-ai/claude-agent-sdk` and `ANTHROPIC_API_KEY` for SDK mode. Alternatively, use `--watch` to point at a directory where an agent will write output. +Parse an ESLint config and emit a markdown rules section suitable for pasting into an instruction file. Stylistic rules are reported but excluded from the output by default. ```bash -# SDK mode: invoke Claude, verify, report -ruleprobe run CLAUDE.md --task rest-endpoint --agent claude-code --model sonnet --format text - -# Watch mode: wait for output in a directory, then verify -ruleprobe run CLAUDE.md --watch ./agent-output --timeout 300 --format json +ruleprobe extract .eslintrc.json +ruleprobe extract eslint.config.js --output rules-section.md ``` | Option | Default | Description | |--------|---------|-------------| -| `--task ` | `rest-endpoint` | Task template to give the agent | -| `--agent ` | `claude-code` | Agent identifier | -| `--model ` | `sonnet` | Model to use for the agent | -| `--format ` | `text` | Report format: `text`, `json`, `markdown`, or `rdjson` | -| `--output-dir ` | none | Directory to persist agent output | -| `--watch ` | none | Watch a directory for agent output instead of invoking | -| `--timeout ` | `300` | Watch mode timeout in seconds | -| `--allow-symlinks` | `false` | Follow symlinks outside the working directory | -| `--config ` | auto-discovered | Path to config file | -| `--project ` | none | tsconfig.json path for type-aware checks | +| `--output ` | stdout | Write output to file | diff --git a/docs/matchers.md b/docs/matchers.md index fed33ff..b95615a 100644 --- a/docs/matchers.md +++ b/docs/matchers.md @@ -1,8 +1,8 @@ # Built-in Matchers -RuleProbe ships 102 matchers across 14 categories. Each matcher maps a natural-language instruction to a deterministic check. +RuleProbe ships 34 ESLint-mappable matchers across 7 categories. Each matcher maps a natural-language instruction to a deterministic check. -The parser is conservative: if it can't confidently map an instruction to a check, it skips it and reports the line as unparseable. Use `--show-unparseable` to see skipped lines, and `--llm-extract` or `--rubric-decompose` to handle the remainder. +The parser is conservative: if it can't confidently map an instruction to a check, it skips it and reports the line as unparseable. Use `--show-unparseable` to see skipped lines, and `--llm-extract` or `--rubric-decompose` to handle the remainder. Rules that cannot map to ESLint (test file requirements, project config, git conventions) are reported as unmappable so you can enforce them through other tooling. ## Verifier Engines @@ -10,37 +10,25 @@ The parser is conservative: if it can't confidently map an instruction to a chec |----------|-------|-------| | AST | TypeScript / JavaScript | Full structural analysis via ts-morph | | AST (--project) | TypeScript / JavaScript | Requires `--project tsconfig.json` for cross-file type checking | -| Tree-sitter | TypeScript, JavaScript, Python, Go | Naming and function-length checks via WASM grammars | | Regex | Any text file | Line-level pattern matching | | Filesystem | Disk structure | File existence, naming, config presence | -| Preference | TypeScript / JavaScript | Compliance ratios for "prefer X over Y" patterns | -| Tooling | package.json / lockfiles / configs | Package manager, test framework, linter/formatter detection | -| Config-file | CI configs, git hooks, env tools | CI pipeline, pre-commit hooks, developer environment tool detection | -| Git-history | Git log | Commit message conventions, branch naming, signed commits | ## Category Summary | Category | Matchers | Verifier(s) | |----------|------:|-------------| -| naming | 9 | AST, Filesystem, Tree-sitter | +| naming | 5 | AST, Filesystem | | forbidden-pattern | 5 | AST, Regex | -| structure | 9 | AST, Filesystem | -| test-requirement | 5 | AST, Filesystem, Regex | -| import-pattern | 5 | AST, Regex | -| error-handling | 4 | AST | -| type-safety | 6 | AST, Regex | -| code-style | 12 | AST, Regex, Tree-sitter | -| dependency | 2 | Filesystem, Regex | -| preference | 8 | Preference | -| file-structure | 5 | Filesystem | -| tooling | 14 | Tooling, Config-file | -| testing | 3 | Filesystem, Regex | -| workflow | 15 | Config-file, Git-history | -| **Total** | **102** | | +| structure | 4 | AST, Filesystem | +| import-pattern | 4 | AST | +| error-handling | 2 | AST | +| type-safety | 5 | AST, Regex | +| code-style | 9 | AST, Regex | +| **Total** | **34** | | ## Matcher Table -### naming (9) +### naming (5) | Example instruction | What gets checked | Verifier | |-------------------|-------------------|----------| @@ -49,10 +37,6 @@ The parser is conservative: if it can't confidently map an instruction to a chec | "PascalCase for types" | Interface and type alias names | AST | | "kebab-case file names" | File names on disk | Filesystem | | "UPPER_CASE for constants" | Const declarations at module scope | AST | -| "kebab-case directories" | Directory names on disk | Filesystem | -| "Python snake_case functions" | Python function names via tree-sitter | Tree-sitter | -| "Python PascalCase classes" | Python class names via tree-sitter | Tree-sitter | -| "Go naming conventions" | Exported PascalCase, unexported camelCase | Tree-sitter | ### forbidden-pattern (5) @@ -62,52 +46,34 @@ The parser is conservative: if it can't confidently map an instruction to a chec | "no console.log" | Call expressions in AST | AST | | "no console.warn/error" | Extended console method calls | AST | | "no var" | Var declarations in all scopes | AST | -| "no TODO/FIXME comments" | Comment marker detection | Regex | +| "max line length 120" | Line character count | Regex | -### structure (9) +### structure (4) | Example instruction | What gets checked | Verifier | |-------------------|-------------------|----------| | "named exports only" | Export declarations | AST | | "max 300 lines per file" | File line count | Filesystem | -| "max line length 120" | Line character count | Regex | | "JSDoc on public functions" | JSDoc presence | AST | -| "strict mode" | tsconfig.json compilerOptions.strict | Filesystem | -| "no barrel files" | Index re-export detection | AST | -| "README must exist" | File existence on disk | Filesystem | -| "CHANGELOG must exist" | File existence on disk | Filesystem | -| "formatter config required" | .prettierrc / .eslintrc detection | Filesystem | - -### test-requirement (5) - -| Example instruction | What gets checked | Verifier | -|-------------------|-------------------|----------| -| "test file for every source file" | Matching test files exist | Filesystem | -| "test files named *.test.ts" | Test file naming convention | Filesystem | -| "no .only in tests" | Focused test detection | Regex | -| "no .skip in tests" | Skipped test detection | Regex | -| "no setTimeout in tests" | Timer usage in test files | AST | +| "no unused exports" | Exported symbols imported elsewhere | AST (--project) | -### import-pattern (5) +### import-pattern (4) | Example instruction | What gets checked | Verifier | |-------------------|-------------------|----------| | "no path aliases" | Import specifiers | AST | | "no deep relative imports" | Import depth | AST | | "no namespace imports" | Star import detection | AST | -| "ban specific packages" | Forbidden import sources | Regex | | "no wildcard re-exports" | `export *` detection | AST | -### error-handling (4) +### error-handling (2) | Example instruction | What gets checked | Verifier | |-------------------|-------------------|----------| | "no empty catch blocks" | Catch clause body inspection | AST | | "throw Error instances only" | Throw expression types | AST | -| "try/catch for async operations" | Async function error handling | AST | -| "contextual error logging" | Error context in log calls | AST | -### type-safety (6) +### type-safety (5) | Example instruction | What gets checked | Verifier | |-------------------|-------------------|----------| @@ -116,108 +82,27 @@ The parser is conservative: if it can't confidently map an instruction to a chec | "no non-null assertions" | `!` postfix operator | AST | | "no @ts-ignore / @ts-nocheck" | Directive comment detection | Regex | | "no implicit any" | Untyped parameters and variables | AST (--project) | -| "no unused exports" | Exported symbols imported elsewhere | AST (--project) | -### code-style (12) +### code-style (9) | Example instruction | What gets checked | Verifier | |-------------------|-------------------|----------| +| "no TODO/FIXME comments" | Comment marker detection | Regex | +| "consistent semicolons" | Missing or unexpected semicolons | Regex | +| "prefer const" | `let` that is never reassigned | AST | | "no nested ternary" | Ternary depth analysis | AST | | "no magic numbers" | Numeric literal usage | AST | | "no else after return" | Redundant else branches | AST | | "max function length" | Function body line count | AST | | "max parameters per function" | Parameter count | AST | -| "single/double quote style" | Quote consistency in imports | Regex | -| "prefer const" | `let` that is never reassigned | AST | -| "consistent semicolons" | Missing or unexpected semicolons | Regex | -| "concise conditionals" | Unnecessary braces around single-statement bodies | AST | -| "Python max function length" | Python function body line count | Tree-sitter | -| "Go max function length" | Go function body line count | Tree-sitter | -| "no unresolved imports" | Relative import resolution | AST (--project) | - -### dependency (2) - -| Example instruction | What gets checked | Verifier | -|-------------------|-------------------|----------| -| "pin dependency versions" | Exact version strings in package.json | Filesystem | -| "ban specific packages" | Package presence in dependencies | Regex | - -### preference (8) - -| Example instruction | What gets checked | Verifier | -|-------------------|-------------------|----------| -| "prefer const over let" | Ratio of const to let declarations | Preference | -| "prefer named over default exports" | Ratio of named to default exports | Preference | -| "prefer interface over type" | Ratio of interface to type alias declarations | Preference | -| "prefer async/await over .then()" | Ratio of await to .then() chains | Preference | -| "prefer arrow over function declarations" | Ratio of arrow to function expressions | Preference | -| "prefer template literals" | Ratio of template literals to concatenation | Preference | -| "prefer optional chaining" | Ratio of ?. to nested conditionals | Preference | -| "prefer functional components" | Ratio of functional to class components | Preference | - -### file-structure (5) - -| Example instruction | What gets checked | Verifier | -|-------------------|-------------------|----------| -| "tests/ directory exists" | Directory existence | Filesystem | -| "components/ directory exists" | Directory existence | Filesystem | -| ".env file exists" | File existence | Filesystem | -| "module index files" | Index file presence per module | Filesystem | -| "src/ directory exists" | Directory existence | Filesystem | - -### tooling (14) - -| Example instruction | What gets checked | Verifier | -|-------------------|-------------------|----------| -| "use pnpm" | pnpm lockfile present | Tooling | -| "use yarn" | yarn lockfile present | Tooling | -| "use bun" | bun lockfile present | Tooling | -| "use vitest" | vitest in devDependencies or config | Tooling | -| "use jest" | jest in devDependencies or config | Tooling | -| "use pytest" | pytest in requirements/pyproject | Tooling | -| "use eslint" | eslint config present | Tooling | -| "use prettier" | prettier config present | Tooling | -| "use biome" | biome config present | Tooling | -| "use flox" | flox environment detected | Config-file | -| "use nix" | nix flake or shell.nix detected | Config-file | -| "use devcontainer" | devcontainer.json detected | Config-file | -| "use mise" | mise config detected | Config-file | -| "use volta" | volta config detected | Config-file | - -### testing (3) - -| Example instruction | What gets checked | Verifier | -|-------------------|-------------------|----------| -| "colocate test files" | Test files next to source files | Filesystem | -| "use describe/it blocks" | Test structure patterns | Regex | -| "no console in tests" | Console calls in test files | Regex | - -### workflow (15) - -| Example instruction | What gets checked | Verifier | -|-------------------|-------------------|----------| -| "conventional commits" | Commit message format in git log | Git-history | -| "commit prefix required" | Commit message prefix pattern | Git-history | -| "branch naming convention" | Branch name pattern | Git-history | -| "signed commits" | GPG signature on commits | Git-history | -| "commit scope required" | Scope in conventional commit messages | Git-history | -| "CI runs lint" | Lint step in CI config | Config-file | -| "CI runs tests" | Test step in CI config | Config-file | -| "CI runs typecheck" | Typecheck step in CI config | Config-file | -| "CI config present" | CI config file existence | Config-file | -| "pre-commit runs tests" | Test hook in pre-commit config | Config-file | -| "pre-commit runs lint" | Lint hook in pre-commit config | Config-file | -| "husky configured" | Husky hooks directory present | Config-file | -| "lefthook configured" | Lefthook config present | Config-file | -| "npm test script" | test script in package.json | Config-file | -| "npm lint script" | lint script in package.json | Config-file | +| "use single quotes" | Quote consistency | Regex | ## Adding Matchers -Matchers are defined across 10 `src/parsers/rule-patterns*.ts` files. Each matcher has: +Matchers are defined across 4 `src/parsers/rule-patterns*.ts` files. Each matcher has: -- **id**: unique slug (e.g., `naming-camelCase-1`) -- **category**: one of the 14 categories above +- **id**: unique slug (e.g., `naming-camelcase-variables`) +- **category**: one of the 7 categories above - **keywords / pattern**: regex that matches the natural-language instruction - **verifier**: which engine runs the check - **patternType**: the specific check function (e.g., `camelCase`, `no-any`, `max-line-length`) diff --git a/docs/single-repo-guide.md b/docs/single-repo-guide.md index 13dfae4..e69de29 100644 --- a/docs/single-repo-guide.md +++ b/docs/single-repo-guide.md @@ -1,210 +0,0 @@ -## Errors Found (with evidence) - -| Claim in original | Actual (from data) | Source | -|---|---|---| -| "202 instruction files from 195 repos" | **580 files from 568 repos** | `per-file-results.json`: 580 entries, 568 unique `repo` values | -| "13% extraction rate / 87% unenforceable" | **3.8% extraction rate / 96.2% non-rule** | `analysis.json`: 309 extracted / 8222 total lines | -| "917 rules extracted" | **309 rules extracted** | `all-extracted.json`: 309 entries; `analysis.json` confirms | -| "7,072 total lines" | **8,222 total lines** | `analysis.json`: `totalInstructionLines: 8222` | -| "6,155 non-rule lines" | **7,913 unparseable** | `analysis.json`: `totalUnparseable: 7913` | -| "CLAUDE.md (127), AGENTS.md (73), .cursorrules (1), GEMINI.md (1)" | **AGENTS.md 149, CLAUDE.md 111, .cursorrules 102, .windsurfrules 95, GEMINI.md 89, copilot-instructions.md 34** | `per-file-results.json` filename counts | -| "35 files had zero extractions" | **430 of 580 files (74.1%)** | `per-file-results.json`: 430 entries with `extractedCount: 0` | -| CLAUDE.md parse rate 14.1%, AGENTS.md 11.2%, .cursorrules 40.7% | CLAUDE.md 2.5%, AGENTS.md 4.9%, .cursorrules 5.2%, copilot-instructions.md 5.9% | Per-type computation from `per-file-results.json` | -| "averaging 5.5 rules per file" | **2.1 rules per file** (among files with >0 rules) | 309 rules / 150 files with extractions | -| "44.9% had parse rates above 20%" | **4.1% (24 files)** | `per-file-results.json`: 13 at 20-29% + 11 at 30-49% = 24/580 | -| Names ClickHouse, Deno, PostHog, Expo | **Not in the dataset** | `per-file-results.json` has no matching repos for these orgs | -| "unjs/* repos all had CLAUDE.md" | **No unjs repos in dataset** | Zero matches for "unjs" in `per-file-results.json` | -| Excalidraw 16 rules, 81.2% | **9 rules, 66.1% deterministic / 71.3% semantic avg** | `e2e-verification-report.md` section 2: 9 verdicts listed | -| PostHog 15 rules, 80% | **4 rules, 25% deterministic** | `e2e-verification-report.md` section 3 | -| Codex 9 rules, Zed 7 rules, Cline 5 rules | **No data in any data file** | Not in `e2e-verification-report.md` or any other report file | -| "73% of rules were non-structural" | **Cannot verify** from available data files; the E2E report only covers excalidraw (9 rules) and PostHog (4 rules), total 13 rules, not 52 | -| "$0.06 total across 5 repos" | **$0.00 for excalidraw** (all fast-path); PostHog also 0 LLM calls; other 3 repos unverifiable | `e2e-verification-report.md` section 4 | - ---- - -## Corrected Post - -``` ---- -title: We Parsed 580 AI Instruction Files. 96% of the Content Can't Be Verified. -published: false -description: Analysis of real CLAUDE.md, AGENTS.md, .cursorrules, and other instruction files from 568 GitHub repos. Almost everything you write in instruction files is unenforceable. -tags: ai, programming, typescript, opensource -cover_image: https://your-cover-image-url.png ---- - -Every AI coding agent reads an instruction file. CLAUDE.md, AGENTS.md, .cursorrules, whatever your agent uses. You write rules in it. The agent says "Done." And you have no idea whether it followed any of them. - -We wanted to know what's actually inside these files. Not what people think they contain, but what a machine can extract and verify. So we scraped instruction files from 568 public GitHub repos with 10+ stars, ran them through a parser that identifies machine-verifiable rules, and counted what came out. - -The short version: 3.8% of the lines in a typical instruction file are verifiable coding rules. The other 96% is markdown headers, code examples, project descriptions, build commands, agent behavior directives, and contextual prose. - -## The dataset - -580 instruction files from 568 repos, including Sentry (43k stars), PingCAP/TiDB (40k), Lerna (36k), Dragonfly (30k), Kubernetes/kops (17k), javascript-obfuscator (16k), RabbitMQ (14k), Google APIs (14k), Redpanda (12k), Cloudflare (947/725), and hundreds of others. A mix of six file formats: AGENTS.md (149 files), CLAUDE.md (111), .cursorrules (102), .windsurfrules (95), GEMINI.md (89), and copilot-instructions.md (34). - -The parser reads each file and classifies every line: is this a rule that can be checked against code, or is it something else? "Something else" includes headers, blank lines, code blocks, explanatory prose, build instructions, and agent personality configuration. - -{% card %} -Corpus stats: 8,222 total instruction lines parsed. 309 rules extracted. 7,913 lines classified as non-rule content. -{% endcard %} - -## What instruction files actually contain - -The 96% that isn't rules breaks down into several categories. Some of it is necessary context (project structure explanations, build command documentation). Some of it is agent behavior configuration ("be succinct," "avoid providing explanations"). Some of it is just markdown formatting overhead. - -Here's what stood out: 430 of the 580 files (74%) had zero extractable rules. Of those, 67 were completely empty to the parser: zero extracted, zero unparseable. Many were single-line redirects. Dragonfly's .cursorrules (30k stars) says "READ AGENTS.md." Umi's .cursorrules (16k stars) contains the single word "RULE.md." Mautic's GEMINI.md says "Read and follow all instructions in ./AGENTS.md." - -At the other end, some files were almost entirely rules. Apache Skywalking-java's CLAUDE.md extracted 6 rules from 26 lines (23%). Cloudflare chanfana's AGENTS.md: 5 rules from 21 lines (24%). But those files tend to be short, focused lists of concrete instructions. - -The heavy files tell a different story. javascript-obfuscator's CLAUDE.md (16k stars): 197 lines, zero rules extracted. JunDamin/hwpapi's CLAUDE.md: 100 lines, zero rules. These files are documentation with no machine-verifiable instructions embedded. - -{% details Parse rate distribution across all 580 files %} - -| Parse Rate | Files | Percentage | -|-----------|------:|----------:| -| 0% (no rules) | 430 | 74.1% | -| 1-9% | 70 | 12.1% | -| 10-19% | 54 | 9.3% | -| 20-29% | 13 | 2.2% | -| 30-49% | 11 | 1.9% | -| >= 80% | 2 | 0.3% | - -Only 2 files (0.3%) had parse rates at or above 80%. Nearly three quarters had zero. - -{% enddetails %} - -## Types of content the parser correctly skips - -This is worth clarifying because "3.8% extraction rate" sounds like the parser is broken. It isn't. These are lines that genuinely aren't rules: - -Markdown structure (headers, horizontal rules, blank lines). Code examples showing how to use a function or run a command. Project descriptions explaining what the repo does. Build and deployment instructions. Links to external documentation. Agent behavior directives that have no code-level representation ("be concise," "ask before making changes"). Workflow instructions ("use this branch strategy," "run tests before pushing"). - -The parser isn't failing on these. It's correctly identifying them as not-rules. The denominator is every line in the file, not every line that looks like it could be a rule. - -## What a "verifiable rule" looks like - -The 309 rules that did get extracted map to concrete checks. Things like: - -- "Use camelCase for function names" (AST naming check) -- "No any types" (TypeScript type safety check) -- "Use named exports, not default exports" (import pattern check) -- "Prefer const over let" (preference ratio check) -- "Test files must exist for every source file" (filesystem check) -- "Use Yarn, not npm" (tooling check) - -Each rule gets a category, a verifier type (AST, filesystem, regex, tree-sitter, preference, tooling, config-file, or git-history), and a qualifier (always, prefer, when-possible, avoid-unless, try-to, never). - -{% details Rule extraction by category %} - -| Category | Rules Extracted | -|----------|------:| -| naming | 169 | -| structure | 44 | -| code-style | 29 | -| forbidden-pattern | 24 | -| type-safety | 20 | -| dependency | 12 | -| error-handling | 5 | -| import-pattern | 4 | -| test-requirement | 2 | - -Naming rules dominate: 55% of all extracted rules. This makes sense. "Use camelCase" and "use kebab-case filenames" are the most concrete, unambiguous instructions people write. - -{% enddetails %} - -{% details Rule extraction by instruction file type %} - -| Type | Files | Files with Rules | Rules Extracted | Total Lines | Rate | -|------|------:|------:|------:|------:|------:| -| AGENTS.md | 149 | 49 | 97 | 1,961 | 4.9% | -| CLAUDE.md | 111 | 20 | 38 | 1,501 | 2.5% | -| .cursorrules | 102 | 37 | 79 | 1,508 | 5.2% | -| .windsurfrules | 95 | 22 | 50 | 1,866 | 2.7% | -| GEMINI.md | 89 | 9 | 12 | 830 | 1.4% | -| copilot-instructions.md | 34 | 13 | 33 | 556 | 5.9% | - -copilot-instructions.md had the highest extraction rate (5.9%), likely because those files are typically shorter and more prescriptive. GEMINI.md files had the lowest (1.4%). - -{% enddetails %} - -## E2E verification: does excalidraw follow its own instruction files? - -We ran the full pipeline on excalidraw (~95k stars): parse the instruction files, then verify the actual codebase against the extracted rules. Excalidraw has both a CLAUDE.md and a copilot-instructions.md. - -The parser found 9 verifiable rules across both files. Deterministic analysis scored 66.1% compliance. Semantic analysis (structural fingerprinting of 626 source files) produced 9 verdicts, all resolved via fast-path vector similarity with zero LLM calls and zero cost: - -| Rule | Compliance | Method | -|------|-----------|--------| -| Prefer functional components | 0.976 | structural-fast-path | -| PascalCase type naming | 0.976 | structural-fast-path | -| Async try/catch usage | 0.983 | structural-fast-path | -| Contextual error logging | 0.979 | structural-fast-path | -| Yarn as package manager | 0.50 | no matching topic | -| TypeScript required | 0.50 | no matching topic | -| Optional chaining preference | 0.50 | no matching topic | -| camelCase variables | 0.50 | no matching topic | -| UPPER_CASE constants | 0.50 | no matching topic | - -Rules that match established code pattern topics (component-structure, error-handling) score 0.97+, meaning the codebase's structural fingerprint strongly matches the instruction. Rules about tooling or naming conventions that don't map to structural AST patterns get a neutral 0.50. - -The privacy test confirmed: 626 files scanned, all file IDs are opaque sequential integers, no source code strings, file paths, variable names, or comments appear in any payload sent to an LLM. In this case, no LLM was even called. - -## What this means for anyone writing instruction files - -If you're writing a CLAUDE.md or AGENTS.md right now, roughly 96% of what you type can't be mechanically verified. That doesn't mean it's useless. Agent behavior configuration, project context, workflow documentation all have value. But if you think you're writing enforceable rules, you're probably writing documentation. - -To write rules that can actually be checked: - -**Use imperative verbs with specific targets.** "Use camelCase for all function names" is verifiable. "Follow good naming conventions" isn't. - -**Specify the tool or pattern, not the principle.** "Prefer const over let" is a ratio check. "Write immutable code" is philosophy. - -**Include the file patterns your rules apply to.** "All .ts files must use named exports" scopes the check. "Use named exports" is vague. - -**Keep rules and documentation separate.** Rules are instructions. Documentation explains why. Mixing them dilutes both. - -{% cta https://github.com/moonrunnerkc/ruleprobe %} -RuleProbe on GitHub: parse your own instruction files and see what's actually verifiable -{% endcta %} - -## The tool - -RuleProbe is the parser and verifier behind this analysis. It reads 7 instruction file formats, extracts machine-verifiable rules using 102 built-in matchers across 14 categories, and checks agent output against each one. Deterministic by default, no API keys needed for the core pipeline. Optional semantic analysis for pattern-matching and consistency rules. - -```bash -npx ruleprobe parse CLAUDE.md --show-unparseable -npx ruleprobe verify CLAUDE.md ./src --format summary -``` - -The `--show-unparseable` flag shows you exactly which lines were skipped and why. That's often the most useful output: it tells you which of your "rules" aren't rules at all. - -{% embed https://github.com/moonrunnerkc/ruleprobe %} -``` - ---- - -## Evidence Index - -Every number in the corrected post maps to a data file: - -| Claim | Source file | How to verify | -|---|---|---| -| 580 files, 568 repos | `scraped-instructions/per-file-results.json` | `len(data)` = 580, `len(set(repo))` = 568 | -| 8,222 total lines | `scraped-instructions/analysis.json` | `totalInstructionLines: 8222` | -| 309 rules extracted | `scraped-instructions/all-extracted.json` | `len(data)` = 309 | -| 7,913 unparseable | `scraped-instructions/analysis.json` | `totalUnparseable: 7913` | -| 3.8% extraction rate | computed | 309 / 8222 = 0.0376 | -| 430 files zero extraction | `per-file-results.json` | count where `extractedCount == 0` | -| 67 files 0/0 | `per-file-results.json` | count where both counts are 0 | -| File type breakdown | `per-file-results.json` | count by filename pattern | -| Per-type rates | `per-file-results.json` | sum extracted/total per type | -| Category distribution | `all-extracted.json` | count by `category` field | -| Parse rate distribution | `per-file-results.json` | bucket by `extractionRate` field (integer %) | -| Named repos (Sentry 43k, TiDB 40k, etc.) | `per-file-results.json` | sort by `stars` field | -| Redirect examples (Dragonfly, Umi, Mautic) | `scraped-instructions/files/` | actual file content verified | -| 102 matchers, 14 categories | `docs/matchers.md` | category summary table sums to 102 | -| Excalidraw 9 rules, 66.1% | `docs/verification/e2e-verification-report.md` sect. 2 | verbatim from report | -| Excalidraw 9 semantic verdicts, 0 LLM calls | `e2e-verification-report.md` sect. 2 + 4 | verbatim | -| Excalidraw verdict scores (0.976, 0.983, etc.) | `e2e-verification-report.md` sect. 2 | verdict table | -| Privacy: 626 files, opaque IDs, 13 patterns passed | `e2e-verification-report.md` sect. 5 | verbatim | diff --git a/package-lock.json b/package-lock.json index 7eca599..a46cc36 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ruleprobe", - "version": "1.0.0", + "version": "4.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ruleprobe", - "version": "1.0.0", + "version": "4.0.0", "license": "MIT", "dependencies": { "chalk": "5.6.2", @@ -24,6 +24,7 @@ }, "devDependencies": { "@types/node": "22.19.17", + "tsx": "^4.21.0", "typescript": "5.9.3", "vitest": "2.1.9" }, @@ -320,6 +321,23 @@ "node": ">=12" } }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/netbsd-x64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", @@ -337,6 +355,23 @@ "node": ">=12" } }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/openbsd-x64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", @@ -354,6 +389,23 @@ "node": ">=12" } }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/sunos-x64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", @@ -1211,6 +1263,19 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/get-tsconfig": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", @@ -1455,6 +1520,16 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/rollup": { "version": "4.60.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.1.tgz", @@ -1730,6 +1805,459 @@ "code-block-writer": "^13.0.3" } }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/package.json b/package.json index e86d6e3..9375c4c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "ruleprobe", - "version": "4.0.0", - "description": "Verify whether AI coding agents follow the instruction files they're given", + "version": "4.5.0", + "description": "Translate instruction files into ESLint configs, detect drift, and extract rules", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -9,21 +9,25 @@ "ruleprobe": "dist/cli.js" }, "scripts": { - "build": "tsc && mkdir -p dist/runner/task-templates && cp src/runner/task-templates/*.md dist/runner/task-templates/", + "prebuild": "rm -rf dist", + "build": "tsc", "test": "vitest run", "test:watch": "vitest", "lint": "tsc --noEmit", - "audit": "npm audit --audit-level=moderate" + "audit": "npm audit --audit-level=moderate", + "collect": "tsx scripts/dataset/collect.ts" }, "keywords": [ "ai", "coding-agent", + "eslint", + "drift-detection", "instruction-adherence", - "verification", "CLAUDE.md", "AGENTS.md", "cursorrules", - "benchmark" + "lint-config", + "rule-extraction" ], "author": "Brad (Aftermath Technologies)", "license": "MIT", @@ -34,6 +38,14 @@ "engines": { "node": ">=18.0.0" }, + "files": [ + "dist", + "action.yml", + "action-scripts", + "README.md", + "LICENSE", + "CHANGELOG.md" + ], "dependencies": { "chalk": "5.6.2", "commander": "12.1.0", @@ -47,6 +59,7 @@ }, "devDependencies": { "@types/node": "22.19.17", + "tsx": "^4.21.0", "typescript": "5.9.3", "vitest": "2.1.9" } diff --git a/src/action/comment.ts b/src/action/comment.ts new file mode 100644 index 0000000..fc9b404 --- /dev/null +++ b/src/action/comment.ts @@ -0,0 +1,97 @@ +/** + * Format drift detection results as a GitHub PR comment. + * + * Produces markdown with a summary line, a table of drift items, + * and the dedup marker () used to find + * and update existing comments instead of posting duplicates. + */ + +import type { DriftItem, DriftResult } from '../drift/types.js'; + +/** Hidden marker used to find and update existing RuleProbe drift comments. */ +const DRIFT_MARKER = ''; + +/** Threshold for wrapping content in a collapsible details block. */ +const COLLAPSE_THRESHOLD = 10; + +/** + * Format a drift count as a summary line. + * + * @param count - Number of drift issues detected + * @returns A human-readable summary string + */ +export function formatDriftSummary(count: number): string { + if (count === 0) return 'No drift detected'; + if (count === 1) return '1 drift issue detected'; + return `${count} drift issues detected`; +} + +/** Format a single drift item as a table row. */ +function formatItemRow(item: DriftItem): string { + const mdSev = item.mdSeverity ?? '-'; + const eslintSev = item.eslintSeverity ?? '-'; + const mdOpts = item.mdOptions && item.mdOptions.length > 0 + ? JSON.stringify(item.mdOptions) + : '-'; + const eslintOpts = item.eslintOptions && item.eslintOptions.length > 0 + ? JSON.stringify(item.eslintOptions) + : '-'; + return `| ${item.kind} | \`${item.ruleName}\` | ${mdSev} | ${eslintSev} | ${mdOpts} | ${eslintOpts} |`; +} + +/** Format drift items as a markdown table. */ +function formatDriftTable(items: DriftItem[]): string { + const header = '| Kind | Rule | MD Severity | ESLint Severity | MD Options | ESLint Options |'; + const separator = '|------|------|-------------|-----------------|------------|----------------|'; + const rows = items.map(formatItemRow); + return [header, separator, ...rows].join('\n'); +} + +/** + * Format a drift result as a GitHub PR comment body. + * + * Uses the ruleprobe-drift marker for deduplication. Short drift + * lists are shown inline; longer ones are wrapped in a collapsible + * details block. + * + * @param result - The drift comparison result + * @returns A formatted markdown string ready to post as a PR comment + */ +export function formatDriftComment(result: DriftResult): string { + if (!result.hasDrift) { + return [ + DRIFT_MARKER, + '## RuleProbe: No drift detected', + '', + `No drift between \`${result.mdFile}\` and \`${result.eslintFile}\`.`, + ].join('\n'); + } + + const summary = formatDriftSummary(result.items.length); + const table = formatDriftTable(result.items); + + if (result.items.length >= COLLAPSE_THRESHOLD) { + return [ + DRIFT_MARKER, + `## RuleProbe: ${summary}`, + '', + `Between \`${result.mdFile}\` and \`${result.eslintFile}\`:`, + '', + '
', + `${summary}`, + '', + table, + '', + '
', + ].join('\n'); + } + + return [ + DRIFT_MARKER, + `## RuleProbe: ${summary}`, + '', + `Between \`${result.mdFile}\` and \`${result.eslintFile}\`:`, + '', + table, + ].join('\n'); +} \ No newline at end of file diff --git a/src/action/detect-changes.ts b/src/action/detect-changes.ts new file mode 100644 index 0000000..ef468df --- /dev/null +++ b/src/action/detect-changes.ts @@ -0,0 +1,85 @@ +/** + * Detect whether a PR's changed files are relevant to drift detection. + * + * Drift detection should run when instruction files (CLAUDE.md, + * AGENTS.md, .cursorrules) or ESLint config files are modified. + * If only unrelated files changed, the action skips to save time. + */ + +/** File names that are instruction files. */ +const INSTRUCTION_FILES = new Set([ + 'CLAUDE.md', + 'AGENTS.md', + '.cursorrules', +]); + +/** ESLint config filenames, ordered by preference (flat config first). */ +const ESLINT_CONFIG_FILES = [ + 'eslint.config.ts', + 'eslint.config.mjs', + 'eslint.config.js', + 'eslint.config.cjs', + 'eslint.config.json', + '.eslintrc.js', + '.eslintrc.cjs', + '.eslintrc.json', + '.eslintrc.yml', + '.eslintrc.yaml', + '.eslintrc', +]; + +/** Set of all ESLint config filenames for quick lookup. */ +const ESLINT_CONFIG_SET = new Set(ESLINT_CONFIG_FILES); + +/** + * Check whether any changed files are relevant to drift detection. + * + * Relevant files include instruction files (CLAUDE.md, AGENTS.md, + * .cursorrules) and ESLint config files. If the user specified + * explicit file paths, those are also considered relevant. + * + * @param changedFiles - List of file paths changed in the PR + * @param opts - Optional explicit file paths to consider relevant + * @param opts.instructionFile - Explicit instruction file path + * @param opts.eslintFile - Explicit ESLint config file path + * @returns true if drift detection should run + */ +export function shouldRunDrift( + changedFiles: string[], + opts?: { instructionFile?: string; eslintFile?: string }, +): boolean { + if (changedFiles.length === 0) return false; + + const explicitFiles = new Set(); + if (opts?.instructionFile) explicitFiles.add(opts.instructionFile); + if (opts?.eslintFile) explicitFiles.add(opts.eslintFile); + + for (const file of changedFiles) { + const basename = file.split('/').pop() ?? file; + + if (INSTRUCTION_FILES.has(basename)) return true; + if (ESLINT_CONFIG_SET.has(basename)) return true; + if (explicitFiles.has(file)) return true; + } + + return false; +} + +/** + * Auto-detect the ESLint config file from a list of repository files. + * + * Prefers flat config (eslint.config.*) over legacy (.eslintrc.*). + * Only detects config files at the repository root, not in subdirectories. + * + * @param filesInRepo - List of file paths in the repository root + * @returns The detected ESLint config filename, or undefined if none found + */ +export function autoDetectEslintFile(filesInRepo: string[]): string | undefined { + const rootFiles = new Set(filesInRepo.filter((f) => !f.includes('/'))); + + for (const configName of ESLINT_CONFIG_FILES) { + if (rootFiles.has(configName)) return configName; + } + + return undefined; +} \ No newline at end of file diff --git a/src/action/main.ts b/src/action/main.ts new file mode 100644 index 0000000..3efb8b0 --- /dev/null +++ b/src/action/main.ts @@ -0,0 +1,176 @@ +/** + * GitHub Action entry point. + * + * Reads action inputs from environment variables, constructs the + * GitHub context, and delegates to the runner. This file is the + * thin shell that the action.yml calls via `node dist/action/main.js`. + */ + +import { runAction } from './runner.js'; +import type { ActionInputs, ActionDeps, GitHubContext } from './types.js'; +import { execFile } from 'node:child_process'; +import { openSync, writeSync, closeSync } from 'node:fs'; +import * as fs from 'node:fs/promises'; + +/** Parse action inputs from environment variables. */ +function parseInputs(): ActionInputs { + const mode = process.env['INPUT_MODE'] ?? 'drift'; + return { + mode: mode === 'verify' ? 'verify' : 'drift', + instructionFile: process.env['INPUT_INSTRUCTION_FILE'] ?? '', + eslintFile: process.env['INPUT_ESLINT_FILE'] || undefined, + regenerateOnDrift: process.env['INPUT_REGENERATE_ON_DRIFT'] === 'true', + commentOnPr: process.env['INPUT_COMMENT_ON_PR'] !== 'false', + failOnDrift: process.env['INPUT_FAIL_ON_DRIFT'] === 'true', + outputDir: process.env['INPUT_OUTPUT_DIR'] || undefined, + agent: process.env['INPUT_AGENT'] || undefined, + model: process.env['INPUT_MODEL'] || undefined, + format: process.env['INPUT_FORMAT'] || undefined, + severity: process.env['INPUT_SEVERITY'] || undefined, + failOnViolation: process.env['INPUT_FAIL_ON_VIOLATION'] !== 'false', + reviewdogFormat: process.env['INPUT_REVIEWDOG_FORMAT'] === 'true', + }; +} + +/** Parse GitHub context from environment variables and event payload. */ +async function parseContext(): Promise { + const eventPath = process.env['GITHUB_EVENT_PATH'] ?? ''; + let prNumber: number | undefined; + + if (eventPath) { + try { + const eventData = JSON.parse(await fs.readFile(eventPath, 'utf-8')) as { + pull_request?: { number: number }; + }; + prNumber = eventData.pull_request?.number; + } catch { + // Not a PR event or invalid payload + } + } + + return { + repository: process.env['GITHUB_REPOSITORY'] ?? '', + apiUrl: process.env['GITHUB_API_URL'] ?? 'https://api.github.com', + prNumber, + token: process.env['GITHUB_TOKEN'] ?? '', + workspace: process.env['GITHUB_WORKSPACE'] ?? process.cwd(), + eventPath, + eventName: process.env['GITHUB_EVENT_NAME'] ?? '', + }; +} + +/** Create production deps that use real I/O. */ +function createDeps(): ActionDeps { + return { + async runCommand(command: string, args: string[]): Promise { + return new Promise((resolve) => { + const child = execFile(command, args); + if (child.stdout) child.stdout.pipe(process.stdout); + if (child.stderr) child.stderr.pipe(process.stderr); + child.on('close', (code) => resolve(code ?? 1)); + }); + }, + + async getChangedFiles(context: GitHubContext): Promise { + if (!context.prNumber) return []; + + const { stdout } = await execAsync('gh', [ + 'pr', 'diff', String(context.prNumber), + '--name-only', + '--repo', context.repository, + ]); + + return stdout.trim().split('\n').filter(Boolean); + }, + + async postComment(context: GitHubContext, prNumber: number, body: string, marker: string): Promise { + // Look for an existing comment with the marker + const { stdout: comments } = await execAsync('gh', [ + 'api', + `repos/${context.repository}/issues/${prNumber}/comments?per_page=100`, + '--jq', `.[] | select(.body | contains("${marker}")) | .id`, + ]); + + const existingId = comments.trim().split('\n')[0]?.trim(); + + const escapedBody = JSON.stringify(body); + + if (existingId) { + await execAsync('gh', [ + 'api', '-X', 'PATCH', + `repos/${context.repository}/issues/comments/${existingId}`, + '-f', `body=${escapedBody}`, + ]); + } else { + await execAsync('gh', [ + 'api', '-X', 'POST', + `repos/${context.repository}/issues/${prNumber}/comments`, + '-f', `body=${escapedBody}`, + ]); + } + }, + + async exec(command: string, args: string[]): Promise<{ stdout: string; exitCode: number }> { + return execAsync(command, args); + }, + + async writeFile(filePath: string, content: string): Promise { + await fs.writeFile(filePath, content, 'utf-8'); + }, + + async readFile(filePath: string): Promise { + return fs.readFile(filePath, 'utf-8'); + }, + + info(message: string): void { + console.log(message); + }, + + warn(message: string): void { + console.warn(`::warning::${message}`); + }, + + setOutput(name: string, value: string): void { + const outputFile = process.env['GITHUB_OUTPUT']; + if (outputFile) { + const fd = openSync(outputFile, 'a'); + writeSync(fd, `${name}=${value}\n`); + closeSync(fd); + } + }, + + setFailed(message: string): void { + console.error(`::error::${message}`); + process.exitCode = 1; + }, + }; +} + +/** Promisified execFile. */ +function execAsync(command: string, args: string[]): Promise<{ stdout: string; exitCode: number }> { + return new Promise((resolve) => { + execFile(command, args, { maxBuffer: 1024 * 1024 * 10 }, (err, stdout) => { + resolve({ stdout: stdout ?? '', exitCode: err ? 1 : 0 }); + }); + }); +} + +/** Main entry point. */ +async function main(): Promise { + const inputs = parseInputs(); + const context = await parseContext(); + const deps = createDeps(); + + if (!inputs.instructionFile) { + deps.setFailed('instruction-file input is required.'); + return; + } + + await runAction(inputs, context, deps); +} + +main().catch((err: unknown) => { + const message = err instanceof Error ? err.message : String(err); + console.error(`::error::${message}`); + process.exitCode = 2; +}); \ No newline at end of file diff --git a/src/action/regenerate.ts b/src/action/regenerate.ts new file mode 100644 index 0000000..057dc31 --- /dev/null +++ b/src/action/regenerate.ts @@ -0,0 +1,43 @@ +/** + * Regenerate an ESLint config from an instruction file and open + * a follow-up PR on a deterministic branch. + * + * The branch name is derived from the instruction file path so + * repeated runs against the same file always target the same branch. + */ + +import { createHash } from 'node:crypto'; + +/** Prefix for regeneration branches. */ +const BRANCH_PREFIX = 'ruleprobe/sync-'; + +/** + * Generate a deterministic branch name from the instruction file path. + * + * @param instructionFile - Path to the instruction file + * @returns A branch name like "ruleprobe/sync-abc1234" + */ +export function branchNameFor(instructionFile: string): string { + const hash = createHash('sha256').update(instructionFile).digest('hex').slice(0, 8); + return `${BRANCH_PREFIX}${hash}`; +} + +/** + * Generate a commit message for the regenerated config. + * + * @param instructionFile - Path to the instruction file + * @returns A conventional-commit message + */ +export function commitMessageFor(instructionFile: string): string { + return `chore: sync eslint config with ${instructionFile}`; +} + +/** + * Generate a PR title for the regeneration PR. + * + * @param instructionFile - Path to the instruction file + * @returns A descriptive PR title + */ +export function prTitleFor(instructionFile: string): string { + return `Sync ESLint config with ${instructionFile} (drift detected)`; +} \ No newline at end of file diff --git a/src/action/runner.ts b/src/action/runner.ts new file mode 100644 index 0000000..2f604f9 --- /dev/null +++ b/src/action/runner.ts @@ -0,0 +1,250 @@ +/** + * Action runner: orchestrates the GitHub Action logic. + * + * Reads inputs, decides whether to run drift detection based on + * changed files, executes the appropriate ruleprobe command, + * posts PR comments, and optionally opens a regeneration PR. + * All I/O is injected via ActionDeps for testability. + */ + +import { shouldRunDrift, autoDetectEslintFile } from './detect-changes.js'; +import { formatDriftComment, formatDriftSummary } from './comment.js'; +import { branchNameFor, commitMessageFor, prTitleFor } from './regenerate.js'; +import type { DriftResult } from '../drift/types.js'; +import type { ActionInputs, ActionDeps, GitHubContext } from './types.js'; + +/** Resolve the ruleprobe CLI command. Uses RULEPROBE_BIN if set, otherwise npx. */ +function ruleprobeBin(): string { + return process.env['RULEPROBE_BIN'] ?? 'npx'; +} + +/** Build args for running ruleprobe via npx or direct path. */ +function ruleprobeArgs(command: string, baseArgs: string[]): string[] { + const bin = process.env['RULEPROBE_BIN']; + if (bin) { + return [command, ...baseArgs]; + } + return ['ruleprobe', command, ...baseArgs]; +} + +/** + * Run the full action logic. + * + * @param inputs - Action inputs parsed from env vars + * @param context - GitHub context for the current run + * @param deps - Injectable dependencies for I/O operations + */ +export async function runAction( + inputs: ActionInputs, + context: GitHubContext, + deps: ActionDeps, +): Promise { + if (inputs.mode === 'verify') { + await runVerify(inputs, context, deps); + return; + } + + await runDrift(inputs, context, deps); +} + +/** Run drift detection mode. */ +async function runDrift( + inputs: ActionInputs, + context: GitHubContext, + deps: ActionDeps, +): Promise { + // Get changed files from the PR + const changedFiles = await deps.getChangedFiles(context); + + // Determine the eslint config file + const eslintFile = inputs.eslintFile + ?? autoDetectEslintFile(changedFiles) + ?? await findEslintFileInWorkspace(deps, context.workspace); + + if (!eslintFile) { + deps.setFailed('No ESLint config file found. Specify one with the eslint-file input.'); + return; + } + + // Check if any relevant files changed + if (!shouldRunDrift(changedFiles, { instructionFile: inputs.instructionFile, eslintFile })) { + deps.info('No relevant files changed, skipping drift detection.'); + deps.setOutput('drift-count', '0'); + deps.setOutput('has-drift', 'false'); + return; + } + + deps.info(`Running drift detection: ${inputs.instructionFile} vs ${eslintFile}`); + + // Run ruleprobe drift + const driftExitCode = await deps.runCommand(ruleprobeBin(), ruleprobeArgs('drift', [ + inputs.instructionFile, + eslintFile, + '--format', 'json', + '--output', `${context.workspace}/.ruleprobe-drift.json`, + ])); + + if (driftExitCode === 2) { + deps.setFailed('Drift detection failed with an execution error.'); + return; + } + + // Read the JSON result + const driftJson = await deps.readFile(`${context.workspace}/.ruleprobe-drift.json`); + let result: DriftResult; + try { + result = JSON.parse(driftJson) as DriftResult; + } catch { + deps.setFailed('Failed to parse drift detection output.'); + return; + } + + // Also run text format for the log + await deps.runCommand(ruleprobeBin(), ruleprobeArgs('drift', [ + inputs.instructionFile, + eslintFile, + '--format', 'text', + ])); + + // Set outputs + const driftCount = result.items.length; + deps.setOutput('drift-count', String(driftCount)); + deps.setOutput('has-drift', String(result.hasDrift)); + + deps.info(formatDriftSummary(driftCount)); + + // Post PR comment + if (inputs.commentOnPr && context.prNumber) { + const commentBody = formatDriftComment(result); + await deps.postComment(context, context.prNumber, commentBody, 'ruleprobe-drift'); + } + + // Fail if drift detected and failOnDrift is true + if (result.hasDrift && inputs.failOnDrift) { + deps.setFailed(`${driftCount} drift issue(s) detected.`); + return; + } + + // Regenerate if requested and drift detected + if (inputs.regenerateOnDrift && result.hasDrift && context.prNumber) { + await regenerateConfig(inputs, context, deps, eslintFile); + } +} + +/** Run verification mode (legacy). */ +async function runVerify( + inputs: ActionInputs, + context: GitHubContext, + deps: ActionDeps, +): Promise { + const outputDir = inputs.outputDir ?? 'src'; + + // Run the text report + const textExitCode = await deps.runCommand(ruleprobeBin(), ruleprobeArgs('verify', [ + inputs.instructionFile, + outputDir, + '--agent', inputs.agent ?? 'ci', + '--model', inputs.model ?? 'unknown', + '--severity', inputs.severity ?? 'all', + '--format', 'text', + '--output', `${context.workspace}/.ruleprobe-report.txt`, + ])); + + // Run the JSON report for programmatic consumption + await deps.runCommand(ruleprobeBin(), ruleprobeArgs('verify', [ + inputs.instructionFile, + outputDir, + '--agent', inputs.agent ?? 'ci', + '--model', inputs.model ?? 'unknown', + '--severity', inputs.severity ?? 'all', + '--format', 'json', + '--output', `${context.workspace}/.ruleprobe-report.json`, + ])); + + // Read JSON report for outputs + const jsonReport = await deps.readFile(`${context.workspace}/.ruleprobe-report.json`); + const report = JSON.parse(jsonReport) as { + summary: { adherenceScore: number; passed: number; failed: number; totalRules: number }; + }; + + deps.setOutput('score', String(Math.round(report.summary.adherenceScore))); + deps.setOutput('passed', String(report.summary.passed)); + deps.setOutput('failed', String(report.summary.failed)); + deps.setOutput('total', String(report.summary.totalRules)); + + if (textExitCode === 1 && inputs.failOnViolation) { + deps.setFailed(`RuleProbe found ${report.summary.failed} rule violation(s).`); + } +} + +/** Find an eslint config file in the workspace by scanning. */ +async function findEslintFileInWorkspace(deps: ActionDeps, workspace: string): Promise { + const candidates = [ + 'eslint.config.ts', + 'eslint.config.mjs', + 'eslint.config.js', + 'eslint.config.cjs', + 'eslint.config.json', + '.eslintrc.js', + '.eslintrc.cjs', + '.eslintrc.json', + '.eslintrc.yml', + '.eslintrc', + ]; + + for (const candidate of candidates) { + try { + await deps.readFile(`${workspace}/${candidate}`); + return candidate; + } catch { + continue; + } + } + + return undefined; +} + +/** Regenerate the eslint config and open a follow-up PR. */ +async function regenerateConfig( + inputs: ActionInputs, + context: GitHubContext, + deps: ActionDeps, + eslintFile: string, +): Promise { + const branch = branchNameFor(inputs.instructionFile); + const message = commitMessageFor(inputs.instructionFile); + const title = prTitleFor(inputs.instructionFile); + + deps.info(`Regenerating eslint config on branch ${branch}`); + + // Create or checkout the sync branch + await deps.exec('git', ['checkout', '-b', branch]); + await deps.exec('git', ['config', 'user.name', 'ruleprobe[bot]']); + await deps.exec('git', ['config', 'user.email', 'ruleprobe[bot]@users.noreply.github.com']); + + // Run ruleprobe lint-config to regenerate + const regenExitCode = await deps.runCommand(ruleprobeBin(), ruleprobeArgs('lint-config', [ + inputs.instructionFile, + '--format', 'flat', + '--output', `${context.workspace}/${eslintFile}`, + ])); + + if (regenExitCode !== 0) { + deps.warn('Failed to regenerate eslint config.'); + return; + } + + // Commit and push + await deps.exec('git', ['add', eslintFile]); + await deps.exec('git', ['commit', '-m', message]); + await deps.exec('git', ['push', 'origin', branch, '--force']); + + // Create PR + await deps.exec('gh', [ + 'pr', 'create', + '--title', title, + '--body', `${message}\n\nAuto-generated by RuleProbe drift detection.`, + '--head', branch, + '--base', 'main', + ]); +} \ No newline at end of file diff --git a/src/action/types.ts b/src/action/types.ts new file mode 100644 index 0000000..ece649f --- /dev/null +++ b/src/action/types.ts @@ -0,0 +1,109 @@ +/** + * Types for the GitHub Action entry point. + * + * The action supports two modes: drift (default) and verify (legacy). + * Drift mode compares instruction files against ESLint config and posts + * results as a PR comment. Verify mode runs the original adherence + * check pipeline. + */ + +import type { DriftResult } from '../drift/types.js'; + +export type { DriftResult }; + +/** Action run mode: drift detection (default) or verification (legacy). */ +export type ActionMode = 'drift' | 'verify'; + +/** Inputs for the GitHub Action, parsed from environment variables. */ +export interface ActionInputs { + /** Run mode: drift or verify. */ + mode: ActionMode; + /** Path to the instruction file (CLAUDE.md, AGENTS.md, .cursorrules). */ + instructionFile: string; + /** Path to the ESLint config file. Auto-detected if not specified. */ + eslintFile?: string; + /** Open a follow-up PR with regenerated config when drift is detected. */ + regenerateOnDrift: boolean; + /** Post drift results as a PR comment. */ + commentOnPr: boolean; + /** Fail the action if drift is detected. */ + failOnDrift: boolean; + /** Directory containing code to verify (verify mode only). */ + outputDir?: string; + /** Agent identifier for verify reports. */ + agent?: string; + /** Model identifier for verify reports. */ + model?: string; + /** Report format: text, json, or markdown. */ + format?: string; + /** Minimum severity to report (verify mode only). */ + severity?: string; + /** Fail the action on any violation (verify mode only). */ + failOnViolation?: boolean; + /** Output in reviewdog rdjson format (verify mode only). */ + reviewdogFormat?: boolean; +} + +/** Outputs from the drift detection run. */ +export interface DriftOutputs { + /** Number of drift issues detected. */ + driftCount: number; + /** Whether any drift was detected. */ + hasDrift: boolean; + /** The full drift result, if drift mode ran. */ + result?: DriftResult; +} + +/** Outputs from the verify run (legacy mode). */ +export interface VerifyOutputs { + /** Adherence score as a percentage. */ + score: number; + /** Number of rules that passed. */ + passed: number; + /** Number of rules that failed. */ + failed: number; + /** Total number of rules checked. */ + total: number; +} + +/** GitHub context available inside a GitHub Action. */ +export interface GitHubContext { + /** Repository owner/name, e.g. "moonrunnerkc/ruleprobe". */ + repository: string; + /** GitHub API base URL. */ + apiUrl: string; + /** PR number, if the event is a pull request. */ + prNumber?: number; + /** GitHub token for API calls. */ + token: string; + /** Path to the workspace directory. */ + workspace: string; + /** Path to the GitHub event JSON file. */ + eventPath: string; + /** The event name (e.g. "pull_request", "push"). */ + eventName: string; +} + +/** Functions that interact with the outside world, injectable for testing. */ +export interface ActionDeps { + /** Run a ruleprobe CLI command and return the exit code. */ + runCommand: (command: string, args: string[]) => Promise; + /** Get the list of changed files in the current PR. */ + getChangedFiles: (context: GitHubContext) => Promise; + /** Post or update a PR comment. */ + postComment: (context: GitHubContext, prNumber: number, body: string, marker: string) => Promise; + /** Run a shell command and capture stdout. */ + exec: (command: string, args: string[]) => Promise<{ stdout: string; exitCode: number }>; + /** Write a string to a file. */ + writeFile: (path: string, content: string) => Promise; + /** Read a file as a string. */ + readFile: (path: string) => Promise; + /** Log an info message. */ + info: (message: string) => void; + /** Log a warning message. */ + warn: (message: string) => void; + /** Set an action output variable. */ + setOutput: (name: string, value: string) => void; + /** Set the action exit code. */ + setFailed: (message: string) => void; +} \ No newline at end of file diff --git a/src/analyzers/project-analyzer.ts b/src/analyzers/project-analyzer.ts index 19d77c7..f04ee08 100644 --- a/src/analyzers/project-analyzer.ts +++ b/src/analyzers/project-analyzer.ts @@ -216,9 +216,8 @@ export function analyzeProject(projectDir: string): ProjectAnalysis { const allRules = files.flatMap((f) => f.ruleSet.rules); const totalRules = allRules.length; const allCategories: RuleCategory[] = [ - 'naming', 'forbidden-pattern', 'structure', 'test-requirement', + 'naming', 'forbidden-pattern', 'structure', 'import-pattern', 'error-handling', 'type-safety', 'code-style', - 'dependency', 'preference', 'file-structure', 'tooling', 'testing', ]; const byCategory = {} as Record; for (const cat of allCategories) { diff --git a/src/cli.ts b/src/cli.ts index 8947859..113af02 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,19 +10,19 @@ import { Command } from 'commander'; import { handleParse } from './commands/parse.js'; import { handleVerify } from './commands/verify.js'; -import { handleCompare } from './commands/compare.js'; -import { handleTasks, handleTask } from './commands/tasks.js'; -import { handleRun } from './commands/run.js'; import { handleAnalyze } from './commands/analyze.js'; +import { handleLintConfig } from './commands/lint-config.js'; +import { handleDrift } from './commands/drift.js'; +import { handleExtract } from './commands/extract.js'; const program = new Command(); program .name('ruleprobe') .description( - 'Verify whether AI coding agents follow the instruction files they\'re given', + 'Translate instruction files into ESLint configs, detect drift, and extract rules', ) - .version('1.0.0'); + .version('4.5.0'); // ── parse ── @@ -85,93 +85,6 @@ program }, ); -// ── tasks ── - -program - .command('tasks') - .description('List available task templates') - .action(() => { - handleTasks(); - }); - -// ── task ── - -program - .command('task') - .description('Output the full task prompt for a given template') - .argument('', 'task template identifier') - .action((templateId: string) => { - handleTask(templateId, exitWithError); - }); - -// ── compare ── - -program - .command('compare') - .description( - 'Run verification against multiple agent outputs and produce a comparison', - ) - .argument('', 'path to instruction file') - .argument('', 'two or more output directories to compare') - .option( - '--agents ', - 'comma-separated labels for each directory', - ) - .option('--format ', 'report format (text|json|markdown)', 'markdown') - .option('--output ', 'write report to file instead of stdout') - .option('--allow-symlinks', 'follow symlinks outside the working directory', false) - .option('--config ', 'path to ruleprobe config file') - .action( - async ( - file: string, - dirs: string[], - opts: { agents?: string; format: string; output?: string; allowSymlinks: boolean; config?: string }, - ) => { - await handleCompare(file, dirs, opts, exitWithError); - }, - ); - -// ── run ── - -program - .command('run') - .description( - 'Invoke an AI agent on a task template, then verify its output', - ) - .argument('', 'path to instruction file') - .option('--task ', 'task template to give the agent', 'rest-endpoint') - .option('--agent ', 'agent identifier', 'claude-code') - .option('--model ', 'model to use for the agent', 'sonnet') - .option('--format ', 'report format (text|json|markdown|rdjson)', 'text') - .option('--output-dir ', 'directory to persist agent output') - .option('--watch ', 'watch a directory for agent output instead of invoking') - .option('--timeout ', 'watch mode timeout in seconds', '300') - .option('--allow-symlinks', 'follow symlinks outside the working directory', false) - .option('--config ', 'path to ruleprobe config file') - .option('--project ', 'tsconfig.json path for type-aware checks') - .action( - async ( - file: string, - opts: { - task: string; - agent: string; - model: string; - format: string; - outputDir?: string; - watch?: string; - timeout: string; - allowSymlinks: boolean; - config?: string; - project?: string; - }, - ) => { - await handleRun(file, { - ...opts, - timeout: parseInt(opts.timeout, 10), - }, exitWithError); - }, - ); - // ── analyze ── program @@ -208,6 +121,48 @@ program }, ); +// ── lint-config ── + +program + .command('lint-config') + .description( + 'Parse an instruction file and emit an ESLint config', + ) + .argument('', 'path to instruction file') + .option('--format ', 'output format (flat|legacy)', 'flat') + .option('--output ', 'write config to file instead of stdout') + .action(async (file: string, opts: { format: string; output?: string }) => { + await handleLintConfig(file, opts, exitWithError); + }); + +// ── drift ── + +program + .command('drift') + .description( + 'Detect drift between a CLAUDE.md instruction file and an ESLint config', + ) + .argument('', 'path to instruction file') + .argument('', 'path to ESLint config file') + .option('--format ', 'output format (text|json|markdown)', 'text') + .option('--output ', 'write report to file instead of stdout') + .action(async (mdFile: string, eslintFile: string, opts: { format: string; output?: string }) => { + await handleDrift(mdFile, eslintFile, opts, exitWithError); + }); + +// ── extract ── + +program + .command('extract') + .description( + 'Extract a rules section from an ESLint config file', + ) + .argument('', 'path to ESLint config file') + .option('--output ', 'write output to file instead of stdout') + .action(async (eslintFile: string, opts: { output?: string }) => { + await handleExtract(eslintFile, opts, exitWithError); + }); + // ── Error handling ── /** diff --git a/src/commands/analyze-formatters.ts b/src/commands/analyze-formatters.ts index ae50cb9..5f03927 100644 --- a/src/commands/analyze-formatters.ts +++ b/src/commands/analyze-formatters.ts @@ -35,10 +35,9 @@ export function computeSummary(analysis: ProjectAnalysis): typeof analysis.summa const adherenceScore = totalRules > 0 ? (passed / totalRules) * 100 : 0; const allCategories: RuleCategory[] = [ - 'naming', 'forbidden-pattern', 'structure', 'test-requirement', + 'naming', 'forbidden-pattern', 'structure', 'import-pattern', 'error-handling', 'type-safety', 'code-style', - 'dependency', 'preference', 'file-structure', 'tooling', 'testing', - 'workflow', 'agent-behavior', + 'agent-behavior', ]; const byCategory = {} as Record; diff --git a/src/commands/compare.ts b/src/commands/compare.ts deleted file mode 100644 index af0c837..0000000 --- a/src/commands/compare.ts +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Handler for the "compare" CLI command. - * - * Runs verification against multiple agent outputs and produces - * a side-by-side comparison report. - */ - -import { existsSync, writeFileSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { parseInstructionFile } from '../parsers/index.js'; -import { verifyOutput } from '../verifier/index.js'; -import { generateReport } from '../index.js'; -import { formatComparisonMarkdown } from '../reporter/markdown.js'; -import { formatTextPlain } from '../reporter/text.js'; -import { validateOutputDir, currentTimestamp } from '../runner/index.js'; -import { resolveSafePath } from '../utils/safe-path.js'; -import { loadConfig, applyConfig } from '../config/index.js'; -import type { AgentRun, AdherenceReport } from '../types.js'; - -/** Options accepted by the compare command. */ -export interface CompareOpts { - agents?: string; - format: string; - output?: string; - allowSymlinks: boolean; - config?: string; -} - -/** - * Execute the compare command. - * - * @param file - Path to the instruction file - * @param dirs - Two or more output directories to compare - * @param opts - Command options - * @param exitWithError - Error handler that terminates the process - */ -export async function handleCompare( - file: string, - dirs: string[], - opts: CompareOpts, - exitWithError: (msg: string) => never, -): Promise { - let filePath: string; - try { - filePath = resolveSafePath(file, undefined, { allowExternal: opts.allowSymlinks }); - } catch (err) { - exitWithError((err as Error).message); - } - - if (!existsSync(filePath)) { - exitWithError(`Instruction file not found: ${filePath}`); - } - - if (dirs.length < 2) { - exitWithError('Compare requires at least 2 output directories.'); - } - - const validFormats = ['text', 'json', 'markdown']; - if (!validFormats.includes(opts.format)) { - exitWithError( - `Invalid format "${opts.format}". Use one of: ${validFormats.join(', ')}`, - ); - } - - const agentLabels = opts.agents - ? opts.agents.split(',').map((s) => s.trim()) - : dirs.map((_, i) => `agent-${i + 1}`); - - if (agentLabels.length !== dirs.length) { - exitWithError( - `Number of agent labels (${agentLabels.length}) does not match ` + - `number of directories (${dirs.length}).`, - ); - } - - const ruleSet = parseInstructionFile(filePath); - - let effectiveRuleSet = ruleSet; - try { - const config = await loadConfig(opts.config); - if (config) { - effectiveRuleSet = applyConfig(ruleSet, config); - } - } catch (err) { - exitWithError(`Config error: ${(err as Error).message}`); - } - - const reports: AdherenceReport[] = []; - - for (let i = 0; i < dirs.length; i++) { - let outDir: string; - try { - outDir = resolveSafePath(dirs[i]!, undefined, { allowExternal: opts.allowSymlinks }); - } catch (err) { - exitWithError( - `Directory ${i + 1} (${dirs[i]}): ${(err as Error).message}`, - ); - } - - try { - validateOutputDir(outDir); - } catch (err) { - exitWithError( - `Directory ${i + 1} (${dirs[i]}): ${(err as Error).message}`, - ); - } - - const results = await verifyOutput(effectiveRuleSet, outDir, { allowSymlinks: opts.allowSymlinks }); - const run: AgentRun = { - agent: agentLabels[i]!, - model: 'unknown', - taskTemplateId: 'manual', - outputDir: outDir, - timestamp: currentTimestamp(), - durationSeconds: null, - }; - - reports.push(generateReport(run, effectiveRuleSet, results)); - } - - let formatted: string; - if (opts.format === 'markdown') { - formatted = formatComparisonMarkdown(reports, agentLabels); - } else if (opts.format === 'json') { - formatted = JSON.stringify(reports, null, 2); - } else { - const parts: string[] = []; - for (const report of reports) { - parts.push(formatTextPlain(report)); - parts.push(''); - parts.push('---'); - parts.push(''); - } - formatted = parts.join('\n'); - } - - if (opts.output) { - writeFileSync(resolve(opts.output), formatted + '\n', 'utf-8'); - process.stdout.write(`Comparison report written to ${opts.output}\n`); - } else { - process.stdout.write(formatted + '\n'); - } -} diff --git a/src/commands/drift.ts b/src/commands/drift.ts new file mode 100644 index 0000000..098c2a3 --- /dev/null +++ b/src/commands/drift.ts @@ -0,0 +1,80 @@ +/** + * drift command handler. + * + * Compares a CLAUDE.md instruction file against an existing ESLint + * config file and reports drift between them. Reports mismatches in + * both directions: rules in md but not in eslint, rules in eslint + * but not in md, severity mismatches, and config-arg mismatches. + * + * Exit codes: 0 = no drift, 1 = drift found, 2 = execution error. + */ + +import { parseInstructionFile } from '../parsers/index.js'; +import { mapRuleSetToEslintConfig } from '../mapper/index.js'; +import { parseEslintConfigAsync } from '../drift/parseEslintConfig.js'; +import { compareConfigs } from '../drift/compareConfigs.js'; +import { formatDriftReport } from '../drift/formatDriftReport.js'; +import { resolveSafePath } from '../utils/safe-path.js'; +import type { DriftFormat } from '../drift/types.js'; +import { writeFileSync } from 'node:fs'; + +/** + * Handle the drift command. + * + * @param mdFile - Path to the instruction file (CLAUDE.md, AGENTS.md, etc.) + * @param eslintFile - Path to the ESLint config file + * @param opts - Command options + * @param opts.format - Output format: 'text', 'json', or 'markdown' + * @param opts.output - Optional path to write output to a file + * @param exitWithError - Callback to exit with an error message + */ +export async function handleDrift( + mdFile: string, + eslintFile: string, + opts: { format: string; output?: string }, + exitWithError: (message: string) => never, +): Promise { + const format: DriftFormat = opts.format === 'json' ? 'json' : opts.format === 'markdown' ? 'markdown' : 'text'; + + // Resolve and validate input paths + const safeInputPath = resolveSafePath(mdFile); + const safeEslintPath = resolveSafePath(eslintFile); + + // Parse the instruction file into rules + let ruleSet; + try { + ruleSet = parseInstructionFile(safeInputPath); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + exitWithError(`Failed to parse instruction file: ${message}`); + } + + // Map rules to ESLint config + const mdConfig = mapRuleSetToEslintConfig(ruleSet); + + // Parse the existing ESLint config (async to support JS/TS files) + let fileConfig; + try { + fileConfig = await parseEslintConfigAsync(safeEslintPath); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + exitWithError(`Failed to parse ESLint config: ${message}`); + } + + // Compare + const result = compareConfigs(mdConfig, fileConfig); + + // Format and output + const output = formatDriftReport(result, format); + + if (opts.output) { + writeFileSync(opts.output, output, 'utf-8'); + } else { + process.stdout.write(output + '\n'); + } + + // Exit code: 0 no drift, 1 drift, 2 error (handled by exitWithError) + if (result.hasDrift) { + process.exitCode = 1; + } +} \ No newline at end of file diff --git a/src/commands/extract.ts b/src/commands/extract.ts new file mode 100644 index 0000000..8ff44d9 --- /dev/null +++ b/src/commands/extract.ts @@ -0,0 +1,52 @@ +/** + * extract command handler. + * + * Parses an ESLint config file and emits a markdown rules section + * suitable for pasting into a CLAUDE.md or other instruction file. + * Subjective/stylistic rules without a prose equivalent are emitted + * as skipped comments. + */ + +import { parseEslintConfigAsync } from '../drift/parseEslintConfig.js'; +import { extractRules, formatRulesMarkdown } from '../extractor/index.js'; +import { resolveSafePath } from '../utils/safe-path.js'; +import { writeFileSync } from 'node:fs'; + +/** + * Handle the extract command. + * + * @param eslintFile - Path to the ESLint config file + * @param opts - Command options + * @param opts.output - Optional path to write output to a file + * @param exitWithError - Callback to exit with an error message + */ +export async function handleExtract( + eslintFile: string, + opts: { output?: string }, + exitWithError: (message: string) => never, +): Promise { + // Resolve and validate input path + const safeInputPath = resolveSafePath(eslintFile); + + // Parse the ESLint config (async to support JS/TS files) + let config; + try { + config = await parseEslintConfigAsync(safeInputPath); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + exitWithError(`Failed to parse ESLint config: ${message}`); + } + + // Extract rules + const result = extractRules(config); + + // Format as markdown + const output = formatRulesMarkdown(result); + + // Write to file or stdout + if (opts.output) { + writeFileSync(opts.output, output, 'utf-8'); + } else { + process.stdout.write(output + '\n'); + } +} \ No newline at end of file diff --git a/src/commands/lint-config.ts b/src/commands/lint-config.ts new file mode 100644 index 0000000..3b1b840 --- /dev/null +++ b/src/commands/lint-config.ts @@ -0,0 +1,57 @@ +/** + * lint-config command handler. + * + * Parses an instruction file and emits a corresponding ESLint + * configuration. Supports flat config (default) and legacy + * .eslintrc format via --format. + */ + +import { parseInstructionFile } from '../parsers/index.js'; +import { mapRuleSetToEslintConfig } from '../mapper/index.js'; +import { emitEslintConfig } from '../emitter/eslint.js'; +import type { EslintFormat } from '../mapper/types.js'; +import { resolveSafePath } from '../utils/safe-path.js'; +import { writeFileSync } from 'node:fs'; + +/** + * Handle the lint-config command. + * + * @param filePath - Path to the instruction file + * @param opts - Command options + * @param opts.format - Output format: 'flat' or 'legacy' + * @param opts.output - Optional path to write output to a file + * @param exitWithError - Callback to exit with an error message + */ +export async function handleLintConfig( + filePath: string, + opts: { format: string; output?: string }, + exitWithError: (message: string) => never, +): Promise { + const format: EslintFormat = opts.format === 'legacy' ? 'legacy' : 'flat'; + + // Resolve and validate input path + const safeInputPath = resolveSafePath(filePath); + + // Parse the instruction file + let ruleSet; + try { + ruleSet = parseInstructionFile(safeInputPath); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + exitWithError(`Failed to parse instruction file: ${message}`); + } + + // Map rules to ESLint config + const eslintConfig = mapRuleSetToEslintConfig(ruleSet); + + // Emit the config + const output = emitEslintConfig(eslintConfig, format); + + // Write to file or stdout + if (opts.output) { + const safeOutputPath = resolveSafePath(opts.output, undefined, { allowExternal: true }); + writeFileSync(safeOutputPath, output, 'utf-8'); + } else { + process.stdout.write(output + '\n'); + } +} \ No newline at end of file diff --git a/src/commands/run.ts b/src/commands/run.ts deleted file mode 100644 index 2a57242..0000000 --- a/src/commands/run.ts +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Handler for the "run" CLI command. - * - * Orchestrates agent invocation (SDK or watch mode), then runs - * verification on the agent's output. Combines the invoke and - * verify steps into a single command. - */ - -import { existsSync, mkdirSync, rmSync } from 'node:fs'; -import { parseInstructionFile } from '../parsers/index.js'; -import { verifyOutput } from '../verifier/index.js'; -import { generateReport } from '../index.js'; -import { formatReport } from '../reporter/index.js'; -import { currentTimestamp } from '../runner/index.js'; -import { resolveSafePath } from '../utils/safe-path.js'; -import { loadConfig, applyConfig } from '../config/index.js'; -import { findTaskTemplate, loadTaskPrompt } from '../runner/task-templates.js'; -import { buildAgentConfig } from '../runner/agent-configs.js'; -import { invokeAgent } from '../runner/agent-invoker.js'; -import { watchForCompletion, countCodeFiles } from '../runner/watch-mode.js'; -import type { AgentRun, ReportFormat, RuleSet } from '../types.js'; -import type { RunOptions } from '../runner/agent-configs.js'; - -/** - * Execute the run command. - * - * In SDK mode: loads task template, invokes agent via SDK, verifies - * output, and prints the report. - * - * In watch mode: watches a directory for agent output, then verifies. - * - * @param file - Path to the instruction file - * @param opts - Run command options - * @param exitWithError - Error handler that terminates the process - */ -export async function handleRun( - file: string, - opts: RunOptions, - exitWithError: (msg: string) => never, -): Promise { - let filePath: string; - try { - filePath = resolveSafePath(file); - } catch (err) { - exitWithError((err as Error).message); - } - - if (!existsSync(filePath)) { - exitWithError(`Instruction file not found: ${filePath}`); - } - - const validFormats: ReportFormat[] = ['text', 'json', 'markdown', 'rdjson']; - if (!validFormats.includes(opts.format as ReportFormat)) { - exitWithError( - `Invalid format "${opts.format}". Use one of: ${validFormats.join(', ')}`, - ); - } - - // Watch mode path - if (opts.watch) { - await handleWatchMode(filePath, opts, exitWithError); - return; - } - - // SDK invocation path - const template = findTaskTemplate(opts.task); - if (!template) { - exitWithError( - `Unknown task template: "${opts.task}". ` + - 'Run "ruleprobe tasks" to see available templates.', - ); - } - - const taskPrompt = loadTaskPrompt(opts.task); - if (!taskPrompt) { - exitWithError(`Task template file not found for: "${opts.task}".`); - } - - const agentConfig = buildAgentConfig(opts.agent, opts.model); - const ruleSet = parseInstructionFile(filePath); - - let effectiveRuleSet: RuleSet = ruleSet; - try { - const config = await loadConfig(opts.config); - if (config) { - effectiveRuleSet = applyConfig(ruleSet, config); - } - } catch (err) { - exitWithError(`Config error: ${(err as Error).message}`); - } - - process.stderr.write( - `Invoking ${agentConfig.agentId} (model: ${agentConfig.model})...\n`, - ); - - let invocationResult; - try { - invocationResult = await invokeAgent( - agentConfig, - taskPrompt, - opts.outputDir, - ); - } catch (err) { - exitWithError((err as Error).message); - } - - if (!invocationResult.success) { - exitWithError( - `Agent invocation failed: ${invocationResult.error ?? 'unknown error'}`, - ); - } - - const outDir = invocationResult.outputDir; - const codeFileCount = countCodeFiles(outDir); - if (codeFileCount === 0) { - exitWithError( - `Agent completed but wrote no code files to ${outDir}`, - ); - } - - process.stderr.write( - `Agent completed in ${invocationResult.durationSeconds?.toFixed(1)}s. ` + - `Found ${codeFileCount} code files. Running verification...\n`, - ); - - const results = await verifyOutput(effectiveRuleSet, outDir, { - allowSymlinks: opts.allowSymlinks, - projectPath: opts.project, - }); - - const run: AgentRun = { - agent: opts.agent, - model: opts.model, - taskTemplateId: opts.task, - outputDir: outDir, - timestamp: currentTimestamp(), - durationSeconds: invocationResult.durationSeconds, - }; - - const report = generateReport(run, effectiveRuleSet, results); - const formatted = formatReport(report, opts.format as ReportFormat); - process.stdout.write(formatted + '\n'); - - // Clean up temp dir if we created one (no --output-dir) - if (!opts.outputDir && existsSync(outDir)) { - try { - rmSync(outDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup - } - } - - const hasViolations = report.summary.failed > 0; - process.exit(hasViolations ? 1 : 0); -} - -/** - * Handle watch mode: wait for agent output, then verify. - */ -async function handleWatchMode( - filePath: string, - opts: RunOptions, - exitWithError: (msg: string) => never, -): Promise { - const watchDir = opts.watch as string; - - if (!existsSync(watchDir)) { - exitWithError(`Watch directory does not exist: ${watchDir}`); - } - - process.stderr.write( - `Watching ${watchDir} for .done marker (timeout: ${opts.timeout}s)...\n`, - ); - - const watchResult = await watchForCompletion({ - watchDir, - timeoutSeconds: opts.timeout, - }); - - if (watchResult.reason === 'timeout') { - process.stderr.write( - `Watch timed out after ${opts.timeout}s. Running verification on current state...\n`, - ); - } else { - process.stderr.write( - `Done marker detected after ${watchResult.durationSeconds.toFixed(1)}s.\n`, - ); - } - - const codeFileCount = countCodeFiles(watchDir); - if (codeFileCount === 0) { - exitWithError(`No code files found in ${watchDir}`); - } - - const ruleSet = parseInstructionFile(filePath); - - let effectiveRuleSet: RuleSet = ruleSet; - try { - const config = await loadConfig(opts.config); - if (config) { - effectiveRuleSet = applyConfig(ruleSet, config); - } - } catch (err) { - exitWithError(`Config error: ${(err as Error).message}`); - } - - const results = await verifyOutput(effectiveRuleSet, watchDir, { - allowSymlinks: opts.allowSymlinks, - projectPath: opts.project, - }); - - const run: AgentRun = { - agent: opts.agent, - model: opts.model, - taskTemplateId: opts.task, - outputDir: watchDir, - timestamp: currentTimestamp(), - durationSeconds: watchResult.durationSeconds, - }; - - const report = generateReport(run, effectiveRuleSet, results); - const formatted = formatReport(report, opts.format as ReportFormat); - process.stdout.write(formatted + '\n'); - - const hasViolations = report.summary.failed > 0; - process.exit(hasViolations ? 1 : 0); -} diff --git a/src/commands/tasks.ts b/src/commands/tasks.ts deleted file mode 100644 index b5fdd7d..0000000 --- a/src/commands/tasks.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Handlers for the "tasks" and "task" CLI commands. - * - * Lists available task templates and outputs full task prompts. - */ - -import { - listTaskTemplates, - findTaskTemplate, - loadTaskPrompt, -} from '../runner/task-templates.js'; - -/** - * Execute the "tasks" command: list all available task templates. - */ -export function handleTasks(): void { - const templates = listTaskTemplates(); - - if (templates.length === 0) { - process.stdout.write('No task templates available.\n'); - return; - } - - process.stdout.write('Available task templates:\n\n'); - for (const t of templates) { - process.stdout.write(` ${t.id}\n`); - process.stdout.write(` ${t.name}\n`); - process.stdout.write(` ${t.description}\n`); - process.stdout.write( - ` Exercises: ${t.exercises.join(', ')}\n`, - ); - process.stdout.write('\n'); - } -} - -/** - * Execute the "task" command: output the full prompt for a specific template. - * - * @param templateId - Task template identifier - * @param exitWithError - Error handler that terminates the process - */ -export function handleTask( - templateId: string, - exitWithError: (msg: string) => never, -): void { - const template = findTaskTemplate(templateId); - - if (!template) { - const available = listTaskTemplates() - .map((t) => t.id) - .join(', '); - exitWithError( - `Unknown task template: "${templateId}"\n` + - `Available templates: ${available}`, - ); - } - - const prompt = loadTaskPrompt(templateId); - - if (prompt === null) { - process.stdout.write( - `Task template "${templateId}" is registered but the prompt file ` + - 'is not yet available.\n', - ); - process.stdout.write(`\nTemplate info:\n`); - process.stdout.write(` Name: ${template.name}\n`); - process.stdout.write(` Description: ${template.description}\n`); - process.stdout.write( - ` Exercises: ${template.exercises.join(', ')}\n`, - ); - return; - } - - process.stdout.write(prompt); -} diff --git a/src/commands/verify.ts b/src/commands/verify.ts index a513ee3..faca31b 100644 --- a/src/commands/verify.ts +++ b/src/commands/verify.ts @@ -71,7 +71,7 @@ export async function handleVerify( exitWithError(`Instruction file not found: ${filePath}`); } - const validFormats: ReportFormat[] = ['text', 'json', 'markdown', 'rdjson']; + const validFormats: ReportFormat[] = ['text', 'json', 'markdown', 'rdjson', 'summary', 'detailed', 'ci']; if (!validFormats.includes(opts.format as ReportFormat)) { exitWithError( `Invalid format "${opts.format}". Use one of: ${validFormats.join(', ')}`, diff --git a/src/config/loader.ts b/src/config/loader.ts index c9cca07..92b4d94 100644 --- a/src/config/loader.ts +++ b/src/config/loader.ts @@ -9,6 +9,16 @@ import { existsSync, readFileSync } from 'node:fs'; import { resolve, join } from 'node:path'; import type { RuleProbeConfig, CustomRule, RuleOverride } from './types.js'; +import type { RuleCategory, VerifierType } from '../types.js'; + +/** Valid rule categories. */ +const VALID_CATEGORIES: Set = new Set([ + 'naming', 'forbidden-pattern', 'structure', 'import-pattern', + 'error-handling', 'type-safety', 'code-style', 'agent-behavior', +]); + +/** Valid verifier types. */ +const VALID_VERIFIERS: Set = new Set(['ast', 'regex', 'filesystem']); /** Config file names searched in order of priority. */ const CONFIG_FILE_NAMES = [ @@ -112,11 +122,17 @@ function validateCustomRule(rule: CustomRule, index: number): void { if (!rule.category || typeof rule.category !== 'string') { throw new Error(`${prefix}.category must be a valid RuleCategory`); } + if (typeof rule.category === 'string' && !VALID_CATEGORIES.has(rule.category)) { + throw new Error(`${prefix}.category must be one of: ${[...VALID_CATEGORIES].join(', ')}; got "${rule.category}"`); + } if (!rule.description || typeof rule.description !== 'string') { throw new Error(`${prefix}.description must be a non-empty string`); } if (!rule.verifier || typeof rule.verifier !== 'string') { - throw new Error(`${prefix}.verifier must be one of: ast, regex, filesystem`); + throw new Error(`${prefix}.verifier must be one of: ${[...VALID_VERIFIERS].join(', ')}`); + } + if (typeof rule.verifier === 'string' && !VALID_VERIFIERS.has(rule.verifier)) { + throw new Error(`${prefix}.verifier must be one of: ${[...VALID_VERIFIERS].join(', ')}; got "${rule.verifier}"`); } if (!rule.pattern || typeof rule.pattern !== 'object') { throw new Error(`${prefix}.pattern must be a VerificationPattern object`); diff --git a/src/dataset/cache.ts b/src/dataset/cache.ts new file mode 100644 index 0000000..c8d9413 --- /dev/null +++ b/src/dataset/cache.ts @@ -0,0 +1,54 @@ +/** + * File-based cache for GitHub API responses. + * + * Stores timestamped JSON entries so the Phase 0 collection script can + * skip redundant network calls on re-runs. + */ + +import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; + +const CACHE_DIR = join(process.cwd(), '.cache'); + +/** + * Build a cache key from a prefix and identifier. + * + * @param prefix - Cache category (e.g. "repo", "file", "search") + * @param identifier - Unique identifier within that category + * @param cacheDir - Override cache directory (for testing) + */ +export function cacheKey(prefix: string, identifier: string, cacheDir = CACHE_DIR): string { + const safe = identifier.replace(/[^a-zA-Z0-9._-]/g, '_'); + return join(cacheDir, `${prefix}-${safe}.json`); +} + +/** + * Read a cached entry if it exists and hasn't expired. + * + * @param key - File path returned by cacheKey() + * @param maxAgeMs - Maximum age in milliseconds before the entry is stale + * @returns The cached data, or null if missing/expired/malformed + */ +export function readCache(key: string, maxAgeMs: number): T | null { + if (!existsSync(key)) return null; + try { + const raw = readFileSync(key, 'utf-8'); + const entry: { timestamp: number; data: T } = JSON.parse(raw); + if (Date.now() - entry.timestamp > maxAgeMs) return null; + return entry.data; + } catch { + return null; + } +} + +/** + * Write an entry to the cache. + * + * @param key - File path returned by cacheKey() + * @param data - Data to cache + */ +export function writeCache(key: string, data: T): void { + mkdirSync(CACHE_DIR, { recursive: true }); + const entry = { timestamp: Date.now(), data }; + writeFileSync(key, JSON.stringify(entry)); +} \ No newline at end of file diff --git a/src/dataset/github-client.ts b/src/dataset/github-client.ts new file mode 100644 index 0000000..e8e5bbd --- /dev/null +++ b/src/dataset/github-client.ts @@ -0,0 +1,226 @@ +/** + * GitHub API client with exponential backoff and caching. + * + * Handles code search, repo metadata, and file content retrieval + * for the Phase 0 data collection pipeline. + */ + +import { cacheKey, readCache, writeCache } from './cache.js'; + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +const MAX_RETRIES = 5; +const BASE_DELAY_MS = 1000; +const MAX_DELAY_MS = 60_000; +const SEARCH_PAGE_DELAY_MS = 2500; +const REPO_FETCH_DELAY_MS = 200; +const FILE_FETCH_DELAY_MS = 200; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface SearchResultItem { + owner: string; + repo: string; + fullName: string; + path: string; + filename: string; +} + +export interface RepoMetadata { + stars: number; + description: string | null; + language: string | null; + defaultBranch: string; + archived: boolean; +} + +export interface QualifiedRepo { + fullName: string; + owner: string; + repo: string; + stars: number; + description: string | null; + language: string | null; + defaultBranch: string; + files: SearchResultItem[]; +} + +// --------------------------------------------------------------------------- +// HTTP with exponential backoff +// --------------------------------------------------------------------------- + +const GITHUB_TOKEN = process.env['GITHUB_TOKEN'] ?? ''; + +const headers: Record = { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'ruleprobe-collect', +}; +if (GITHUB_TOKEN) { + headers['Authorization'] = `Bearer ${GITHUB_TOKEN}`; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Fetch a URL from the GitHub API with exponential backoff on 403/429. + * + * Retries up to MAX_RETRIES times with doubling delay. If a rate limit + * is hit (x-ratelimit-remaining: 0), waits until the reset time instead. + */ +export async function githubFetch(url: string, retries = MAX_RETRIES): Promise { + let delay = BASE_DELAY_MS; + + for (let attempt = 0; attempt <= retries; attempt++) { + const response = await fetch(url, { headers }); + + if (response.ok) { + return response.json(); + } + + const remaining = response.headers.get('x-ratelimit-remaining'); + const resetTime = response.headers.get('x-ratelimit-reset'); + + if ((response.status === 403 || response.status === 429) && attempt < retries) { + if (remaining === '0' && resetTime) { + const resetDate = new Date(parseInt(resetTime, 10) * 1000); + const waitMs = Math.max(resetDate.getTime() - Date.now() + 1000, delay); + console.log(` Rate limited (remaining=0). Waiting ${Math.ceil(waitMs / 1000)}s until ${resetDate.toISOString()}`); + await sleep(waitMs); + continue; + } + + console.log(` ${response.status} on attempt ${attempt + 1}. Retrying in ${delay}ms...`); + await sleep(delay); + delay = Math.min(delay * 2, MAX_DELAY_MS); + continue; + } + + if (response.status === 422) { + console.log(` 422 for ${url}, skipping`); + return null; + } + + if (response.status === 404) { + return null; + } + + console.error(` HTTP ${response.status} for ${url}`); + const body = await response.text(); + console.error(` ${body.slice(0, 200)}`); + return null; + } + + console.error(` Max retries exceeded for ${url}`); + return null; +} + +// --------------------------------------------------------------------------- +// GitHub search + fetch +// --------------------------------------------------------------------------- + +/** Search GitHub code for a specific filename at repo root. */ +export async function searchForFile(filename: string): Promise { + const query = encodeURIComponent(`filename:${filename} path:/`); + const results: SearchResultItem[] = []; + + for (let page = 1; page <= 10; page++) { + const url = `https://api.github.com/search/code?q=${query}&per_page=100&page=${page}`; + console.log(` Searching page ${page} for ${filename}...`); + + const data = await githubFetch(url) as { items?: Array<{ + repository: { owner: { login: string }; name: string; full_name: string }; + path: string; + }> } | null; + + if (!data?.items?.length) break; + + for (const item of data.items) { + const itemPath = item.path; + const isRoot = !itemPath.includes('/'); + const isGithubDir = itemPath.startsWith('.github/'); + if (isRoot || isGithubDir) { + results.push({ + owner: item.repository.owner.login, + repo: item.repository.name, + fullName: item.repository.full_name, + path: itemPath, + filename, + }); + } + } + + console.log(` Found ${data.items.length} results (${results.length} at root so far)`); + if (data.items.length < 100) break; + await sleep(SEARCH_PAGE_DELAY_MS); + } + + return results; +} + +/** Fetch repo metadata (stars, language, etc.) with caching. */ +export async function getRepoMetadata(owner: string, repo: string): Promise { + const key = cacheKey('repo', `${owner}-${repo}`); + const cached = readCache(key, 24 * 60 * 60 * 1000); + if (cached) return cached; + + const url = `https://api.github.com/repos/${owner}/${repo}`; + const data = await githubFetch(url) as { + stargazers_count: number; + description: string | null; + language: string | null; + default_branch: string; + archived: boolean; + } | null; + + if (!data) return null; + + const metadata: RepoMetadata = { + stars: data.stargazers_count, + description: data.description, + language: data.language, + defaultBranch: data.default_branch, + archived: data.archived, + }; + + writeCache(key, metadata); + return metadata; +} + +/** Download raw file content from GitHub with caching. */ +export async function downloadFileContent( + owner: string, + repo: string, + path: string, + branch: string, +): Promise { + const key = cacheKey('file', `${owner}-${repo}-${path.replace(/\//g, '_')}`); + const cached = readCache(key, 7 * 24 * 60 * 60 * 1000); + if (cached) return cached; + + const url = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`; + const response = await fetch(url, { headers: { 'User-Agent': 'ruleprobe-collect' } }); + + if (!response.ok) { + console.log(` Failed to download ${owner}/${repo}/${path}: HTTP ${response.status}`); + return null; + } + + const content = await response.text(); + writeCache(key, content); + return content; +} + +/** Sleep between API calls to avoid rate limiting. */ +export function apiDelay(type: 'search' | 'repo' | 'file'): Promise { + const ms = type === 'search' ? SEARCH_PAGE_DELAY_MS + : type === 'repo' ? REPO_FETCH_DELAY_MS + : FILE_FETCH_DELAY_MS; + return sleep(ms); +} \ No newline at end of file diff --git a/src/dataset/summary.ts b/src/dataset/summary.ts new file mode 100644 index 0000000..055b0ee --- /dev/null +++ b/src/dataset/summary.ts @@ -0,0 +1,226 @@ +/** + * Statistics and summary generation for Phase 0 data collection. + * + * Pure functions for computing medians, percentiles, histograms, + * clustering unparseable patterns, and rendering the go/no-go SUMMARY.md. + */ + +import { parseInstructionContent, detectFileType } from '../parsers/index.js'; +import type { RuleSet, RuleCategory } from '../types.js'; + +export interface PerFileResult { + repoUrl: string; + repoStars: number; + filePath: string; + sourceType: string; + parseableRuleCount: number; + categoryBreakdown: Partial>; + unparseableLines: string[]; + parseError: string | null; +} + +/** Parse instruction file content and extract structured results. */ +export function parseFileContent(content: string, filePath: string): PerFileResult { + try { + const ruleSet: RuleSet = parseInstructionContent(content, filePath); + const sourceType = ruleSet.sourceType; + const parseableRuleCount = ruleSet.rules.length; + + const categoryBreakdown: Partial> = {}; + for (const rule of ruleSet.rules) { + categoryBreakdown[rule.category] = (categoryBreakdown[rule.category] ?? 0) + 1; + } + + const unparseableLines = ruleSet.unparseable.filter((line) => { + const trimmed = line.trim(); + return trimmed.length > 0 && !trimmed.startsWith('#') && !trimmed.startsWith('```'); + }); + + return { + repoUrl: '', + repoStars: 0, + filePath, + sourceType, + parseableRuleCount, + categoryBreakdown, + unparseableLines, + parseError: null, + }; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + return { + repoUrl: '', + repoStars: 0, + filePath, + sourceType: detectFileType(filePath), + parseableRuleCount: 0, + categoryBreakdown: {}, + unparseableLines: [], + parseError: message, + }; + } +} + +/** Compute the median of a numeric array. Returns 0 for empty input. */ +export function computeMedian(values: number[]): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + const left = sorted[mid - 1] ?? 0; + const right = sorted[mid] ?? 0; + return sorted.length % 2 !== 0 ? right : (left + right) / 2; +} + +/** Compute a percentile value from a numeric array. Returns 0 for empty input. */ +export function computePercentile(values: number[], percentile: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.ceil((percentile / 100) * sorted.length) - 1; + return sorted[Math.max(0, index)] ?? 0; +} + +/** Render a text histogram of rule counts per file. */ +export function buildHistogram(values: number[]): string { + if (values.length === 0) return 'No data'; + + const buckets = new Map([ + ['0', 0], + ['1-4', 0], + ['5-9', 0], + ['10-19', 0], + ['20-49', 0], + ['50+', 0], + ]); + + for (const v of values) { + if (v === 0) buckets.set('0', (buckets.get('0') ?? 0) + 1); + else if (v < 5) buckets.set('1-4', (buckets.get('1-4') ?? 0) + 1); + else if (v < 10) buckets.set('5-9', (buckets.get('5-9') ?? 0) + 1); + else if (v < 20) buckets.set('10-19', (buckets.get('10-19') ?? 0) + 1); + else if (v < 50) buckets.set('20-49', (buckets.get('20-49') ?? 0) + 1); + else buckets.set('50+', (buckets.get('50+') ?? 0) + 1); + } + + const maxCount = Math.max(...buckets.values()); + const lines: string[] = []; + for (const [label, count] of buckets) { + const barLength = maxCount > 0 ? Math.round((count / maxCount) * 40) : 0; + const bar = '#'.repeat(barLength); + lines.push(` ${label.padStart(5)} | ${bar} ${count}`); + } + return lines.join('\n'); +} + +/** Cluster unparseable lines by normalized form, sorted by frequency. */ +export function clusterUnparseable(lines: string[]): Array<{ pattern: string; count: number }> { + const normalized = new Map(); + + for (const line of lines) { + const key = line + .toLowerCase() + .replace(/\s+/g, ' ') + .replace(/^[-*#>\s]+/, '') + .trim(); + if (key.length < 3) continue; + + const existing = normalized.get(key); + if (existing) { + existing.count++; + } else { + normalized.set(key, { original: line, count: 1 }); + } + } + + return [...normalized.entries()] + .sort((a, b) => b[1].count - a[1].count) + .map(([key, val]) => ({ pattern: key.slice(0, 120), count: val.count })); +} + +/** Generate the Phase 0 SUMMARY.md content from per-file results. */ +export function generateSummary(results: PerFileResult[]): string { + const ruleCounts = results.map((r) => r.parseableRuleCount); + const median = computeMedian(ruleCounts); + const p75 = computePercentile(ruleCounts, 75); + const p90 = computePercentile(ruleCounts, 90); + const totalRules = ruleCounts.reduce((a, b) => a + b, 0); + const totalUnparseable = results.reduce((sum, r) => sum + r.unparseableLines.length, 0); + const filesWithRules = ruleCounts.filter((c) => c > 0).length; + const totalFiles = results.length; + + const categoryTotals: Record = {}; + for (const r of results) { + for (const [cat, count] of Object.entries(r.categoryBreakdown)) { + categoryTotals[cat] = (categoryTotals[cat] ?? 0) + count; + } + } + const topCategories = Object.entries(categoryTotals) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10); + + const allUnparseable = results.flatMap((r) => r.unparseableLines); + const topUnparseable = clusterUnparseable(allUnparseable).slice(0, 10); + + const parseErrors = results.filter((r) => r.parseError !== null).length; + + const goDecision = median >= 5 && p75 >= 10; + const verdict = goDecision ? 'GO' : 'NO-GO'; + + const sourceTypes: Record = {}; + for (const r of results) { + sourceTypes[r.sourceType] = (sourceTypes[r.sourceType] ?? 0) + 1; + } + + const lines: string[] = [ + '# Phase 0 Summary', + '', + `**Verdict: ${verdict}**`, + '', + `Median parseable rules per file: **${median}** (threshold: >= 5)`, + `75th percentile: **${p75}** (threshold: >= 10)`, + `90th percentile: ${p90}`, + '', + goDecision + ? 'The dataset contains enough mechanically-mappable rules to justify the translator.' + : 'The dataset does not contain enough mechanically-mappable rules. The numbers are published as-is to seed Phase 5 visibility.', + '', + '## Dataset Overview', + '', + '| Metric | Value |', + '|--------|-------|', + `| Repos analyzed | ${totalFiles} |`, + `| Files with >= 1 rule | ${filesWithRules} |`, + `| Total rules extracted | ${totalRules} |`, + `| Total unparseable lines | ${totalUnparseable} |`, + `| Files with parse errors | ${parseErrors} |`, + '', + '## Source Types', + '', + ...Object.entries(sourceTypes) + .sort((a, b) => b[1] - a[1]) + .map(([type, count]) => `- ${type}: ${count}`), + '', + '## Rule Count Distribution', + '', + buildHistogram(ruleCounts), + '', + '## Top 10 Rule Categories', + '', + '| Category | Count |', + '|----------|-------|', + ...topCategories.map(([cat, count]) => `| ${cat} | ${count} |`), + '', + '## Top 10 Unparseable Patterns', + '', + '| Pattern | Occurrences |', + '|---------|-------------|', + ...topUnparseable.map(({ pattern, count }) => `| ${pattern} | ${count} |`), + '', + '## Decision Criteria', + '', + '- **Median >= 5 rules per file**: measurement threshold for "enough signal"', + '- **Top quartile (P75) >= 10 rules per file**: measurement threshold for "enough depth"', + '- Both must pass for GO. Either failing triggers NO-GO.', + ]; + + return lines.join('\n'); +} \ No newline at end of file diff --git a/src/drift/compareConfigs.ts b/src/drift/compareConfigs.ts new file mode 100644 index 0000000..8ed8439 --- /dev/null +++ b/src/drift/compareConfigs.ts @@ -0,0 +1,122 @@ +/** + * Drift comparison between CLAUDE.md-derived ESLint config and an + * existing ESLint config file. + * + * Compares the two configs rule-by-rule and reports mismatches: + * md-only, eslint-only, severity-mismatch, and config-arg-mismatch. + * Unparseable rules from CLAUDE.md are excluded from comparison + * since they have no ESLint equivalent to compare against. + */ + +import type { EslintConfig } from '../mapper/types.js'; +import type { DriftItem, DriftResult, ParsedEslintConfig } from './types.js'; + +/** Deep-compare two values for equality using JSON serialization. */ +function deepEqual(a: unknown, b: unknown): boolean { + return JSON.stringify(a) === JSON.stringify(b); +} + +/** + * Compare a CLAUDE.md-derived ESLint config against an existing config file. + * + * Produces a DriftResult listing all mismatches between the two configs. + * Rules in the unmappable list are excluded, since they have no ESLint + * equivalent to compare against. + * + * @param mdConfig - The ESLint config derived from CLAUDE.md + * @param fileConfig - The parsed ESLint config from an existing file + * @returns A DriftResult with all detected mismatches + */ +export function compareConfigs( + mdConfig: EslintConfig, + fileConfig: ParsedEslintConfig, +): DriftResult { + const items: DriftItem[] = []; + + // Build lookup for eslint file rules by name + const fileRulesByName = new Map( + fileConfig.rules.map((r) => [r.ruleName, r]), + ); + + // Build lookup for md rules by name (skip unmappable) + const mdRulesByName = new Map( + mdConfig.rules.map((r) => [r.ruleName, r]), + ); + + // Collect unmappable rule names to exclude from comparison + const unmappableRuleIds = new Set( + mdConfig.unmappable.map((u) => u.sourceRuleId), + ); + + // Check md rules: present in md but missing/different in eslint + for (const mdRule of mdConfig.rules) { + const fileRule = fileRulesByName.get(mdRule.ruleName); + + if (!fileRule || fileRule.severity === 'off') { + // Rule exists in md but is absent or disabled in eslint + items.push({ + kind: 'md-only', + ruleName: mdRule.ruleName, + mdRuleId: mdRule.sourceRuleId, + mdDescription: mdRule.description, + mdSeverity: mdRule.severity, + eslintSeverity: fileRule?.severity, + message: fileRule + ? `${mdRule.ruleName} is disabled (off) in eslint config but expected ${mdRule.severity} by CLAUDE.md` + : `${mdRule.ruleName} is in CLAUDE.md but not in eslint config`, + }); + continue; + } + + // Rule exists in both; check severity + const severityDiffers = mdRule.severity !== fileRule.severity; + + // Check options + const mdOpts = mdRule.options ?? []; + const fileOpts = fileRule.options ?? []; + const optionsDiffer = !deepEqual(mdOpts, fileOpts); + + if (optionsDiffer) { + // Config-arg mismatch subsumes severity mismatch when both differ + items.push({ + kind: 'config-arg-mismatch', + ruleName: mdRule.ruleName, + mdSeverity: mdRule.severity, + eslintSeverity: fileRule.severity, + mdOptions: mdOpts, + eslintOptions: fileOpts, + message: severityDiffers + ? `${mdRule.ruleName}: CLAUDE.md says ${mdRule.severity} with ${JSON.stringify(mdOpts)}, eslint says ${fileRule.severity} with ${JSON.stringify(fileOpts)}` + : `${mdRule.ruleName}: CLAUDE.md says ${JSON.stringify(mdOpts)}, eslint says ${JSON.stringify(fileOpts)}`, + }); + } else if (severityDiffers) { + items.push({ + kind: 'severity-mismatch', + ruleName: mdRule.ruleName, + mdSeverity: mdRule.severity, + eslintSeverity: fileRule.severity, + message: `${mdRule.ruleName}: CLAUDE.md says ${mdRule.severity}, eslint says ${fileRule.severity}`, + }); + } + } + + // Check eslint rules: present in eslint but not in md + for (const fileRule of fileConfig.rules) { + if (fileRule.severity === 'off') continue; // disabled rules don't count as eslint-only + if (mdRulesByName.has(fileRule.ruleName)) continue; // already compared above + + items.push({ + kind: 'eslint-only', + ruleName: fileRule.ruleName, + eslintSeverity: fileRule.severity, + message: `${fileRule.ruleName} is in eslint config but not derived from CLAUDE.md`, + }); + } + + return { + items, + mdFile: mdConfig.sourceFile, + eslintFile: fileConfig.sourceFile, + hasDrift: items.length > 0, + }; +} \ No newline at end of file diff --git a/src/drift/formatDriftReport.ts b/src/drift/formatDriftReport.ts new file mode 100644 index 0000000..121f52d --- /dev/null +++ b/src/drift/formatDriftReport.ts @@ -0,0 +1,110 @@ +/** + * Drift report formatting. + * + * Produces text, JSON, or markdown output from a DriftResult. + * Exit code convention: 0 = no drift, 1 = drift, 2 = execution error. + */ + +import type { DriftResult, DriftFormat, DriftItem } from './types.js'; + +/** Count drift items by kind. */ +function countByKind(items: DriftItem[]): Record { + const counts: Record = { + 'md-only': 0, + 'eslint-only': 0, + 'severity-mismatch': 0, + 'config-arg-mismatch': 0, + }; + for (const item of items) { + counts[item.kind] = (counts[item.kind] ?? 0) + 1; + } + return counts; +} + +/** Format a single drift item as text. */ +function formatItemText(item: DriftItem): string { + const parts: string[] = [` [${item.kind}] ${item.ruleName}`]; + if (item.mdRuleId) parts.push(` rule: ${item.mdRuleId}`); + if (item.mdDescription) parts.push(` description: ${item.mdDescription}`); + if (item.mdSeverity) parts.push(` md severity: ${item.mdSeverity}`); + if (item.eslintSeverity) parts.push(` eslint severity: ${item.eslintSeverity}`); + if (item.mdOptions && item.mdOptions.length > 0) { + parts.push(` md options: ${JSON.stringify(item.mdOptions)}`); + } + if (item.eslintOptions && item.eslintOptions.length > 0) { + parts.push(` eslint options: ${JSON.stringify(item.eslintOptions)}`); + } + return parts.join('\n'); +} + +/** Format a drift report as plain text. */ +function formatText(result: DriftResult): string { + if (!result.hasDrift) { + return `No drift detected between ${result.mdFile} and ${result.eslintFile}`; + } + + const counts = countByKind(result.items); + const lines: string[] = [ + `Drift detected between ${result.mdFile} and ${result.eslintFile}`, + '', + 'Summary:', + ` ${counts['md-only']} md-only, ${counts['eslint-only']} eslint-only, ${counts['severity-mismatch']} severity-mismatch, ${counts['config-arg-mismatch']} config-arg-mismatch`, + '', + 'Details:', + ]; + + for (const item of result.items) { + lines.push(formatItemText(item)); + } + + return lines.join('\n'); +} + +/** Format a drift report as JSON. */ +function formatJson(result: DriftResult): string { + return JSON.stringify(result, null, 2); +} + +/** Format a drift report as markdown. */ +function formatMarkdown(result: DriftResult): string { + if (!result.hasDrift) { + return `## No drift detected\n\nNo drift between \`${result.mdFile}\` and \`${result.eslintFile}\`.`; + } + + const lines: string[] = [ + '## Drift detected', + '', + `Between \`${result.mdFile}\` and \`${result.eslintFile}\`:`, + '', + '| Kind | Rule | MD Severity | ESLint Severity | MD Options | ESLint Options |', + '|------|------|-------------|-----------------|------------|----------------|', + ]; + + for (const item of result.items) { + const mdOpts = item.mdOptions && item.mdOptions.length > 0 ? JSON.stringify(item.mdOptions) : '-'; + const eslintOpts = item.eslintOptions && item.eslintOptions.length > 0 ? JSON.stringify(item.eslintOptions) : '-'; + lines.push( + `| ${item.kind} | \`${item.ruleName}\` | ${item.mdSeverity ?? '-'} | ${item.eslintSeverity ?? '-'} | ${mdOpts} | ${eslintOpts} |`, + ); + } + + return lines.join('\n'); +} + +/** + * Format a DriftResult as text, JSON, or markdown. + * + * @param result - The drift comparison result + * @param format - Output format: 'text' (default), 'json', or 'markdown' + * @returns Formatted string ready for output + */ +export function formatDriftReport(result: DriftResult, format: DriftFormat = 'text'): string { + switch (format) { + case 'json': + return formatJson(result); + case 'markdown': + return formatMarkdown(result); + default: + return formatText(result); + } +} \ No newline at end of file diff --git a/src/drift/parseEslintConfig.ts b/src/drift/parseEslintConfig.ts new file mode 100644 index 0000000..1257fcf --- /dev/null +++ b/src/drift/parseEslintConfig.ts @@ -0,0 +1,194 @@ +/** + * ESLint config file parser. + * + * Reads ESLint config files and extracts rule entries with severity + * and options. Supports: + * - .eslintrc.json and eslint.config.json (synchronous JSON parsing) + * - .eslintrc.js, .eslintrc.cjs, .eslintrc.mjs (async dynamic import) + * - eslint.config.js, .mjs, .cjs (async dynamic import) + * - eslint.config.ts (async dynamic import) + * + * Uses direct JSON parsing for JSON configs rather than ESLint's + * loadConfigFromFile because loadConfigFromFile requires all referenced + * plugins to be installed, which is brittle in CI/CD pipelines. + */ + +import { readFileSync } from 'node:fs'; +import { extname, resolve } from 'node:path'; +import type { ParsedEslintConfig, ParsedEslintRule } from './types.js'; + +/** Normalize a severity value to "error" | "warn" | "off". */ +function normalizeSeverity(severity: unknown): 'error' | 'warn' | 'off' { + if (severity === 'error' || severity === 2) return 'error'; + if (severity === 'warn' || severity === 1) return 'warn'; + if (severity === 'off' || severity === 0) return 'off'; + return 'off'; +} + +/** + * Parse a rule entry from the config format. + * + * ESLint rules can be: + * - A severity string or number: "error", "warn", "off", 0, 1, 2 + * - An array where the first element is severity: ["error", { ...options }] + */ +function parseRuleEntry(ruleName: string, value: unknown): ParsedEslintRule { + if (typeof value === 'string' || typeof value === 'number') { + return { + ruleName, + severity: normalizeSeverity(value), + options: [], + }; + } + + if (Array.isArray(value) && value.length >= 1) { + const severity = normalizeSeverity(value[0]); + const options = value.slice(1); + return { ruleName, severity, options }; + } + + // Fallback: treat as off if unparseable + return { ruleName, severity: 'off', options: [] }; +} + +/** + * Extract rules from a legacy .eslintrc config object. + * + * Legacy configs have a top-level "rules" key with rule entries. + */ +function extractLegacyRules(configObj: Record): ParsedEslintRule[] { + const rulesObj = configObj['rules']; + if (!rulesObj || typeof rulesObj !== 'object' || Array.isArray(rulesObj)) { + return []; + } + + const rules: ParsedEslintRule[] = []; + for (const [ruleName, ruleValue] of Object.entries(rulesObj as Record)) { + rules.push(parseRuleEntry(ruleName, ruleValue)); + } + return rules; +} + +/** + * Extract rules from a flat config array. + * + * Flat config is an array of config objects, each of which may + * have a "rules" key. + */ +function extractFlatConfigRules(configArray: unknown[]): ParsedEslintRule[] { + const rules: ParsedEslintRule[] = []; + for (const configObj of configArray) { + if (configObj && typeof configObj === 'object' && !Array.isArray(configObj)) { + const obj = configObj as Record; + if (obj['rules'] && typeof obj['rules'] === 'object' && !Array.isArray(obj['rules'])) { + for (const [ruleName, ruleValue] of Object.entries(obj['rules'] as Record)) { + rules.push(parseRuleEntry(ruleName, ruleValue)); + } + } + } + } + return rules; +} + +/** File extensions that require dynamic import (JS-like configs). */ +const JS_EXTENSIONS = new Set(['.js', '.cjs', '.mjs', '.ts']); + +/** + * Parse an ESLint config file synchronously. + * + * Only supports JSON config files. For JS/TS config files, + * use parseEslintConfigAsync instead. + * + * @param filePath - Absolute or relative path to the ESLint config file + * @returns A ParsedEslintConfig with all rules found in the file + * @throws If the file is a JS/TS config or cannot be read/parsed + */ +export function parseEslintConfig(filePath: string): ParsedEslintConfig { + const ext = extname(filePath).toLowerCase(); + + if (JS_EXTENSIONS.has(ext)) { + throw new Error( + `JS/TS ESLint config files (${filePath}) require runtime module resolution. ` + + `Use the async parseEslintConfigAsync function instead.`, + ); + } + + // JSON config files + const content = readFileSync(filePath, 'utf-8'); + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`Failed to parse ESLint config at ${filePath}: ${message}`); + } + + // Determine config format: array = flat config, object = legacy + if (Array.isArray(parsed)) { + return { + rules: extractFlatConfigRules(parsed), + sourceFile: filePath, + }; + } + + if (parsed && typeof parsed === 'object') { + return { + rules: extractLegacyRules(parsed as Record), + sourceFile: filePath, + }; + } + + throw new Error(`Unexpected ESLint config format at ${filePath}: expected object or array`); +} + +/** + * Parse an ESLint config file asynchronously. + * + * Supports all formats including JS/TS config files that require + * dynamic import. JSON configs are parsed synchronously as a fast path. + * + * @param filePath - Absolute or relative path to the ESLint config file + * @returns A ParsedEslintConfig with all rules found in the file + * @throws If the file cannot be read, parsed, or imported + */ +export async function parseEslintConfigAsync(filePath: string): Promise { + const ext = extname(filePath).toLowerCase(); + + // JSON configs can be parsed synchronously + if (!JS_EXTENSIONS.has(ext)) { + return parseEslintConfig(filePath); + } + + // JS/TS configs require dynamic import + const absolutePath = resolve(filePath); + const fileUrl = new URL(`file://${absolutePath}`).href; + + let mod: unknown; + try { + mod = await import(/* @vite-ignore */ fileUrl); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + throw new Error( + `Failed to import ESLint config at ${filePath}: ${message}. ` + + `Ensure all plugin dependencies are installed, or convert to .eslintrc.json for reliable parsing.`, + ); + } + + const config = (mod as Record)['default'] ?? mod; + + if (Array.isArray(config)) { + return { + rules: extractFlatConfigRules(config), + sourceFile: filePath, + }; + } + + if (config && typeof config === 'object') { + return { + rules: extractLegacyRules(config as Record), + sourceFile: filePath, + }; + } + + throw new Error(`Unexpected ESLint config format in ${filePath}: expected object or array`); +} \ No newline at end of file diff --git a/src/drift/types.ts b/src/drift/types.ts new file mode 100644 index 0000000..0bf46ba --- /dev/null +++ b/src/drift/types.ts @@ -0,0 +1,70 @@ +/** + * Types for the drift detection module. + * + * Drift detection compares the ESLint config derived from a CLAUDE.md + * instruction file against an existing ESLint config file, reporting + * mismatches in both directions. + */ + +import type { EslintSeverity } from '../mapper/types.js'; + +/** A normalized ESLint rule entry parsed from a config file. */ +export interface ParsedEslintRule { + /** The rule name, e.g. "no-console" or "@typescript-eslint/no-explicit-any". */ + ruleName: string; + /** Severity level as found in the config file. "off" means the rule is disabled. */ + severity: 'error' | 'warn' | 'off'; + /** Rule-specific options (everything after the severity entry). */ + options: unknown[]; +} + +/** A parsed ESLint config file, ready for comparison. */ +export interface ParsedEslintConfig { + /** All rule entries found in the config. */ + rules: ParsedEslintRule[]; + /** Path to the source config file. */ + sourceFile: string; +} + +/** The kind of drift between two configs. */ +export type DriftKind = + | 'md-only' + | 'eslint-only' + | 'severity-mismatch' + | 'config-arg-mismatch'; + +/** A single drift item describing a mismatch. */ +export interface DriftItem { + kind: DriftKind; + /** The ESLint rule name where the drift was found. */ + ruleName: string; + /** The RuleProbe rule ID (for md-only items). */ + mdRuleId?: string; + /** Description from the CLAUDE.md mapping (for md-only items). */ + mdDescription?: string; + /** Severity from the CLAUDE.md mapping (for severity/config mismatches). */ + mdSeverity?: EslintSeverity; + /** Severity from the ESLint config file (for severity/config mismatches). */ + eslintSeverity?: 'error' | 'warn' | 'off'; + /** Options from the CLAUDE.md mapping (for config-arg mismatches). */ + mdOptions?: unknown[]; + /** Options from the ESLint config file (for config-arg mismatches). */ + eslintOptions?: unknown[]; + /** Human-readable explanation of the drift. */ + message: string; +} + +/** The result of comparing a CLAUDE.md mapping against an ESLint config. */ +export interface DriftResult { + /** All drift items found. */ + items: DriftItem[]; + /** The CLAUDE.md source file path. */ + mdFile: string; + /** The ESLint config source file path. */ + eslintFile: string; + /** Whether any drift was detected. */ + hasDrift: boolean; +} + +/** Output format for drift reports. */ +export type DriftFormat = 'text' | 'json' | 'markdown'; \ No newline at end of file diff --git a/src/emitter/eslint.ts b/src/emitter/eslint.ts new file mode 100644 index 0000000..503b4ea --- /dev/null +++ b/src/emitter/eslint.ts @@ -0,0 +1,221 @@ +/** + * ESLint config emitter. + * + * Serializes an EslintConfig to a runnable ESLint configuration string. + * Supports flat config (default) and legacy .eslintrc format. + * + * Flat config emits valid ES module JavaScript that can be imported by ESLint. + * Legacy config emits valid JSON suitable for .eslintrc.json files. + * + * Unmappable rules are emitted as commented sections with the original + * instruction text and a one-line reason explaining why no ESLint rule + * can enforce them. + */ + +import type { EslintConfig, EslintFormat, EslintRuleEntry } from '../mapper/types.js'; + +/** Map ESLint plugin names to their npm package identifiers. */ +const PLUGIN_PACKAGES: Record = { + '@typescript-eslint': '@typescript-eslint/eslint-plugin', + 'import': 'eslint-plugin-import', + 'unicorn': 'eslint-plugin-unicorn', + 'jsdoc': 'eslint-plugin-jsdoc', +}; + +/** Format a rule option value for JS output. */ +function formatOptionValue(value: unknown, indent: string): string { + if (value === null || value === undefined) { + return 'null'; + } + if (typeof value === 'string') { + return `'${value.replace(/'/g, "\\'")}'`; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + if (Array.isArray(value)) { + if (value.length === 0) return '[]'; + const items = value.map((v) => formatOptionValue(v, indent)); + return `[${items.join(', ')}]`; + } + if (typeof value === 'object') { + const obj = value as Record; + const entries = Object.entries(obj); + if (entries.length === 0) return '{}'; + const inner = entries + .map(([key, val]) => `${indent} ${key}: ${formatOptionValue(val, indent + ' ')}`) + .join(',\n'); + return `{\n${inner}\n${indent}}`; + } + return String(value); +} + +/** Build a flat config rule entry string as an object property. */ +function flatConfigRuleEntry(entry: EslintRuleEntry): string { + const severity = `'${entry.severity}'`; + if (entry.options && Array.isArray(entry.options) && entry.options.length > 0) { + const opts = entry.options.map((o) => formatOptionValue(o, ' ')).join(', '); + return `'${entry.ruleName}': [${severity}, ${opts}]`; + } + return `'${entry.ruleName}': [${severity}]`; +} + +/** Build a legacy .eslintrc rule entry string as a JSON property. */ +function legacyConfigRuleEntry(entry: EslintRuleEntry): string { + const severity = entry.severity; + if (entry.options && Array.isArray(entry.options) && entry.options.length > 0) { + const opts = entry.options.map((o) => JSON.stringify(o)).join(', '); + return ` "${entry.ruleName}": [${JSON.stringify(severity)}, ${opts}]`; + } + return ` "${entry.ruleName}": ${JSON.stringify(severity)}`; +} + +/** Generate a flat config import section for required plugins. */ +function flatConfigImports(plugins: string[]): string { + if (plugins.length === 0) return ''; + + const lines: string[] = []; + for (const plugin of plugins) { + const pkg = PLUGIN_PACKAGES[plugin] ?? plugin; + lines.push(`import ${plugin.replace(/[^a-zA-Z0-9]/g, '_')}Plugin from '${pkg}';`); + } + return lines.join('\n') + '\n\n'; +} + +/** Generate plugin object entries for flat config. */ +function flatConfigPluginEntries(plugins: string[]): string { + if (plugins.length === 0) return ''; + + const entries = plugins.map((p) => { + const varName = p.replace(/[^a-zA-Z0-9]/g, '_'); + // Property keys with special chars need quotes + const key = /^[$_a-zA-Z][$_a-zA-Z0-9]*$/.test(p) ? p : `'${p}'`; + return ` ${key}: ${varName}Plugin,`; + }); + return entries.join('\n') + '\n'; +} + +/** Build the flat config output string. */ +function emitFlatConfig(config: EslintConfig): string { + const lines: string[] = []; + + // Header comment + lines.push('// ESLint flat config generated by RuleProbe'); + lines.push(`// Source: ${config.sourceFile}`); + lines.push(''); + + // Imports for plugins + if (config.plugins.length > 0) { + lines.push(flatConfigImports(config.plugins)); + } + + // Config export + lines.push('export default ['); + lines.push(' {'); + + // Plugin entries as an object + if (config.plugins.length > 0) { + lines.push(' plugins: {'); + lines.push(flatConfigPluginEntries(config.plugins)); + lines.push(' },'); + } + + // Rules + lines.push(' rules: {'); + for (let i = 0; i < config.rules.length; i++) { + const entry = config.rules[i]!; + const entryStr = flatConfigRuleEntry(entry); + const comma = i < config.rules.length - 1 ? ',' : ''; + lines.push(` ${entryStr}${comma} // ${entry.description}`); + } + lines.push(' },'); + lines.push(' },'); + lines.push('];'); + + // Unmappable rules as comments + if (config.unmappable.length > 0) { + lines.push(''); + lines.push('// ── Unmappable rules ──'); + lines.push('// The following rules from your instruction file have no direct'); + lines.push('// ESLint equivalent. Consider alternative enforcement approaches.'); + lines.push(''); + for (const rule of config.unmappable) { + lines.push(`// [${rule.sourceRuleId}] ${rule.reason}`); + lines.push(`// Original: ${rule.sourceText}`); + lines.push(''); + } + } + + return lines.join('\n'); +} + +/** Build the legacy .eslintrc output string as valid JSON. */ +function emitLegacyConfig(config: EslintConfig): string { + // Build the config as a plain object, then serialize to JSON + const configObj: Record = {}; + + // Plugins + if (config.plugins.length > 0) { + configObj['plugins'] = config.plugins; + } + + // Extends + if (config.plugins.length > 0) { + const extendsList = ['eslint:recommended']; + for (const plugin of config.plugins) { + extendsList.push(`plugin:${plugin}/recommended`); + } + configObj['extends'] = extendsList; + } + + // Rules as an object with severity + options + const rulesObj: Record = {}; + for (const entry of config.rules) { + if (entry.options && Array.isArray(entry.options) && entry.options.length > 0) { + rulesObj[entry.ruleName] = [entry.severity, ...entry.options]; + } else { + rulesObj[entry.ruleName] = entry.severity; + } + } + configObj['rules'] = rulesObj; + + // Serialize to JSON with indentation + const jsonStr = JSON.stringify(configObj, null, 2); + + // Append unmappable rules as a comment block after the JSON + if (config.unmappable.length === 0) { + return jsonStr; + } + + const commentLines: string[] = [ + '', + '// Unmappable rules (not part of the JSON config):', + '// The following rules have no direct ESLint equivalent.', + '', + ]; + for (const rule of config.unmappable) { + commentLines.push(`// [${rule.sourceRuleId}] ${rule.reason}`); + commentLines.push(`// Original: ${rule.sourceText}`); + commentLines.push(''); + } + + return jsonStr + commentLines.join('\n'); +} + +/** + * Serialize an EslintConfig to a runnable ESLint configuration string. + * + * Flat config produces valid ES module JavaScript with plugin imports + * and a proper `export default [...]` structure. Legacy config produces + * valid JSON suitable for .eslintrc.json files. + * + * @param config - The mapped ESLint config + * @param format - Output format: 'flat' (default) or 'legacy' + * @returns A string containing the ESLint configuration + */ +export function emitEslintConfig(config: EslintConfig, format: EslintFormat = 'flat'): string { + if (format === 'legacy') { + return emitLegacyConfig(config); + } + return emitFlatConfig(config); +} \ No newline at end of file diff --git a/src/extractor/index.ts b/src/extractor/index.ts new file mode 100644 index 0000000..2ad2a56 --- /dev/null +++ b/src/extractor/index.ts @@ -0,0 +1,272 @@ +/** + * ESLint config to CLAUDE.md rules section extractor. + * + * Takes a parsed ESLint config and emits a markdown rules section. + * Each ESLint rule is reverse-mapped to a one-line prose instruction. + * Subjective/stylistic rules without a prose equivalent are emitted + * as skipped comments. + */ + +import type { ParsedEslintConfig } from '../drift/types.js'; +import { findByEslintRuleName, findAllByEslintRuleName } from '../mappings/index.js'; +import { getProseForRule, isStylisticRule } from '../mappings/prose-templates.js'; + +/** A rule that was reverse-mapped to a prose instruction. */ +export interface ExtractedRule { + /** The one-line prose instruction. */ + prose: string; + /** The ESLint rule name that produced this instruction. */ + eslintRuleName: string; + /** The RuleProbe pattern type, if a reverse mapping exists. */ + patternType: string | null; +} + +/** A rule that was skipped during extraction. */ +export interface SkippedRule { + /** The ESLint rule name. */ + eslintRuleName: string; + /** Why the rule was skipped. */ + reason: 'stylistic' | 'no-mapping' | 'off'; + /** The severity from the config (for context). */ + severity: string; +} + +/** The result of extracting rules from an ESLint config. */ +export interface ExtractionResult { + /** Successfully extracted prose instructions. */ + rules: ExtractedRule[]; + /** Rules that were skipped with reasons. */ + skipped: SkippedRule[]; + /** Path to the source ESLint config file. */ + sourceFile: string; +} + +/** + * Extract prose rules from a parsed ESLint config. + * + * For each rule in the config: + * - If severity is "off", skip it + * - If the rule maps to a RuleProbe pattern, generate prose + * - If the rule is stylistic, mark it as skipped + * - If no mapping exists, mark it as skipped + * + * @param config - A parsed ESLint config + * @returns An ExtractionResult with extracted rules and skipped rules + */ +export function extractRules(config: ParsedEslintConfig): ExtractionResult { + const rules: ExtractedRule[] = []; + const skipped: SkippedRule[] = []; + + // Track which naming convention selectors we've already emitted prose for + const seenNamingSelectors = new Set(); + + for (const eslintRule of config.rules) { + // Skip disabled rules + if (eslintRule.severity === 'off') { + continue; + } + + const ruleName = eslintRule.ruleName; + + // Check if this is a stylistic rule + if (isStylisticRule(ruleName)) { + skipped.push({ + eslintRuleName: ruleName, + reason: 'stylistic', + severity: eslintRule.severity, + }); + continue; + } + + // Try to generate prose from the template + const prose = getProseForRule(ruleName, eslintRule.options); + + if (prose !== null) { + // Special handling: naming-convention produces multiple prose entries + if (ruleName === '@typescript-eslint/naming-convention') { + const namingEntries = extractNamingEntries(eslintRule.options); + for (const entry of namingEntries) { + if (!seenNamingSelectors.has(entry.selector)) { + seenNamingSelectors.add(entry.selector); + rules.push({ + prose: entry.prose, + eslintRuleName: ruleName, + patternType: entry.patternType, + }); + } + } + continue; + } + + // Find the corresponding pattern type(s) + const allMappings = findAllByEslintRuleName(ruleName); + const patternType = allMappings.length > 0 ? allMappings[0]!.patternType : null; + + rules.push({ + prose, + eslintRuleName: ruleName, + patternType, + }); + continue; + } + + // No prose template found, check if there's a mapping entry + const mapping = findByEslintRuleName(ruleName); + if (mapping) { + // There's a mapping but no prose template (shouldn't happen, but handle it) + rules.push({ + prose: mapping.description, + eslintRuleName: ruleName, + patternType: mapping.patternType, + }); + continue; + } + + // No mapping at all + skipped.push({ + eslintRuleName: ruleName, + reason: 'no-mapping', + severity: eslintRule.severity, + }); + } + + return { + rules, + skipped, + sourceFile: config.sourceFile, + }; +} + +/** + * Format an ExtractionResult as a markdown fragment. + * + * Produces a "## Rules" section with bullet points for each + * extracted rule, followed by an HTML comment block listing + * skipped rules. + */ +export function formatRulesMarkdown(result: ExtractionResult): string { + const lines: string[] = []; + + lines.push('## Rules'); + lines.push(''); + + for (const rule of result.rules) { + lines.push(`- ${rule.prose}`); + } + + if (result.skipped.length > 0) { + lines.push(''); + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Extract naming convention entries from ESLint rule options. + * + * Parses the naming-convention rule's options array and generates + * prose entries for each selector/format combination. + */ +function extractNamingEntries(options: unknown[]): Array<{ prose: string; selector: string; patternType: string }> { + if (options.length === 0) { + return [{ prose: 'Enforce naming conventions for TypeScript identifiers.', selector: 'default', patternType: 'camelCase' }]; + } + + const obj = options[0]; + if (!obj || typeof obj !== 'object' || !('rules' in (obj as Record))) { + return [{ prose: 'Enforce naming conventions for TypeScript identifiers.', selector: 'default', patternType: 'camelCase' }]; + } + + const rules = (obj as Record)['rules']; + if (!Array.isArray(rules)) { + return [{ prose: 'Enforce naming conventions for TypeScript identifiers.', selector: 'default', patternType: 'camelCase' }]; + } + + const entries: Array<{ prose: string; selector: string; patternType: string }> = []; + + for (const rule of rules) { + if (typeof rule !== 'object' || rule === null) continue; + const r = rule as Record; + const selector = String(r['selector'] ?? 'default'); + const format = r['format']; + + if (!Array.isArray(format)) continue; + + for (const f of format) { + const fmt = String(f); + if (fmt === 'PascalCase') { + const targets = pascalTargets(selector); + entries.push({ + prose: `Use PascalCase for ${targets}.`, + selector: `PascalCase-${selector}`, + patternType: 'PascalCase', + }); + } else if (fmt === 'camelCase') { + const targets = camelTargets(selector); + entries.push({ + prose: `Use camelCase for ${targets}.`, + selector: `camelCase-${selector}`, + patternType: 'camelCase', + }); + } else if (fmt === 'UPPER_CASE') { + const targets = upperTargets(selector); + entries.push({ + prose: `Use UPPER_CASE for ${targets}.`, + selector: `UPPER_CASE-${selector}`, + patternType: 'UPPER_CASE', + }); + } + } + } + + return entries.length > 0 ? entries : [{ prose: 'Enforce naming conventions for TypeScript identifiers.', selector: 'default', patternType: 'camelCase' }]; +} + +/** Map ESLint selector names to human-readable form for PascalCase rules. */ +function pascalTargets(selector: string): string { + const map: Record = { + 'class': 'classes', + 'interface': 'interfaces', + 'typeAlias': 'type aliases', + 'enum': 'enums', + 'enumMember': 'enum members', + 'default': 'types and interfaces', + }; + return map[selector] ?? selector; +} + +/** Map ESLint selector names to human-readable form for camelCase rules. */ +function camelTargets(selector: string): string { + const map: Record = { + 'variable': 'variables', + 'function': 'functions', + 'parameter': 'parameters', + 'classMethod': 'class methods', + 'classProperty': 'class properties', + 'objectLiteralProperty': 'object properties', + 'typeProperty': 'type properties', + 'default': 'variables and functions', + }; + return map[selector] ?? selector; +} + +/** Map ESLint selector names to human-readable form for UPPER_CASE rules. */ +function upperTargets(selector: string): string { + const map: Record = { + 'variable': 'constants', + 'default': 'constants', + }; + return map[selector] ?? selector; +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 51f5052..32ac799 100644 --- a/src/index.ts +++ b/src/index.ts @@ -68,13 +68,6 @@ export type { } from './llm/index.js'; export type { OpenAiProviderConfig } from './llm/index.js'; -// Agent invocation exports -export { buildAgentConfig } from './runner/agent-configs.js'; -export { invokeAgent, isAgentSdkAvailable, hasAgentOutput } from './runner/agent-invoker.js'; -export { watchForCompletion, countCodeFiles } from './runner/watch-mode.js'; -export type { AgentInvocationConfig, RunOptions } from './runner/agent-configs.js'; -export type { InvocationResult } from './runner/agent-invoker.js'; -export type { WatchOptions, WatchResult } from './runner/watch-mode.js'; /** * Extract rules from raw markdown content. @@ -131,16 +124,10 @@ export function generateReport( 'naming', 'forbidden-pattern', 'structure', - 'test-requirement', 'import-pattern', 'error-handling', 'type-safety', 'code-style', - 'dependency', - 'preference', - 'file-structure', - 'tooling', - 'testing', ]; const byCategory = {} as Record; for (const cat of allCategories) { diff --git a/src/llm/extract.ts b/src/llm/extract.ts index 9595275..a4b0018 100644 --- a/src/llm/extract.ts +++ b/src/llm/extract.ts @@ -11,13 +11,13 @@ import type { RuleCategory, VerifierType } from '../types.js'; /** Valid categories for LLM-extracted rules. */ const VALID_CATEGORIES: ReadonlySet = new Set([ - 'naming', 'forbidden-pattern', 'structure', 'test-requirement', - 'import-pattern', 'error-handling', 'type-safety', 'code-style', 'dependency', + 'naming', 'forbidden-pattern', 'structure', + 'import-pattern', 'error-handling', 'type-safety', 'code-style', ]); /** Valid verifier types. */ const VALID_VERIFIERS: ReadonlySet = new Set([ - 'ast', 'regex', 'filesystem', 'treesitter', + 'ast', 'regex', 'filesystem', ]); /** Prompt parts returned by buildExtractionPrompt. */ diff --git a/src/llm/rubric-decompose.ts b/src/llm/rubric-decompose.ts index 2aadac7..3297cd7 100644 --- a/src/llm/rubric-decompose.ts +++ b/src/llm/rubric-decompose.ts @@ -11,13 +11,13 @@ import type { DecomposedRubric, DecompositionResult, RubricCheck } from './rubri /** Valid categories for rubric checks. */ const VALID_CATEGORIES: ReadonlySet = new Set([ - 'naming', 'forbidden-pattern', 'structure', 'test-requirement', - 'import-pattern', 'error-handling', 'type-safety', 'code-style', 'dependency', + 'naming', 'forbidden-pattern', 'structure', + 'import-pattern', 'error-handling', 'type-safety', 'code-style', ]); /** Valid verifier types for rubric checks. */ const VALID_VERIFIERS: ReadonlySet = new Set([ - 'ast', 'regex', 'filesystem', 'treesitter', + 'ast', 'regex', 'filesystem', ]); /** Prompt pair for rubric decomposition. */ diff --git a/src/mapper/index.ts b/src/mapper/index.ts new file mode 100644 index 0000000..f7c331b --- /dev/null +++ b/src/mapper/index.ts @@ -0,0 +1,223 @@ +/** + * RuleProbe-to-ESLint config mapper. + * + * Takes a RuleSet (parsed from an instruction file) and produces an + * EslintConfig with mappable ESLint rule entries and unmappable rules + * annotated with reasons. + * + * Naming conventions are merged: pascalcase-types, camelcase-variables, + * and upper-case-constants all map into a single + * @typescript-eslint/naming-convention config. + */ + +import type { RuleSet, Rule } from '../types.js'; +import type { EslintConfig, EslintRuleEntry, EslintSeverity, UnmappableRule } from './types.js'; +import { UNMAPPABLE_TYPES } from '../mappings/index.js'; +import { mapNoAny } from './mappings/no-any.js'; +import { mapNamedExports } from './mappings/named-exports.js'; +import { mapKebabCaseFiles } from './mappings/kebab-case-files.js'; +import { mapMaxFileLines, mapMaxLineLength } from './mappings/max-lines.js'; +import { mapNoConsoleLog, mapNoConsoleExtended } from './mappings/no-console.js'; +import { mapJsdocRequired } from './mappings/jsdoc-required.js'; +import { + resetNamingAccumulator, + addNamingPattern, + hasNamingEntries, + buildNamingConventionRule, +} from './mappings/naming-convention.js'; +import { + mapNoVar, + mapPreferConst, + mapNoElseAfterReturn, + mapNoNestedTernary, + mapNoMagicNumbers, + mapConsistentSemicolons, + mapQuoteStyle, +} from './mappings/code-style.js'; +import { mapNoEmptyCatch, mapThrowErrorOnly } from './mappings/error-handling.js'; +import { + mapNoEnum, + mapNoTypeAssertions, + mapNonNullAssertions, + mapNoImplicitAny, + mapNoUnusedExports, + mapNoTsDirectives, +} from './mappings/type-safety.js'; +import { mapMaxFunctionLength, mapMaxParams } from './mappings/function-limits.js'; +import { + mapNoWildcardExports, + mapNoNamespaceImports, + mapNoPathAliases, + mapNoDeepRelativeImports, +} from './mappings/imports.js'; +import { mapNoTodoComments } from './mappings/no-todo.js'; + +/** Pattern types that are handled by the naming-convention merger. */ +const NAMING_PATTERN_TYPES = new Set(['PascalCase', 'camelCase', 'UPPER_CASE']); + +/** Pattern types with no ESLint equivalent. Imported from mappings module. */ + +/** + * Map a single RuleProbe rule to an ESLint rule entry. + * + * Returns an EslintRuleEntry if the rule can be mapped, or null if + * it needs special handling (e.g. naming conventions are merged). + */ +function mapRule(rule: Rule): EslintRuleEntry | null { + const { type, expected } = rule.pattern; + const severity: EslintSeverity = rule.severity === 'error' ? 'error' : 'warn'; + + // Naming convention rules are handled separately via the accumulator + if (NAMING_PATTERN_TYPES.has(type)) { + return null; + } + + switch (type) { + // no-any + case 'no-any': + return { ...mapNoAny(), severity, sourceRuleId: rule.id, description: rule.description }; + + // no-console + case 'no-console-log': + return { ...mapNoConsoleLog(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-console-extended': + return { ...mapNoConsoleExtended(), severity, sourceRuleId: rule.id, description: rule.description }; + + // named-exports + case 'named-exports-only': + return { ...mapNamedExports(), severity, sourceRuleId: rule.id, description: rule.description }; + + // kebab-case files + case 'kebab-case': + return { ...mapKebabCaseFiles(), severity, sourceRuleId: rule.id, description: rule.description }; + + // max-lines + case 'max-file-length': + return { ...mapMaxFileLines(expected), severity, sourceRuleId: rule.id, description: rule.description }; + case 'max-line-length': + return { ...mapMaxLineLength(expected), severity, sourceRuleId: rule.id, description: rule.description }; + + // jsdoc + case 'jsdoc-required': + return { ...mapJsdocRequired(), severity, sourceRuleId: rule.id, description: rule.description }; + + // code style + case 'no-var': + return { ...mapNoVar(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'prefer-const': + return { ...mapPreferConst(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-else-after-return': + return { ...mapNoElseAfterReturn(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-nested-ternary': + return { ...mapNoNestedTernary(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-magic-numbers': + return { ...mapNoMagicNumbers(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'consistent-semicolons': + return { ...mapConsistentSemicolons(expected), severity, sourceRuleId: rule.id, description: rule.description }; + case 'quote-style': + return { ...mapQuoteStyle(expected), severity, sourceRuleId: rule.id, description: rule.description }; + + // error handling + case 'no-empty-catch': + return { ...mapNoEmptyCatch(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'throw-error-only': + return { ...mapThrowErrorOnly(), severity, sourceRuleId: rule.id, description: rule.description }; + + // type safety + case 'no-enum': + return { ...mapNoEnum(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-type-assertions': + return { ...mapNoTypeAssertions(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-non-null-assertions': + return { ...mapNonNullAssertions(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-implicit-any': + return { ...mapNoImplicitAny(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-unused-exports': + return { ...mapNoUnusedExports(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-ts-directives': + return { ...mapNoTsDirectives(), severity, sourceRuleId: rule.id, description: rule.description }; + + // function limits + case 'max-function-length': + return { ...mapMaxFunctionLength(expected), severity, sourceRuleId: rule.id, description: rule.description }; + case 'max-params': + return { ...mapMaxParams(expected), severity, sourceRuleId: rule.id, description: rule.description }; + + // imports + case 'no-wildcard-exports': + return { ...mapNoWildcardExports(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-namespace-imports': + return { ...mapNoNamespaceImports(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-path-aliases': + return { ...mapNoPathAliases(), severity, sourceRuleId: rule.id, description: rule.description }; + case 'no-deep-relative-imports': + return { ...mapNoDeepRelativeImports(expected), severity, sourceRuleId: rule.id, description: rule.description }; + + // comments + case 'no-todo-comments': + return { ...mapNoTodoComments(), severity, sourceRuleId: rule.id, description: rule.description }; + + default: + return null; + } +} + +/** + * Map a RuleSet to an EslintConfig. + * + * Iterates all rules in the RuleSet, mapping each to an ESLint rule + * entry where possible. Naming conventions are merged into a single + * @typescript-eslint/naming-convention config. Rules with no ESLint + * equivalent are collected as UnmappableRule entries with reasons. + * + * @param ruleSet - The parsed RuleSet from an instruction file + * @returns An EslintConfig with mappable rules, unmappable rules, and required plugins + */ +export function mapRuleSetToEslintConfig(ruleSet: RuleSet): EslintConfig { + resetNamingAccumulator(); + + const rules: EslintRuleEntry[] = []; + const unmappable: UnmappableRule[] = []; + + for (const rule of ruleSet.rules) { + const { type } = rule.pattern; + + // Naming convention rules are accumulated and merged + if (NAMING_PATTERN_TYPES.has(type)) { + addNamingPattern(type, rule.id); + continue; + } + + const entry = mapRule(rule); + if (entry) { + rules.push(entry); + } else { + // Check if we have a known reason for this pattern type + const reason = UNMAPPABLE_TYPES[type] ?? `No ESLint rule enforces "${type}" constraints.`; + unmappable.push({ + sourceRuleId: rule.id, + sourceText: rule.source, + reason, + }); + } + } + + // Add the merged naming convention rule if any naming rules were found + if (hasNamingEntries()) { + rules.push(buildNamingConventionRule()); + } + + // Deduplicate plugins + const plugins = [...new Set( + rules + .map((r) => r.plugin) + .filter((p): p is string => p !== undefined), + )]; + + return { + rules, + unmappable, + plugins, + sourceFile: ruleSet.sourceFile, + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/code-style.ts b/src/mapper/mappings/code-style.ts new file mode 100644 index 0000000..9b2990a --- /dev/null +++ b/src/mapper/mappings/code-style.ts @@ -0,0 +1,84 @@ +/** + * Mapping: code-style rules → ESLint equivalents + * + * Covers no-var, prefer-const, no-else-after-return, + * no-nested-ternary, no-magic-numbers, semicolons, and quotes. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map no-var pattern. */ +export function mapNoVar(): EslintRuleEntry { + return { + ruleName: 'no-var', + severity: 'error', + sourceRuleId: '', + description: 'No var declarations (use const or let)', + }; +} + +/** Map prefer-const pattern. */ +export function mapPreferConst(): EslintRuleEntry { + return { + ruleName: 'prefer-const', + severity: 'warn', + options: [{ destructuring: 'all' }], + sourceRuleId: '', + description: 'Prefer const for variables that are never reassigned', + }; +} + +/** Map no-else-after-return pattern. */ +export function mapNoElseAfterReturn(): EslintRuleEntry { + return { + ruleName: 'no-else-after-return', + severity: 'warn', + sourceRuleId: '', + description: 'Do not use else after a return statement', + }; +} + +/** Map no-nested-ternary pattern. */ +export function mapNoNestedTernary(): EslintRuleEntry { + return { + ruleName: 'no-nested-ternary', + severity: 'warn', + sourceRuleId: '', + description: 'Nested ternary expressions are not allowed', + }; +} + +/** Map no-magic-numbers pattern. */ +export function mapNoMagicNumbers(): EslintRuleEntry { + return { + ruleName: 'no-magic-numbers', + severity: 'warn', + options: [{ ignore: [0, 1, -1], ignoreArrayIndexes: true, detectObjects: false }], + sourceRuleId: '', + description: 'Magic numbers must be replaced with named constants', + }; +} + +/** Map consistent-semicolons pattern to semi rule. */ +export function mapConsistentSemicolons(expected: string | boolean): EslintRuleEntry { + const semiStyle = expected === 'never' ? 'never' : 'always'; + return { + ruleName: 'semi', + severity: 'warn', + options: [semiStyle], + sourceRuleId: '', + description: 'Enforce consistent semicolon usage', + }; +} + +/** Map quote-style pattern to quotes rule. */ +export function mapQuoteStyle(expected: string | boolean): EslintRuleEntry { + const quoteType = expected === 'single' ? 'single' : 'double'; + return { + ruleName: 'quotes', + severity: 'warn', + options: ['error', quoteType, { avoidEscape: true }], + sourceRuleId: '', + description: `Strings must use ${quoteType} quotes`, + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/error-handling.ts b/src/mapper/mappings/error-handling.ts new file mode 100644 index 0000000..5f1b2be --- /dev/null +++ b/src/mapper/mappings/error-handling.ts @@ -0,0 +1,28 @@ +/** + * Mapping: error-handling rules → ESLint equivalents + * + * Covers no-empty-catch and throw-error-only. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map no-empty-catch pattern to no-empty (with allowCatchParents opt). */ +export function mapNoEmptyCatch(): EslintRuleEntry { + return { + ruleName: 'no-empty', + severity: 'error', + options: [{ allowEmptyCatch: false }], + sourceRuleId: '', + description: 'Catch blocks must not be empty', + }; +} + +/** Map throw-error-only pattern to no-throw-literal. */ +export function mapThrowErrorOnly(): EslintRuleEntry { + return { + ruleName: 'no-throw-literal', + severity: 'error', + sourceRuleId: '', + description: 'Only Error objects may be thrown', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/function-limits.ts b/src/mapper/mappings/function-limits.ts new file mode 100644 index 0000000..9b5cee6 --- /dev/null +++ b/src/mapper/mappings/function-limits.ts @@ -0,0 +1,31 @@ +/** + * Mapping: function size limit rules → ESLint equivalents + * + * Covers max-function-length and max-params. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map max-function-length pattern to max-lines-per-function. */ +export function mapMaxFunctionLength(expected: string | boolean): EslintRuleEntry { + const max = typeof expected === 'string' ? parseInt(expected, 10) : 50; + return { + ruleName: 'max-lines-per-function', + severity: 'warn', + options: [{ max: Number.isNaN(max) ? 50 : max, skipBlankLines: true, skipComments: true }], + sourceRuleId: '', + description: 'Functions must not exceed the maximum line count', + }; +} + +/** Map max-params pattern to max-params. */ +export function mapMaxParams(expected: string | boolean): EslintRuleEntry { + const max = typeof expected === 'string' ? parseInt(expected, 10) : 4; + return { + ruleName: 'max-params', + severity: 'warn', + options: [Number.isNaN(max) ? 4 : max], + sourceRuleId: '', + description: 'Functions must not have too many parameters', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/imports.ts b/src/mapper/mappings/imports.ts new file mode 100644 index 0000000..96689c2 --- /dev/null +++ b/src/mapper/mappings/imports.ts @@ -0,0 +1,53 @@ +/** + * Mapping: import rules → ESLint equivalents + * + * Covers no-wildcard-exports, no-namespace-imports, + * no-path-aliases, and no-deep-relative-imports. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map no-wildcard-exports pattern to import/no-anonymous-default-export. */ +export function mapNoWildcardExports(): EslintRuleEntry { + return { + ruleName: 'import/no-anonymous-default-export', + plugin: 'import', + severity: 'warn', + sourceRuleId: '', + description: 'No wildcard re-exports (use named re-exports)', + }; +} + +/** Map no-namespace-imports pattern to import/no-namespace. */ +export function mapNoNamespaceImports(): EslintRuleEntry { + return { + ruleName: 'import/no-namespace', + plugin: 'import', + severity: 'warn', + sourceRuleId: '', + description: 'Namespace imports (import * as) are not allowed', + }; +} + +/** Map no-path-aliases pattern to no-restricted-imports. */ +export function mapNoPathAliases(): EslintRuleEntry { + return { + ruleName: 'no-restricted-imports', + severity: 'warn', + sourceRuleId: '', + description: 'Imports must use relative paths, not path aliases', + }; +} + +/** Map no-deep-relative-imports pattern to import/no-relative-parent. */ +export function mapNoDeepRelativeImports(expected: string | boolean): EslintRuleEntry { + const maxDepth = typeof expected === 'string' ? parseInt(expected, 10) : 2; + return { + ruleName: 'import/no-relative-parent', + plugin: 'import', + severity: 'warn', + options: [{ maxDepth: Number.isNaN(maxDepth) ? 2 : maxDepth }], + sourceRuleId: '', + description: 'Relative imports must not go too deep', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/jsdoc-required.ts b/src/mapper/mappings/jsdoc-required.ts new file mode 100644 index 0000000..9beba17 --- /dev/null +++ b/src/mapper/mappings/jsdoc-required.ts @@ -0,0 +1,28 @@ +/** + * Mapping: jsdoc-required → jsdoc/require-jsdoc + * + * Requires JSDoc comments on public functions. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map jsdoc-required pattern to jsdoc/require-jsdoc. */ +export function mapJsdocRequired(): EslintRuleEntry { + return { + ruleName: 'jsdoc/require-jsdoc', + plugin: 'jsdoc', + severity: 'warn', + options: [{ + require: { + FunctionDeclaration: true, + MethodDefinition: true, + ClassDeclaration: true, + ArrowFunctionExpression: true, + FunctionExpression: true, + }, + publicOnly: true, + }], + sourceRuleId: '', + description: 'Every public function must have a JSDoc comment', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/kebab-case-files.ts b/src/mapper/mappings/kebab-case-files.ts new file mode 100644 index 0000000..d76d8ba --- /dev/null +++ b/src/mapper/mappings/kebab-case-files.ts @@ -0,0 +1,19 @@ +/** + * Mapping: kebab-case filenames → unicorn/filename-case + * + * Enforces kebab-case for file names using the unicorn plugin. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map kebab-case filename pattern to unicorn/filename-case. */ +export function mapKebabCaseFiles(): EslintRuleEntry { + return { + ruleName: 'unicorn/filename-case', + plugin: 'unicorn', + severity: 'error', + options: [{ cases: { kebab: true } }], + sourceRuleId: '', + description: 'File names must use kebab-case', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/max-lines.ts b/src/mapper/mappings/max-lines.ts new file mode 100644 index 0000000..94692d2 --- /dev/null +++ b/src/mapper/mappings/max-lines.ts @@ -0,0 +1,31 @@ +/** + * Mapping: max-file-length → max-lines + * + * Enforces a maximum number of lines per file. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map max-file-length pattern to max-lines ESLint rule. */ +export function mapMaxFileLines(expected: string | boolean): EslintRuleEntry { + const max = typeof expected === 'string' ? parseInt(expected, 10) : 300; + return { + ruleName: 'max-lines', + severity: 'warn', + options: [{ max: Number.isNaN(max) ? 300 : max, skipBlankLines: true, skipComments: true }], + sourceRuleId: '', + description: 'Files must not exceed the maximum line count', + }; +} + +/** Map max-line-length pattern to max-len ESLint rule. */ +export function mapMaxLineLength(expected: string | boolean): EslintRuleEntry { + const code = typeof expected === 'string' ? parseInt(expected, 10) : 120; + return { + ruleName: 'max-len', + severity: 'warn', + options: [{ code: Number.isNaN(code) ? 120 : code }], + sourceRuleId: '', + description: 'Lines must not exceed the maximum length', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/named-exports.ts b/src/mapper/mappings/named-exports.ts new file mode 100644 index 0000000..8b17fc2 --- /dev/null +++ b/src/mapper/mappings/named-exports.ts @@ -0,0 +1,18 @@ +/** + * Mapping: named-exports-only → import/no-default-export + * + * Requires named exports instead of default exports. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map named-exports-only pattern to import/no-default-export. */ +export function mapNamedExports(): EslintRuleEntry { + return { + ruleName: 'import/no-default-export', + plugin: 'import', + severity: 'error', + sourceRuleId: '', + description: 'Only named exports are allowed, no default exports', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/naming-convention.ts b/src/mapper/mappings/naming-convention.ts new file mode 100644 index 0000000..ab6a608 --- /dev/null +++ b/src/mapper/mappings/naming-convention.ts @@ -0,0 +1,173 @@ +/** + * Mapping: naming rules → @typescript-eslint/naming-convention + * + * Merges multiple naming conventions (camelCase variables, PascalCase types, + * UPPER_CASE constants) into a single naming-convention rule config. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Selector and format for a naming convention entry. */ +interface NamingEntry { + selector: string; + format: string[]; + modifiers?: string[]; + leadingUnderscore?: string; + filter?: string; +} + +/** Map PascalCase types pattern to naming-convention entries. */ +function pascalCaseTypes(): NamingEntry[] { + return [ + { selector: 'class', format: ['PascalCase'] }, + { selector: 'interface', format: ['PascalCase'] }, + { selector: 'typeAlias', format: ['PascalCase'] }, + { selector: 'enum', format: ['PascalCase'] }, + { selector: 'enumMember', format: ['PascalCase'] }, + ]; +} + +/** Map camelCase variables/functions pattern to naming-convention entries. */ +function camelCaseVariables(): NamingEntry[] { + return [ + { + selector: 'variable', + format: ['camelCase', 'UPPER_CASE'], + leadingUnderscore: 'allow', + }, + { + selector: 'function', + format: ['camelCase'], + }, + { + selector: 'parameter', + format: ['camelCase'], + leadingUnderscore: 'allow', + }, + { + selector: 'classMethod', + format: ['camelCase'], + }, + { + selector: 'classProperty', + format: ['camelCase', 'UPPER_CASE'], + leadingUnderscore: 'allow', + }, + { + selector: 'objectLiteralProperty', + format: ['camelCase', 'UPPER_CASE'], + }, + { + selector: 'typeProperty', + format: ['camelCase', 'UPPER_CASE'], + }, + ]; +} + +/** Map UPPER_CASE constants pattern to naming-convention entries. */ +function upperCaseConstants(): NamingEntry[] { + return [ + { + selector: 'variable', + modifiers: ['const'], + format: ['camelCase', 'UPPER_CASE'], + leadingUnderscore: 'allow', + }, + ]; +} + +/** Map camelCase identifiers (general) to naming-convention entries. */ +function camelCaseGeneral(): NamingEntry[] { + return [ + { + selector: 'default', + format: ['camelCase'], + leadingUnderscore: 'allow', + }, + ]; +} + +/** Build naming-convention options from accumulated entries. */ +function buildNamingOptions(entries: NamingEntry[]): unknown[] { + const rules = entries.map((entry) => { + const rule: Record = { + selector: entry.selector, + format: entry.format, + }; + if (entry.modifiers) { + rule['modifiers'] = entry.modifiers; + } + if (entry.leadingUnderscore) { + rule['leadingUnderscore'] = entry.leadingUnderscore; + } + if (entry.filter) { + rule['filter'] = entry.filter; + } + return rule; + }); + return [{ rules }]; +} + +/** Accumulated naming entries for merging. */ +let namingEntries: NamingEntry[] = []; + +/** Source rule IDs that contributed to naming-convention. */ +let namingSourceIds: string[] = []; + +/** Reset accumulated naming entries. Called before processing a RuleSet. */ +export function resetNamingAccumulator(): void { + namingEntries = []; + namingSourceIds = []; +} + +/** + * Add naming entries for a given pattern type. + * + * Returns true if the pattern type was handled, false otherwise. + */ +export function addNamingPattern( + patternType: string, + sourceRuleId: string, +): boolean { + switch (patternType) { + case 'PascalCase': + namingEntries.push(...pascalCaseTypes()); + namingSourceIds.push(sourceRuleId); + return true; + case 'camelCase': + if (namingSourceIds.length === 0) { + // First camelCase rule: use general selectors + namingEntries.push(...camelCaseVariables()); + } + namingSourceIds.push(sourceRuleId); + return true; + case 'UPPER_CASE': + namingEntries.push(...upperCaseConstants()); + namingSourceIds.push(sourceRuleId); + return true; + default: + return false; + } +} + +/** Whether any naming entries have been accumulated. */ +export function hasNamingEntries(): boolean { + return namingEntries.length > 0; +} + +/** Build the merged naming-convention rule entry from accumulated entries. */ +export function buildNamingConventionRule(): EslintRuleEntry { + // If we only have camelCase, use general selectors instead of specific ones + const uniqueTypes = new Set(namingEntries.map((e) => e.format.join(','))); + const entries = namingEntries.length > 0 ? namingEntries : camelCaseGeneral(); + const options = buildNamingOptions(entries); + + return { + ruleName: '@typescript-eslint/naming-convention', + plugin: '@typescript-eslint', + severity: 'error', + options, + sourceRuleId: namingSourceIds.join(', '), + description: 'Naming conventions for TypeScript identifiers', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/no-any.ts b/src/mapper/mappings/no-any.ts new file mode 100644 index 0000000..ad5380d --- /dev/null +++ b/src/mapper/mappings/no-any.ts @@ -0,0 +1,18 @@ +/** + * Mapping: no-any → @typescript-eslint/no-explicit-any + * + * Bans the `any` type in TypeScript code. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map no-any pattern to @typescript-eslint/no-explicit-any. */ +export function mapNoAny(): EslintRuleEntry { + return { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'error', + sourceRuleId: '', + description: 'The "any" type must not be used', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/no-console.ts b/src/mapper/mappings/no-console.ts new file mode 100644 index 0000000..4e2c311 --- /dev/null +++ b/src/mapper/mappings/no-console.ts @@ -0,0 +1,30 @@ +/** + * Mapping: no-console-log and no-console-extended → no-console + * + * Bans console statements in production code. + * The extended variant bans all console methods; + * the basic variant bans only console.log. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map no-console-log to no-console with allow: []. */ +export function mapNoConsoleLog(): EslintRuleEntry { + return { + ruleName: 'no-console', + severity: 'error', + options: [{ allow: [] }], + sourceRuleId: '', + description: 'console.log must not be used in production code', + }; +} + +/** Map no-console-extended to no-console (bans all console methods). */ +export function mapNoConsoleExtended(): EslintRuleEntry { + return { + ruleName: 'no-console', + severity: 'error', + sourceRuleId: '', + description: 'Console statements must not be used', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/no-todo.ts b/src/mapper/mappings/no-todo.ts new file mode 100644 index 0000000..4317ec6 --- /dev/null +++ b/src/mapper/mappings/no-todo.ts @@ -0,0 +1,23 @@ +/** + * Mapping: no-todo-comments → eslint-plugin-no-todo-comment + * + * Bans TODO/FIXME/HACK/XXX comments in production code. + * Uses the `no-todo-comments` rule which is available in + * eslint-plugin-no-todo-comment or similar packages. + * + * If no suitable plugin is available, falls back to a + * no-warning-comments rule as a close approximation. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map no-todo-comments pattern. */ +export function mapNoTodoComments(): EslintRuleEntry { + return { + ruleName: 'no-warning-comments', + severity: 'warn', + options: [{ terms: ['todo', 'fixme', 'hack', 'xxx'], location: 'start' }], + sourceRuleId: '', + description: 'No TODO/FIXME/HACK/XXX comments in production code', + }; +} \ No newline at end of file diff --git a/src/mapper/mappings/type-safety.ts b/src/mapper/mappings/type-safety.ts new file mode 100644 index 0000000..a3e5bbb --- /dev/null +++ b/src/mapper/mappings/type-safety.ts @@ -0,0 +1,79 @@ +/** + * Mapping: type-safety rules → ESLint equivalents + * + * Covers no-enum, no-type-assertions, no-non-null-assertions, + * no-implicit-any, no-unused-exports, and no-ts-directives. + */ + +import type { EslintRuleEntry } from '../types.js'; + +/** Map no-enum pattern to @typescript-eslint/no-enum. */ +export function mapNoEnum(): EslintRuleEntry { + return { + ruleName: '@typescript-eslint/no-enum', + plugin: '@typescript-eslint', + severity: 'warn', + sourceRuleId: '', + description: 'Enums must not be used; prefer union types', + }; +} + +/** Map no-type-assertions pattern to @typescript-eslint/consistent-type-assertions. */ +export function mapNoTypeAssertions(): EslintRuleEntry { + return { + ruleName: '@typescript-eslint/consistent-type-assertions', + plugin: '@typescript-eslint', + severity: 'warn', + options: [{ assertionStyle: 'as', objectLiteralTypeAssertions: 'never' }], + sourceRuleId: '', + description: 'Type assertions (as casts) must not be used', + }; +} + +/** Map no-non-null-assertions pattern to @typescript-eslint/no-non-null-assertion. */ +export function mapNonNullAssertions(): EslintRuleEntry { + return { + ruleName: '@typescript-eslint/no-non-null-assertion', + plugin: '@typescript-eslint', + severity: 'warn', + sourceRuleId: '', + description: 'Non-null assertions (!) must not be used', + }; +} + +/** Map no-implicit-any pattern to @typescript-eslint/no-explicit-any. + * Note: implicit any is caught by TypeScript's noImplicitAny compiler option, + * not by an ESLint rule. This maps to the closest ESLint equivalent. */ +export function mapNoImplicitAny(): EslintRuleEntry { + return { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'warn', + sourceRuleId: '', + description: 'No implicit any types (use @typescript-eslint/no-explicit-any; enable noImplicitAny in tsconfig)', + }; +} + +/** Map no-unused-exports pattern to import/no-unused-modules. */ +export function mapNoUnusedExports(): EslintRuleEntry { + return { + ruleName: 'import/no-unused-modules', + plugin: 'import', + severity: 'warn', + options: [{ missingExports: true, unusedExports: true }], + sourceRuleId: '', + description: 'Exported declarations must be imported by other files', + }; +} + +/** Map no-ts-directives pattern to @typescript-eslint/ban-ts-comment. */ +export function mapNoTsDirectives(): EslintRuleEntry { + return { + ruleName: '@typescript-eslint/ban-ts-comment', + plugin: '@typescript-eslint', + severity: 'error', + options: [{ 'ts-expect-error': 'allow-with-description', 'ts-ignore': true, 'ts-nocheck': true, 'ts-check': false }], + sourceRuleId: '', + description: 'TypeScript suppression directives must not be used', + }; +} \ No newline at end of file diff --git a/src/mapper/types.ts b/src/mapper/types.ts new file mode 100644 index 0000000..dd46754 --- /dev/null +++ b/src/mapper/types.ts @@ -0,0 +1,58 @@ +/** + * Types for the RuleProbe-to-ESLint config mapper. + * + * These types define the intermediate representation between + * parsed RuleProbe rules and emitted ESLint config files. + */ + +/** Output format for the generated ESLint config. */ +export type EslintFormat = 'flat' | 'legacy'; + +/** Severity level that maps to ESLint rule config values. */ +export type EslintSeverity = 'error' | 'warn'; + +/** A single ESLint rule configuration entry. */ +export interface EslintRuleEntry { + /** The ESLint rule name, e.g. "no-console" or "@typescript-eslint/no-explicit-any". */ + ruleName: string; + /** The ESLint plugin name if this rule comes from a plugin, e.g. "@typescript-eslint". */ + plugin?: string; + /** Rule-specific options array (everything after the severity). */ + options?: unknown[]; + /** ESLint severity level. */ + severity: EslintSeverity; + /** The original RuleProbe rule that was mapped. */ + sourceRuleId: string; + /** Human-readable description of the original rule. */ + description: string; +} + +/** A rule that cannot be mapped to a specific ESLint rule. */ +export interface UnmappableRule { + /** The original RuleProbe rule that couldn't be mapped. */ + sourceRuleId: string; + /** The original instruction text from the instruction file. */ + sourceText: string; + /** One-line reason why this rule has no ESLint equivalent. */ + reason: string; +} + +/** The complete output of mapping a RuleSet to ESLint config. */ +export interface EslintConfig { + /** All mappable ESLint rule entries. */ + rules: EslintRuleEntry[]; + /** Rules that have no ESLint equivalent. */ + unmappable: UnmappableRule[]; + /** Plugins required by the mapped rules (deduplicated). */ + plugins: string[]; + /** The original instruction file path. */ + sourceFile: string; +} + +/** A mapping function that takes a RuleProbe pattern type and returns ESLint config. */ +export interface RuleMapping { + /** The RuleProbe pattern type(s) this mapping handles. */ + patternTypes: string[]; + /** Map a pattern's details to an ESLint rule entry, or return null if unmappable. */ + map: (pattern: { type: string; target: string; expected: string | boolean; scope: string }) => EslintRuleEntry | null; +} \ No newline at end of file diff --git a/src/mappings/index.ts b/src/mappings/index.ts new file mode 100644 index 0000000..de45eb0 --- /dev/null +++ b/src/mappings/index.ts @@ -0,0 +1,117 @@ +/** + * Bidirectional mapping table: RuleProbe pattern types <-> ESLint rule names. + * + * Single source of truth for the relationship between RuleProbe's internal + * pattern types and ESLint rule names. Consumed forward by the mapper + * (src/mapper/) and in reverse by the extractor (src/extractor/). + * + * Rules marked stylistic have no meaningful prose equivalent and are + * skipped during extraction. + */ + +/** A single entry in the bidirectional mapping table. */ +export interface MappingEntry { + /** RuleProbe pattern type, e.g. "no-any" or "max-file-length". */ + patternType: string; + /** ESLint rule name, e.g. "@typescript-eslint/no-explicit-any". */ + eslintRuleName: string; + /** ESLint plugin, e.g. "@typescript-eslint" or "import". */ + plugin?: string; + /** Default severity when mapping forward. */ + defaultSeverity: 'error' | 'warn'; + /** One-line description of what this rule enforces. */ + description: string; + /** True if this rule is purely stylistic and should be skipped during extraction. */ + isStylistic?: boolean; +} + +/** + * Pattern types with no ESLint equivalent. + * + * Key is the pattern type, value is a one-line reason explaining why. + * Only includes types produced by remaining matchers. + */ +export const UNMAPPABLE_TYPES: Record = {}; + +/** + * The bidirectional mapping table. + * + * Each entry links a RuleProbe pattern type to its ESLint rule equivalent. + * The forward mapper (src/mapper/) uses this for patternType lookup. + * The reverse extractor (src/extractor/) uses this for eslintRuleName lookup. + */ +export const MAPPINGS: MappingEntry[] = [ + // no-any + { patternType: 'no-any', eslintRuleName: '@typescript-eslint/no-explicit-any', plugin: '@typescript-eslint', defaultSeverity: 'error', description: 'The "any" type must not be used' }, + + // no-console + { patternType: 'no-console-log', eslintRuleName: 'no-console', defaultSeverity: 'error', description: 'console.log must not be used in production code' }, + { patternType: 'no-console-extended', eslintRuleName: 'no-console', defaultSeverity: 'error', description: 'Console statements must not be used' }, + + // named-exports + { patternType: 'named-exports-only', eslintRuleName: 'import/no-default-export', plugin: 'import', defaultSeverity: 'error', description: 'Only named exports are allowed, no default exports' }, + + // kebab-case files + { patternType: 'kebab-case', eslintRuleName: 'unicorn/filename-case', plugin: 'unicorn', defaultSeverity: 'error', description: 'File names must use kebab-case' }, + + // max-lines + { patternType: 'max-file-length', eslintRuleName: 'max-lines', defaultSeverity: 'warn', description: 'Files must not exceed the maximum line count' }, + { patternType: 'max-line-length', eslintRuleName: 'max-len', defaultSeverity: 'warn', description: 'Lines must not exceed the maximum length' }, + + // jsdoc + { patternType: 'jsdoc-required', eslintRuleName: 'jsdoc/require-jsdoc', plugin: 'jsdoc', defaultSeverity: 'warn', description: 'Every public function must have a JSDoc comment' }, + + // code style + { patternType: 'no-var', eslintRuleName: 'no-var', defaultSeverity: 'error', description: 'No var declarations (use const or let)' }, + { patternType: 'prefer-const', eslintRuleName: 'prefer-const', defaultSeverity: 'warn', description: 'Prefer const for variables that are never reassigned' }, + { patternType: 'no-else-after-return', eslintRuleName: 'no-else-after-return', defaultSeverity: 'warn', description: 'Do not use else after a return statement' }, + { patternType: 'no-nested-ternary', eslintRuleName: 'no-nested-ternary', defaultSeverity: 'warn', description: 'Nested ternary expressions are not allowed' }, + { patternType: 'no-magic-numbers', eslintRuleName: 'no-magic-numbers', defaultSeverity: 'warn', description: 'Magic numbers must be replaced with named constants' }, + { patternType: 'consistent-semicolons', eslintRuleName: 'semi', defaultSeverity: 'warn', description: 'Enforce consistent semicolon usage', isStylistic: true }, + { patternType: 'quote-style', eslintRuleName: 'quotes', defaultSeverity: 'warn', description: 'Enforce consistent quote style', isStylistic: true }, + + // error handling + { patternType: 'no-empty-catch', eslintRuleName: 'no-empty', defaultSeverity: 'error', description: 'Catch blocks must not be empty' }, + { patternType: 'throw-error-only', eslintRuleName: 'no-throw-literal', defaultSeverity: 'error', description: 'Only Error objects may be thrown' }, + + // type safety + { patternType: 'no-enum', eslintRuleName: '@typescript-eslint/no-enum', plugin: '@typescript-eslint', defaultSeverity: 'warn', description: 'Enums must not be used; prefer union types' }, + { patternType: 'no-type-assertions', eslintRuleName: '@typescript-eslint/consistent-type-assertions', plugin: '@typescript-eslint', defaultSeverity: 'warn', description: 'Type assertions (as casts) must not be used' }, + { patternType: 'no-non-null-assertions', eslintRuleName: '@typescript-eslint/no-non-null-assertion', plugin: '@typescript-eslint', defaultSeverity: 'warn', description: 'Non-null assertions (!) must not be used' }, + { patternType: 'no-implicit-any', eslintRuleName: '@typescript-eslint/no-implicit-any', plugin: '@typescript-eslint', defaultSeverity: 'warn', description: 'No implicit any types' }, + { patternType: 'no-unused-exports', eslintRuleName: 'no-unused-vars', plugin: '@typescript-eslint', defaultSeverity: 'warn', description: 'Exported declarations must be imported by other files' }, + { patternType: 'no-ts-directives', eslintRuleName: '@typescript-eslint/ban-ts-comment', plugin: '@typescript-eslint', defaultSeverity: 'error', description: 'TypeScript suppression directives must not be used' }, + + // function limits + { patternType: 'max-function-length', eslintRuleName: 'max-lines-per-function', defaultSeverity: 'warn', description: 'Functions must not exceed the maximum line count' }, + { patternType: 'max-params', eslintRuleName: 'max-params', defaultSeverity: 'warn', description: 'Functions must not have too many parameters' }, + + // imports + { patternType: 'no-wildcard-exports', eslintRuleName: 'import/no-namespace', plugin: 'import', defaultSeverity: 'warn', description: 'No wildcard imports; use named imports' }, + { patternType: 'no-namespace-imports', eslintRuleName: '@typescript-eslint/consistent-type-imports', plugin: '@typescript-eslint', defaultSeverity: 'warn', description: 'Use import type for type-only imports' }, + { patternType: 'no-path-aliases', eslintRuleName: 'no-restricted-imports', defaultSeverity: 'warn', description: 'Imports must use relative paths, not path aliases' }, + { patternType: 'no-deep-relative-imports', eslintRuleName: 'import/no-relative-parent', plugin: 'import', defaultSeverity: 'warn', description: 'Relative imports must not go too deep' }, + + // comments + { patternType: 'no-todo-comments', eslintRuleName: 'no-warning-comments', defaultSeverity: 'warn', description: 'No TODO/FIXME/HACK/XXX comments in production code' }, + + // naming convention (special: multiple pattern types map to one ESLint rule) + { patternType: 'PascalCase', eslintRuleName: '@typescript-eslint/naming-convention', plugin: '@typescript-eslint', defaultSeverity: 'error', description: 'Use PascalCase for types and interfaces' }, + { patternType: 'camelCase', eslintRuleName: '@typescript-eslint/naming-convention', plugin: '@typescript-eslint', defaultSeverity: 'error', description: 'Use camelCase for variables and functions' }, + { patternType: 'UPPER_CASE', eslintRuleName: '@typescript-eslint/naming-convention', plugin: '@typescript-eslint', defaultSeverity: 'error', description: 'Use UPPER_CASE for constants' }, +]; + +/** Find a mapping entry by RuleProbe pattern type. */ +export function findByPatternType(patternType: string): MappingEntry | undefined { + return MAPPINGS.find((m) => m.patternType === patternType); +} + +/** Find all mapping entries for an ESLint rule name (may return multiple for many-to-one mappings). */ +export function findAllByEslintRuleName(eslintRuleName: string): MappingEntry[] { + return MAPPINGS.filter((m) => m.eslintRuleName === eslintRuleName); +} + +/** Find the first mapping entry for an ESLint rule name. */ +export function findByEslintRuleName(eslintRuleName: string): MappingEntry | undefined { + return MAPPINGS.find((m) => m.eslintRuleName === eslintRuleName); +} \ No newline at end of file diff --git a/src/mappings/prose-templates.ts b/src/mappings/prose-templates.ts new file mode 100644 index 0000000..0cbd13d --- /dev/null +++ b/src/mappings/prose-templates.ts @@ -0,0 +1,311 @@ +/** + * Prose templates for reverse mapping: ESLint rule -> human-readable instruction. + * + * Each template function takes an ESLint rule's options array and returns a + * one-line prose instruction suitable for a CLAUDE.md rules section. Rules + * with config args surface those args in the prose (e.g. "Files must not + * exceed 300 lines"). + * + * Stylistic rules (semi, quotes) have no meaningful prose equivalent and + * are skipped during extraction. + */ + +/** + * Prose template lookup: ESLint rule name -> template function. + * + * Template functions receive the rule's options array and return a prose string. + * Rules without meaningful config args can ignore the options parameter. + */ +const PROSE_TEMPLATES: ReadonlyMap string> = new Map([ + // no-any + ['@typescript-eslint/no-explicit-any', (_options: unknown[]) => 'Never use the `any` type; narrow with schema checks or type guards at boundaries.'], + + // no-console + ['no-console', (options: unknown[]) => { + const allow = extractAllowList(options); + if (allow.length === 0) { + return 'No `console` statements in production code.'; + } + return `No \`console\` statements except ${allow.map((m) => `\`${m}\``).join(', ')}.`; + }], + + // named-exports + ['import/no-default-export', () => 'Use named exports only; no default exports.'], + + // kebab-case files + ['unicorn/filename-case', (options: unknown[]) => { + const cases = extractFilenameCases(options); + if (cases.length > 0) { + return `File names must use ${cases.join(' or ')} naming.`; + } + return 'File names must use kebab-case.'; + }], + + // max-lines + ['max-lines', (options: unknown[]) => { + const max = extractNumericOption(options, 'max', 300); + return `Files must not exceed ${max} lines.`; + }], + + // max-len + ['max-len', (options: unknown[]) => { + const code = extractNumericOption(options, 'code', 120); + return `Lines must not exceed ${code} characters.`; + }], + + // jsdoc + ['jsdoc/require-jsdoc', (_options: unknown[]) => 'Every exported function must have a JSDoc comment.'], + + // code style + ['no-var', (_options: unknown[]) => 'Use `const` or `let`; never `var`.'], + ['prefer-const', (_options: unknown[]) => 'Prefer `const` for variables that are never reassigned.'], + ['no-else-after-return', (_options: unknown[]) => 'Do not use `else` after a `return`.'], + ['no-nested-ternary', (_options: unknown[]) => 'No nested ternary expressions.'], + ['no-magic-numbers', (_options: unknown[]) => 'Magic numbers must be replaced with named constants.'], + + // stylistic (no meaningful prose) + ['semi', (_options: unknown[]) => 'Enforce consistent semicolon usage.'], + ['quotes', (_options: unknown[]) => 'Enforce consistent quote style.'], + + // error handling + ['no-empty', (options: unknown[]) => { + const allowEmptyCatch = extractAllowEmptyCatch(options); + if (allowEmptyCatch === false) { + return 'Catch blocks must not be empty.'; + } + return 'Empty blocks are not allowed.'; + }], + ['no-throw-literal', (_options: unknown[]) => 'Only `Error` objects may be thrown.'], + + // type safety + ['@typescript-eslint/no-enum', (_options: unknown[]) => 'Do not use enums; prefer union types.'], + ['@typescript-eslint/consistent-type-assertions', (_options: unknown[]) => 'Do not use type assertions (`as` casts).'], + ['@typescript-eslint/no-non-null-assertion', (_options: unknown[]) => 'Do not use non-null assertions (`!`).'], + ['@typescript-eslint/no-implicit-any', (_options: unknown[]) => 'No implicit `any` types.'], + ['no-unused-vars', (_options: unknown[]) => 'Exported declarations must be used by other files.'], + ['@typescript-eslint/ban-ts-comment', (_options: unknown[]) => 'Do not use TypeScript suppression directives (`@ts-expect-error`, etc.).'], + + // function limits + ['max-lines-per-function', (options: unknown[]) => { + const max = extractNumericOption(options, 'max', 50); + return `Functions must not exceed ${max} lines.`; + }], + ['max-params', (options: unknown[]) => { + const max = extractNumericParam(options, 4); + return `Functions must not have more than ${max} parameters.`; + }], + + // imports + ['import/no-namespace', (_options: unknown[]) => 'No wildcard imports; use named imports.'], + ['@typescript-eslint/consistent-type-imports', (_options: unknown[]) => 'Use `import type` for type-only imports.'], + ['no-restricted-imports', (_options: unknown[]) => 'Imports must use relative paths, not path aliases.'], + ['import/no-relative-parent', (_options: unknown[]) => 'Relative imports must not traverse too many parent directories.'], + + // comments + ['no-warning-comments', (options: unknown[]) => { + const terms = extractWarningCommentTerms(options); + if (terms.length > 0) { + return `No ${terms.map((t) => `\`${t}\``).join(', ')} comments in production code.`; + } + return 'No TODO/FIXME/HACK/XXX comments in production code.'; + }], + + // naming convention (special: complex options) + ['@typescript-eslint/naming-convention', (options: unknown[]) => { + return formatNamingConventionProse(options); + }], +]); + +/** Whether a rule is purely stylistic and should be skipped during extraction. */ +const STYLISTIC_RULES: ReadonlySet = new Set([ + 'semi', + 'quotes', +]); + +/** + * Get the prose instruction for an ESLint rule. + * + * @param ruleName - The ESLint rule name + * @param options - The rule's options array from the config + * @returns The prose instruction string, or null if the rule is stylistic + * and should be skipped + */ +export function getProseForRule(ruleName: string, options: unknown[]): string | null { + if (STYLISTIC_RULES.has(ruleName)) { + return null; + } + + const template = PROSE_TEMPLATES.get(ruleName); + if (template) { + return template(options); + } + + return null; +} + +/** + * Check whether an ESLint rule is purely stylistic. + * + * Stylistic rules have no meaningful prose equivalent and should + * be skipped during extraction. + */ +export function isStylisticRule(ruleName: string): boolean { + return STYLISTIC_RULES.has(ruleName); +} + +// ── Option extraction helpers ── + +/** Extract a numeric option value from an ESLint rule's options. */ +function extractNumericOption(options: unknown[], key: string, fallback: number): number { + if (options.length === 0) return fallback; + const obj = options[0]; + if (obj && typeof obj === 'object' && key in (obj as Record)) { + const val = (obj as Record)[key]; + return typeof val === 'number' ? val : fallback; + } + return fallback; +} + +/** Extract a numeric param from options (for rules like max-params where options is [number]). */ +function extractNumericParam(options: unknown[], fallback: number): number { + if (options.length === 0) return fallback; + const val = options[0]; + return typeof val === 'number' ? val : fallback; +} + +/** Extract the allow list from no-console options. */ +function extractAllowList(options: unknown[]): string[] { + if (options.length === 0) return []; + const obj = options[0]; + if (obj && typeof obj === 'object' && 'allow' in (obj as Record)) { + const allow = (obj as Record)['allow']; + if (Array.isArray(allow)) { + return allow.map(String); + } + } + return []; +} + +/** Extract filename cases from unicorn/filename-case options. */ +function extractFilenameCases(options: unknown[]): string[] { + if (options.length === 0) return []; + const obj = options[0]; + if (obj && typeof obj === 'object' && 'cases' in (obj as Record)) { + const cases = (obj as Record)['cases']; + if (typeof cases === 'object' && cases !== null) { + return Object.entries(cases as Record) + .filter(([, v]) => v === true) + .map(([k]) => k); + } + } + return []; +} + +/** Extract allowEmptyCatch from no-empty options. */ +function extractAllowEmptyCatch(options: unknown[]): boolean | undefined { + if (options.length === 0) return undefined; + const obj = options[0]; + if (obj && typeof obj === 'object' && 'allowEmptyCatch' in (obj as Record)) { + return Boolean((obj as Record)['allowEmptyCatch']); + } + return undefined; +} + +/** Extract terms from no-warning-comments options. */ +function extractWarningCommentTerms(options: unknown[]): string[] { + if (options.length === 0) return []; + const obj = options[0]; + if (obj && typeof obj === 'object' && 'terms' in (obj as Record)) { + const terms = (obj as Record)['terms']; + if (Array.isArray(terms)) { + return terms.map(String); + } + } + return []; +} + +/** + * Format naming-convention prose from ESLint options. + * + * Parses the selector/format rules and produces natural-language + * instructions like "Use PascalCase for types and interfaces. + * Use camelCase for variables and functions." + */ +function formatNamingConventionProse(options: unknown[]): string { + if (options.length === 0) return 'Enforce naming conventions for TypeScript identifiers.'; + + const obj = options[0]; + if (!obj || typeof obj !== 'object' || !('rules' in (obj as Record))) { + return 'Enforce naming conventions for TypeScript identifiers.'; + } + + const rules = (obj as Record)['rules']; + if (!Array.isArray(rules)) { + return 'Enforce naming conventions for TypeScript identifiers.'; + } + + const selectorFormats = new Map>(); + + for (const rule of rules) { + if (typeof rule !== 'object' || rule === null) continue; + const r = rule as Record; + const selector = String(r['selector'] ?? 'default'); + const format = r['format']; + if (Array.isArray(format)) { + const existing = selectorFormats.get(selector) ?? new Set(); + for (const f of format) { + existing.add(String(f)); + } + selectorFormats.set(selector, existing); + } + } + + const parts: string[] = []; + + const pascalSelectors: string[] = []; + const camelSelectors: string[] = []; + const upperSelectors: string[] = []; + + for (const [selector, formats] of selectorFormats) { + if (formats.has('PascalCase')) pascalSelectors.push(selector); + if (formats.has('camelCase')) camelSelectors.push(selector); + if (formats.has('UPPER_CASE')) upperSelectors.push(selector); + } + + if (pascalSelectors.length > 0) { + const names = humanizeSelectors(pascalSelectors); + parts.push(`Use PascalCase for ${names}.`); + } + if (camelSelectors.length > 0) { + const names = humanizeSelectors(camelSelectors); + parts.push(`Use camelCase for ${names}.`); + } + if (upperSelectors.length > 0) { + const names = humanizeSelectors(upperSelectors); + parts.push(`Use UPPER_CASE for ${names}.`); + } + + return parts.length > 0 ? parts.join(' ') : 'Enforce naming conventions for TypeScript identifiers.'; +} + +/** Convert ESLint selector names to human-readable form. */ +function humanizeSelectors(selectors: string[]): string { + const readable: Record = { + 'class': 'classes', + 'interface': 'interfaces', + 'typeAlias': 'type aliases', + 'enum': 'enums', + 'enumMember': 'enum members', + 'variable': 'variables', + 'function': 'functions', + 'parameter': 'parameters', + 'classMethod': 'class methods', + 'classProperty': 'class properties', + 'objectLiteralProperty': 'object properties', + 'typeProperty': 'type properties', + 'default': 'identifiers', + }; + return selectors + .map((s) => readable[s] ?? s) + .join(', '); +} \ No newline at end of file diff --git a/src/parsers/legacy-rule-extractor.ts b/src/parsers/legacy-rule-extractor.ts index c392acb..4e2e50b 100644 --- a/src/parsers/legacy-rule-extractor.ts +++ b/src/parsers/legacy-rule-extractor.ts @@ -12,23 +12,15 @@ import { RULE_MATCHERS } from './rule-patterns.js'; import { EXTENDED_RULE_MATCHERS } from './rule-patterns-extended.js'; import { PROJECT_RULE_MATCHERS } from './rule-patterns-project.js'; import { ADVANCED_RULE_MATCHERS } from './rule-patterns-advanced.js'; -import { PREFERENCE_MATCHERS } from './rule-patterns-preference.js'; -import { FILE_STRUCTURE_MATCHERS } from './rule-patterns-file-structure.js'; -import { TOOLING_MATCHERS } from './rule-patterns-tooling.js'; -import { TESTING_MATCHERS } from './rule-patterns-testing.js'; import { detectQualifier } from './qualifier-detector.js'; import { INSTRUCTION_PATTERNS } from './instruction-patterns.js'; -/** Combined matcher list: base, extended, project, advanced, preference, file-structure, tooling, testing. */ +/** Combined matcher list: base, extended, project, advanced. */ const ALL_MATCHERS = [ ...RULE_MATCHERS, ...EXTENDED_RULE_MATCHERS, ...PROJECT_RULE_MATCHERS, ...ADVANCED_RULE_MATCHERS, - ...PREFERENCE_MATCHERS, - ...FILE_STRUCTURE_MATCHERS, - ...TOOLING_MATCHERS, - ...TESTING_MATCHERS, ]; /** Counter for generating unique rule IDs across extraction runs. */ diff --git a/src/parsers/rule-assembler-helpers.ts b/src/parsers/rule-assembler-helpers.ts index c986e30..d55e906 100644 --- a/src/parsers/rule-assembler-helpers.ts +++ b/src/parsers/rule-assembler-helpers.ts @@ -2,113 +2,35 @@ * Rule assembler helper functions and constants. * * Extracted from rule-assembler.ts for the 300-line file limit. - * Contains: category mapping, ID prefix generation, pattern building, - * text formatting, and deduplication. + * Contains: category mapping, text formatting, and deduplication. */ -import type { Rule, RuleCategory, VerifierType, VerificationPattern } from '../types.js'; -import type { StatementCategory, ClassifiedStatement } from './pipeline-types.js'; +import type { Rule } from '../types.js'; +import type { StatementCategory } from './pipeline-types.js'; /** * Map statement categories to rule categories and verifier types. - * Categories not in this map are not directly verifiable. + * All categories are null because generic classification without a + * concrete matcher implementation produces false-passing rules. + * Only statements that match a specific deterministic matcher + * produce verifiable rules. Everything else goes to unparseable. */ -export const CATEGORY_MAP: Record = { - IMPERATIVE_DIRECT: { - ruleCategory: 'code-style', - verifier: 'regex', - severity: 'error', - }, - IMPERATIVE_QUALIFIED: { - ruleCategory: 'code-style', - verifier: 'regex', - severity: 'warning', - }, - PREFER_PATTERN: { - ruleCategory: 'preference', - verifier: 'preference', - severity: 'warning', - }, - TOOLING_COMMAND: { - ruleCategory: 'tooling', - verifier: 'tooling', - severity: 'warning', - }, - FILE_STRUCTURE: { - ruleCategory: 'file-structure', - verifier: 'filesystem', - severity: 'warning', - }, - NAMING_CONVENTION: { - ruleCategory: 'naming', - verifier: 'ast', - severity: 'error', - }, - WORKFLOW: { - ruleCategory: 'workflow', - verifier: 'config-file', - severity: 'warning', - }, - CODE_STYLE: { - ruleCategory: 'code-style', - verifier: 'regex', - severity: 'warning', - }, +export const CATEGORY_MAP: Record = { + IMPERATIVE_DIRECT: null, + IMPERATIVE_QUALIFIED: null, + PREFER_PATTERN: null, + TOOLING_COMMAND: null, + FILE_STRUCTURE: null, + NAMING_CONVENTION: null, + WORKFLOW: null, + CODE_STYLE: null, PATTERN_REFERENCE: null, AGENT_BEHAVIOR: null, - LANGUAGE_SPECIFIC: { - ruleCategory: 'code-style', - verifier: 'ast', - severity: 'warning', - }, + LANGUAGE_SPECIFIC: null, CONTEXT_ONLY: null, UNKNOWN: null, }; -/** - * Map a statement category to a rule ID prefix. - * - * @param category - The statement category - * @returns A kebab-case prefix for rule IDs - */ -export function categoryToIdPrefix(category: StatementCategory): string { - const prefixes: Record = { - IMPERATIVE_DIRECT: 'imperative', - IMPERATIVE_QUALIFIED: 'qualified', - PREFER_PATTERN: 'prefer', - TOOLING_COMMAND: 'tooling-cmd', - FILE_STRUCTURE: 'file-struct', - NAMING_CONVENTION: 'naming-conv', - WORKFLOW: 'workflow', - CODE_STYLE: 'code-style', - PATTERN_REFERENCE: 'pattern-ref', - AGENT_BEHAVIOR: 'agent-behavior', - LANGUAGE_SPECIFIC: 'lang-specific', - CONTEXT_ONLY: 'context', - UNKNOWN: 'unknown', - }; - return prefixes[category] ?? 'unknown'; -} - -/** - * Build a generic verification pattern from a classified statement. - * - * @param stmt - The classified statement - * @returns A verification pattern with category-based type - */ -export function buildGenericPattern(stmt: ClassifiedStatement): VerificationPattern { - return { - type: stmt.category.toLowerCase().replace(/_/g, '-'), - target: 'project', - expected: stmt.text, - scope: 'project', - }; -} - /** * Truncate description to a reasonable length. * @@ -165,4 +87,4 @@ export function deduplicateAssembledRules(rules: Rule[]): Rule[] { } return result; -} +} \ No newline at end of file diff --git a/src/parsers/rule-assembler.ts b/src/parsers/rule-assembler.ts index 4d90d70..754ad10 100644 --- a/src/parsers/rule-assembler.ts +++ b/src/parsers/rule-assembler.ts @@ -2,50 +2,31 @@ * Pass 3: Rule assembly. * * Converts classified statements into Rule[] compatible with the v2.0.0 - * pipeline. Rules that are classifiable but not matchable to existing - * matchers (WORKFLOW, CODE_STYLE, PATTERN_REFERENCE) are still extracted - * with appropriate category and verifier. Deterministic matchers skip them. - * - * Also attempts to match against the existing 82 matchers for backwards - * compatibility: if a statement matches a specific matcher, prefer that - * over the generic classification. + * pipeline. Only statements that match a concrete deterministic matcher + * produce verifiable rules. All other statements go to unparseable, + * preventing false passes from generic rules with no check implementation. */ -import type { Rule, RuleCategory, VerifierType } from '../types.js'; +import type { Rule } from '../types.js'; import type { ClassifiedStatement } from './pipeline-types.js'; import { detectQualifier } from './qualifier-detector.js'; import { RULE_MATCHERS } from './rule-patterns.js'; import { EXTENDED_RULE_MATCHERS } from './rule-patterns-extended.js'; import { PROJECT_RULE_MATCHERS } from './rule-patterns-project.js'; import { ADVANCED_RULE_MATCHERS } from './rule-patterns-advanced.js'; -import { PREFERENCE_MATCHERS } from './rule-patterns-preference.js'; -import { FILE_STRUCTURE_MATCHERS } from './rule-patterns-file-structure.js'; -import { TOOLING_MATCHERS } from './rule-patterns-tooling.js'; -import { TESTING_MATCHERS } from './rule-patterns-testing.js'; -import { CONFIG_FILE_MATCHERS } from './rule-patterns-config-file.js'; -import { GIT_HISTORY_MATCHERS } from './rule-patterns-git-history.js'; import type { RuleMatcher } from '../types.js'; import { - CATEGORY_MAP, - categoryToIdPrefix, - buildGenericPattern, truncateDescription, stripFormatting, deduplicateAssembledRules, } from './rule-assembler-helpers.js'; -/** Combined matcher list from the existing v2.0.0 pipeline. */ +/** Combined matcher list for the rule assembly pipeline. */ const ALL_MATCHERS: RuleMatcher[] = [ ...RULE_MATCHERS, ...EXTENDED_RULE_MATCHERS, ...PROJECT_RULE_MATCHERS, ...ADVANCED_RULE_MATCHERS, - ...PREFERENCE_MATCHERS, - ...FILE_STRUCTURE_MATCHERS, - ...TOOLING_MATCHERS, - ...TESTING_MATCHERS, - ...CONFIG_FILE_MATCHERS, - ...GIT_HISTORY_MATCHERS, ]; /** Counter for generating unique rule IDs. */ @@ -114,22 +95,9 @@ export function assembleRules(statements: ClassifiedStatement[]): { continue; } - // No existing matcher: build a generic rule from classification - const mapping = CATEGORY_MAP[stmt.category]; - if (!mapping) { - // Non-verifiable category (WORKFLOW, PATTERN_REFERENCE) - // Still extract as a rule but mark as not verifiable - const rule = buildGenericRule(stmt); - if (rule) { - rules.push(rule); - } else { - unparseable.push(stmt.text); - } - continue; - } - - const rule = buildClassifiedRule(stmt, mapping); - rules.push(rule); + // No existing matcher: classification-only, not deterministically verifiable. + // Send to unparseable rather than creating a false-passing generic rule. + unparseable.push(stmt.text); } return { @@ -184,69 +152,4 @@ function tryMatchExisting( return matched; } -/** - * Build a rule from a classified statement that matched no existing matcher. - */ -function buildClassifiedRule( - stmt: ClassifiedStatement, - mapping: { ruleCategory: RuleCategory; verifier: VerifierType; severity: 'error' | 'warning' }, -): Rule { - assemblerCounter++; - const qualifier = detectQualifier(stmt.text); - const confidenceLevel = stmt.confidence >= 0.9 ? 'high' - : stmt.confidence >= 0.7 ? 'medium' - : 'low'; - - return { - id: `${categoryToIdPrefix(stmt.category)}-${assemblerCounter}`, - category: mapping.ruleCategory, - source: stmt.text, - description: truncateDescription(stmt.text), - severity: mapping.severity, - verifier: mapping.verifier, - pattern: buildGenericPattern(stmt), - confidence: confidenceLevel, - extractionMethod: 'static', - section: stmt.sectionHeader || undefined, - qualifier, - }; -} - -/** - * Build a rule for non-verifiable categories (AGENT_BEHAVIOR, PATTERN_REFERENCE). - * These are extracted but flagged as not currently verifiable by code tools. - */ -function buildGenericRule(stmt: ClassifiedStatement): Rule | null { - assemblerCounter++; - const qualifier = detectQualifier(stmt.text); - - // Map non-verifiable categories to the closest rule category - const categoryMap: Record = { - AGENT_BEHAVIOR: 'agent-behavior', - PATTERN_REFERENCE: 'code-style', - }; - const ruleCategory = categoryMap[stmt.category]; - if (!ruleCategory) { - return null; - } - - return { - id: `${categoryToIdPrefix(stmt.category)}-${assemblerCounter}`, - category: ruleCategory, - source: stmt.text, - description: truncateDescription(stmt.text), - severity: 'warning', - verifier: 'regex', - pattern: { - type: stmt.category.toLowerCase().replace(/_/g, '-'), - target: 'project', - expected: stmt.text, - scope: 'project', - }, - confidence: stmt.confidence >= 0.7 ? 'medium' : 'low', - extractionMethod: 'static', - section: stmt.sectionHeader || undefined, - qualifier, - }; -} diff --git a/src/parsers/rule-extractor.ts b/src/parsers/rule-extractor.ts index c392acb..4e2e50b 100644 --- a/src/parsers/rule-extractor.ts +++ b/src/parsers/rule-extractor.ts @@ -12,23 +12,15 @@ import { RULE_MATCHERS } from './rule-patterns.js'; import { EXTENDED_RULE_MATCHERS } from './rule-patterns-extended.js'; import { PROJECT_RULE_MATCHERS } from './rule-patterns-project.js'; import { ADVANCED_RULE_MATCHERS } from './rule-patterns-advanced.js'; -import { PREFERENCE_MATCHERS } from './rule-patterns-preference.js'; -import { FILE_STRUCTURE_MATCHERS } from './rule-patterns-file-structure.js'; -import { TOOLING_MATCHERS } from './rule-patterns-tooling.js'; -import { TESTING_MATCHERS } from './rule-patterns-testing.js'; import { detectQualifier } from './qualifier-detector.js'; import { INSTRUCTION_PATTERNS } from './instruction-patterns.js'; -/** Combined matcher list: base, extended, project, advanced, preference, file-structure, tooling, testing. */ +/** Combined matcher list: base, extended, project, advanced. */ const ALL_MATCHERS = [ ...RULE_MATCHERS, ...EXTENDED_RULE_MATCHERS, ...PROJECT_RULE_MATCHERS, ...ADVANCED_RULE_MATCHERS, - ...PREFERENCE_MATCHERS, - ...FILE_STRUCTURE_MATCHERS, - ...TOOLING_MATCHERS, - ...TESTING_MATCHERS, ]; /** Counter for generating unique rule IDs across extraction runs. */ diff --git a/src/parsers/rule-patterns-advanced.ts b/src/parsers/rule-patterns-advanced.ts index 04ca433..0d3dda1 100644 --- a/src/parsers/rule-patterns-advanced.ts +++ b/src/parsers/rule-patterns-advanced.ts @@ -1,16 +1,14 @@ /** - * Advanced rule matchers: type-aware and tree-sitter checks. + * Advanced rule matchers: type-aware AST checks. * - * Contains matchers that require --project (type-aware AST checks) - * or tree-sitter (Python/Go checks). Merged with other matcher - * arrays in rule-extractor.ts. + * Contains matchers that require --project for type-aware analysis. + * Merged with other matcher arrays in rule-extractor.ts. */ import type { RuleMatcher } from '../types.js'; /** - * Matchers for type-aware TypeScript checks and tree-sitter - * language checks (Python, Go). + * Matchers for type-aware TypeScript checks. */ export const ADVANCED_RULE_MATCHERS: RuleMatcher[] = [ // Type-aware checks (require --project flag) @@ -48,143 +46,6 @@ export const ADVANCED_RULE_MATCHERS: RuleMatcher[] = [ type: 'no-unused-exports', target: '*.ts', expected: false, scope: 'project', }), }, - { - id: 'import-no-unresolved', - patterns: [ - /\bno\s+unresolved\s+imports?\b/i, - /\bimports?\s+must\s+(?:be\s+)?resolvable\b/i, - /\bno\s+broken\s+imports?\b/i, - /\ball\s+imports?\s+must\s+resolve\b/i, - ], - category: 'import-pattern', - verifier: 'ast', - description: 'All relative imports must resolve to existing files (requires --project)', - severity: 'error', - buildPattern: () => ({ - type: 'no-unresolved-imports', target: '*.ts', expected: false, scope: 'project', - }), - }, - - // Tree-sitter checks (Python/Go) - { - id: 'naming-python-snake-case', - patterns: [ - /\bpython\b.*\bsnake[_\s]*case\b/i, - /\bsnake[_\s]*case\b.*\bpython\b/i, - /\bpython\s+function\s+names?\b.*\bsnake/i, - ], - category: 'naming', - verifier: 'treesitter', - description: 'Python functions must use snake_case naming', - severity: 'error', - buildPattern: () => ({ - type: 'python-snake-case', target: 'python', expected: 'snake_case', scope: 'file', - }), - }, - { - id: 'naming-python-class', - patterns: [ - /\bpython\b.*\bclass\b.*\bPascal\s*Case\b/i, - /\bPascal\s*Case\b.*\bpython\b.*\bclass/i, - /\bpython\s+class\s+names?\b.*\bPascal/i, - ], - category: 'naming', - verifier: 'treesitter', - description: 'Python classes must use PascalCase naming', - severity: 'error', - buildPattern: () => ({ - type: 'python-class-naming', target: 'python', expected: 'PascalCase', scope: 'file', - }), - }, - { - id: 'naming-go-conventions', - patterns: [ - /\bgo\b.*\bnaming\s+conventions?\b/i, - /\bgo\b.*\bPascalCase\b.*\bexported\b/i, - /\bgo\b.*\bcamelCase\b.*\bunexported\b/i, - /\bgo\s+(?:function|method)\s+naming\b/i, - ], - category: 'naming', - verifier: 'treesitter', - description: 'Go functions follow naming conventions (exported: PascalCase, unexported: camelCase)', - severity: 'error', - buildPattern: () => ({ - type: 'go-naming', target: 'go', expected: 'conventions', scope: 'file', - }), - }, - { - id: 'style-python-function-length', - patterns: [ - /\bpython\b.*\bmax(?:imum)?\s+(?:function|method)\s+length\b/i, - /\bpython\b.*\bfunction\b.*\b(?:under|less\s+than|max|<=?)\s*\d+\s*lines?\b/i, - ], - category: 'code-style', - verifier: 'treesitter', - description: 'Python functions must not exceed maximum line count', - severity: 'warning', - buildPattern: (_line: string, match: RegExpMatchArray) => { - const numMatch = match[0].match(/\d+/); - const maxLines = numMatch ? numMatch[0] : '50'; - return { - type: 'function-length', target: 'python', expected: maxLines, scope: 'file', - }; - }, - }, - { - id: 'style-go-function-length', - patterns: [ - /\bgo\b.*\bmax(?:imum)?\s+(?:function|method)\s+length\b/i, - /\bgo\b.*\bfunction\b.*\b(?:under|less\s+than|max|<=?)\s*\d+\s*lines?\b/i, - ], - category: 'code-style', - verifier: 'treesitter', - description: 'Go functions must not exceed maximum line count', - severity: 'warning', - buildPattern: (_line: string, match: RegExpMatchArray) => { - const numMatch = match[0].match(/\d+/); - const maxLines = numMatch ? numMatch[0] : '50'; - return { - type: 'function-length', target: 'go', expected: maxLines, scope: 'file', - }; - }, - }, - - { - id: 'style-concise-conditionals', - patterns: [ - /\bavoid\b.*\b(?:unnecessary|unneeded)\s+(?:curly\s+)?braces?\b/i, - /\b(?:unnecessary|unneeded)\s+(?:curly\s+)?braces?\b.*\bavoid\b/i, - /\bconcise\s+(?:syntax|conditional|style)\b.*\b(?:if|conditional|brace)/i, - /\bno\s+(?:curly\s+)?braces?\b.*\bsingle\b.*\bstatement/i, - /\bsingle[\s-]line\b.*\bno\s+(?:curly\s+)?braces?\b/i, - ], - category: 'code-style', - verifier: 'ast', - description: 'Avoid unnecessary braces around single-statement bodies in conditionals', - severity: 'warning', - buildPattern: () => ({ - type: 'concise-conditionals', target: '*.ts', expected: false, scope: 'file', - }), - }, - - // Filesystem checks - { - id: 'naming-kebab-case-directories', - patterns: [ - /\bkebab[\s-]*case\b.*\b(?:director(?:y|ies)|folder)/i, - /\b(?:director(?:y|ies)|folder)\b.*\bkebab[\s-]*case\b/i, - /\blowercase\s+with\s+dashes?\b.*\b(?:director(?:y|ies)|folder)/i, - /\b(?:director(?:y|ies)|folder)\b.*\blowercase\s+with\s+dashes?\b/i, - /\b(?:director(?:y|ies)|folder)\s+names?:?\s*kebab/i, - ], - category: 'naming', - verifier: 'filesystem', - description: 'Directory names must use kebab-case (lowercase with dashes)', - severity: 'error', - buildPattern: () => ({ - type: 'kebab-case-directories', target: 'directories', expected: 'kebab-case', scope: 'project', - }), - }, // Additional regex checks { @@ -278,4 +139,4 @@ export const ADVANCED_RULE_MATCHERS: RuleMatcher[] = [ type: 'no-wildcard-exports', target: '*.ts', expected: false, scope: 'file', }), }, -]; +]; \ No newline at end of file diff --git a/src/parsers/rule-patterns-config-file.ts b/src/parsers/rule-patterns-config-file.ts deleted file mode 100644 index 2d5b1db..0000000 --- a/src/parsers/rule-patterns-config-file.ts +++ /dev/null @@ -1,299 +0,0 @@ -/** - * Config-file rule matchers. - * - * Matches instructions about CI/CD, git hooks, npm scripts, - * and environment tool requirements. - */ - -import type { RuleMatcher } from '../types.js'; - -/** Matchers for rules verifiable via config file inspection. */ -export const CONFIG_FILE_MATCHERS: RuleMatcher[] = [ - { - id: 'config-ci-lint', - patterns: [ - /\bCI\b.*\b(?:run|execut)\w*\s+(?:lint|eslint|biome)/i, - /\b(?:lint|eslint|biome)\b.*\b(?:in\s+)?CI\b/i, - /\bpipeline\b.*\b(?:lint|eslint)/i, - /\bgithub\s+actions?\b.*\b(?:lint)/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'CI pipeline must run lint checks', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'ci-command-present', - target: 'lint', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-ci-test', - patterns: [ - /\bCI\b.*\b(?:run|execut)\w*\s+(?:test|spec)/i, - /\btests?\b.*\b(?:in\s+)?CI\b/i, - /\bpipeline\b.*\btests?\b/i, - /\bgithub\s+actions?\b.*\b(?:test)/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'CI pipeline must run tests', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'ci-command-present', - target: 'test', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-ci-typecheck', - patterns: [ - /\bCI\b.*\b(?:run|execut)\w*\s+(?:type\s*check|tsc)/i, - /\btype\s*check\b.*\b(?:in\s+)?CI\b/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'CI pipeline must run type checking', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'ci-command-present', - target: 'tsc', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-ci-present', - patterns: [ - /\bset\s+up\s+CI\b/i, - /\bconfigure\s+CI\b/i, - /\b(?:must|should)\s+(?:have|include)\s+CI\b/i, - /\bgithub\s+actions?\s+workflow/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'Project must have CI configuration', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'ci-config-present', - target: 'ci', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-pre-commit-test', - patterns: [ - /\brun\s+(?:npm\s+)?test\s+(?:before|prior\s+to)\s+commit/i, - /\bpre[- ]commit\b.*\btest/i, - /\btest\b.*\bpre[- ]commit\b/i, - /\bbefore\s+commit(?:ting)?\b.*\brun\s+(?:the\s+)?tests?\b/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'Pre-commit hook must run tests', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'pre-commit-check', - target: 'test', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-pre-commit-lint', - patterns: [ - /\brun\s+(?:npm\s+)?lint\s+(?:before|prior\s+to)\s+commit/i, - /\bpre[- ]commit\b.*\blint/i, - /\blint\b.*\bpre[- ]commit\b/i, - /\bbefore\s+commit(?:ting)?\b.*\brun\s+(?:the\s+)?lint/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'Pre-commit hook must run linting', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'pre-commit-check', - target: 'lint', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-husky', - patterns: [ - /\buse\s+husky\b/i, - /\bhusky\b.*\bgit\s+hooks?\b/i, - /\bgit\s+hooks?\b.*\bhusky\b/i, - /\bset\s+up\s+husky\b/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'Project must use husky for git hooks', - severity: 'warning', - confidence: 'high', - buildPattern: () => ({ - type: 'git-hook-present', - target: 'pre-commit', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-lefthook', - patterns: [ - /\buse\s+lefthook\b/i, - /\blefthook\b.*\bgit\s+hooks?\b/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'Project must use lefthook for git hooks', - severity: 'warning', - confidence: 'high', - buildPattern: () => ({ - type: 'git-hook-present', - target: 'pre-commit', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-script-test', - patterns: [ - /\bpackage\.json\b.*\btest\s+script\b/i, - /\b(?:add|include|define)\s+(?:a\s+)?(?:npm\s+)?test\s+script\b/i, - /\b(?:npm|yarn|pnpm)\s+test\b.*\bmust\s+(?:be\s+)?(?:defined|present|configured)\b/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'package.json must have a test script', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'script-present', - target: 'test', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-script-lint', - patterns: [ - /\bpackage\.json\b.*\blint\s+script\b/i, - /\b(?:add|include|define)\s+(?:a\s+)?(?:npm\s+)?lint\s+script\b/i, - ], - category: 'workflow', - verifier: 'config-file', - description: 'package.json must have a lint script', - severity: 'warning', - confidence: 'medium', - buildPattern: () => ({ - type: 'script-present', - target: 'lint', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-env-flox', - patterns: [ - /\buse\s+flox\b/i, - /\bflox\s+(?:environment|dev|shell)\b/i, - /\benvironment\b.*\bflox\b/i, - ], - category: 'tooling', - verifier: 'config-file', - description: 'Project must use flox for environment management', - severity: 'warning', - confidence: 'high', - buildPattern: () => ({ - type: 'env-tool-present', - target: 'flox', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-env-nix', - patterns: [ - /\buse\s+nix\b/i, - /\bnix\s+(?:shell|flake|develop)\b/i, - /\bnix\b.*\benvironment\b/i, - ], - category: 'tooling', - verifier: 'config-file', - description: 'Project must use nix for environment management', - severity: 'warning', - confidence: 'high', - buildPattern: () => ({ - type: 'env-tool-present', - target: 'nix', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-env-devcontainer', - patterns: [ - /\buse\s+devcontainer/i, - /\bdev\s*container\b.*\bconfigure/i, - /\bconfigure\b.*\bdev\s*container/i, - ], - category: 'tooling', - verifier: 'config-file', - description: 'Project must have devcontainer configuration', - severity: 'warning', - confidence: 'high', - buildPattern: () => ({ - type: 'env-tool-present', - target: 'devcontainer', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-env-mise', - patterns: [ - /\buse\s+mise\b/i, - /\bmise\b.*\btool\s+versions?\b/i, - ], - category: 'tooling', - verifier: 'config-file', - description: 'Project must use mise for tool version management', - severity: 'warning', - confidence: 'high', - buildPattern: () => ({ - type: 'env-tool-present', - target: 'mise', - expected: true, - scope: 'project', - }), - }, - { - id: 'config-env-volta', - patterns: [ - /\buse\s+volta\b/i, - /\bvolta\b.*\bpin\b/i, - /\bpin\b.*\bvolta\b/i, - ], - category: 'tooling', - verifier: 'config-file', - description: 'Project must use volta for Node.js version management', - severity: 'warning', - confidence: 'high', - buildPattern: () => ({ - type: 'env-tool-present', - target: 'volta', - expected: true, - scope: 'project', - }), - }, -]; diff --git a/src/parsers/rule-patterns-extended.ts b/src/parsers/rule-patterns-extended.ts index 746ece0..0441d7b 100644 --- a/src/parsers/rule-patterns-extended.ts +++ b/src/parsers/rule-patterns-extended.ts @@ -195,54 +195,4 @@ export const EXTENDED_RULE_MATCHERS: RuleMatcher[] = [ type: 'max-params', target: '*.ts', expected: match[1] ?? '4', scope: 'file', }), }, - { - id: 'error-async-try-catch', - patterns: [ - /\buse\s+try[\s/]*catch\s+(?:blocks?\s+)?(?:for|with|in)\s+async\b/i, - /\basync\s+(?:operations?|functions?|code)\b.*\btry[\s/]*catch\b/i, - /\btry[\s/]*catch\b.*\basync\s+(?:operations?|functions?|code)\b/i, - /\bwrap\s+async\b.*\btry[\s/]*catch\b/i, - /\balways\s+(?:use\s+)?try[\s/]*catch\b/i, - ], - category: 'error-handling', - verifier: 'ast', - description: 'Async operations must use try/catch error handling', - severity: 'warning', - buildPattern: () => ({ - type: 'async-try-catch', target: '*.ts', expected: 'try-catch-present', scope: 'file', - }), - }, - { - id: 'structure-typescript-required', - patterns: [ - /\buse\s+TypeScript\s+for\s+(?:all|every|new)\b/i, - /\ball\s+(?:new\s+)?code\s+(?:in|must\s+be)\s+TypeScript\b/i, - /\bTypeScript\s+for\s+all\b/i, - /\bwrite\s+(?:all\s+)?(?:new\s+)?(?:code\s+)?in\s+TypeScript\b/i, - /\buse\s+TypeScript\b(?!.*\bstrict\s+mode\b)/i, - ], - category: 'type-safety', - verifier: 'tooling', - description: 'Project must use TypeScript', - severity: 'warning', - buildPattern: () => ({ - type: 'typescript-required', target: 'tsconfig.json', expected: true, scope: 'project', - }), - }, - { - id: 'error-log-contextual', - patterns: [ - /\blog\s+errors?\s+with\s+context/i, - /\balways\s+log\s+errors?\b/i, - /\berror\s+(?:messages?\s+)?(?:must\s+)?include\s+context/i, - /\bcontextual\s+(?:error\s+)?(?:log(?:ging)?|information)\b/i, - ], - category: 'error-handling', - verifier: 'ast', - description: 'Errors must be logged with contextual information', - severity: 'warning', - buildPattern: () => ({ - type: 'error-log-context', target: '*.ts', expected: 'contextual-logging', scope: 'file', - }), - }, ]; diff --git a/src/parsers/rule-patterns-file-structure.ts b/src/parsers/rule-patterns-file-structure.ts deleted file mode 100644 index 62ab2cd..0000000 --- a/src/parsers/rule-patterns-file-structure.ts +++ /dev/null @@ -1,124 +0,0 @@ -/** - * File structure rule matchers. - * - * Matches instructions about directory structure, path existence, - * and file organization patterns. Examples: - * - "Tests go in __tests__/" - * - "Components live in src/components/" - * - "Every module needs an index.ts" - */ - -import type { RuleMatcher } from '../types.js'; - -/** - * Matchers for file structure rules. - */ -export const FILE_STRUCTURE_MATCHERS: RuleMatcher[] = [ - { - id: 'file-structure-tests-dir', - patterns: [ - /\btests?\s+(?:go|live|belong|placed?|should\s+be)\s+in\s+(\S+)/i, - /\btest\s+files?\s+(?:in|under)\s+(\S+)/i, - /\b(?:put|place|keep)\s+tests?\s+(?:in|under)\s+(\S+)/i, - /\b__tests__\s*\//i, - /\btests?\s+director(?:y|ies)\b/i, - ], - category: 'file-structure', - verifier: 'filesystem', - description: 'Test files must be in the specified directory', - severity: 'warning', - buildPattern: (line, match) => { - const dir = match[1]?.replace(/['"`,]/g, '') ?? '__tests__'; - return { - type: 'directory-exists-with-files', - target: dir, - expected: true, - scope: 'project', - }; - }, - }, - { - id: 'file-structure-components-dir', - patterns: [ - /\bcomponents?\s+(?:go|live|belong|placed?|should\s+be)\s+in\s+(\S+)/i, - /\bcomponents?\s+(?:in|under)\s+src\/components?\b/i, - /\b(?:put|place|keep)\s+components?\s+(?:in|under)\s+(\S+)/i, - ], - category: 'file-structure', - verifier: 'filesystem', - description: 'Components must be in the specified directory', - severity: 'warning', - buildPattern: (line, match) => { - const dir = match[1]?.replace(/['"`,]/g, '') ?? 'src/components'; - return { - type: 'directory-exists-with-files', - target: dir, - expected: true, - scope: 'project', - }; - }, - }, - { - id: 'file-structure-env-file', - patterns: [ - /\buse\s+\.env(?:\.local)?\b/i, - /\b\.env\.local\b.*\bfor\b.*\b(?:local|dev)\b/i, - /\blocal\s+config\b.*\b\.env\b/i, - /\b\.env\b.*\blocal\s+(?:config|settings?|environment)\b/i, - ], - category: 'file-structure', - verifier: 'filesystem', - description: 'Environment file must exist for local configuration', - severity: 'warning', - confidence: 'medium', - buildPattern: (line) => { - const envFile = line.includes('.env.local') ? '.env.local' : '.env'; - return { - type: 'file-pattern-exists', - target: envFile, - expected: true, - scope: 'project', - }; - }, - }, - { - id: 'file-structure-module-index', - patterns: [ - /\bevery\s+module\s+(?:needs?|requires?|must\s+have)\s+(?:an?\s+)?index\.\w+/i, - /\ball\s+modules?\s+(?:should|must)\s+have\s+(?:an?\s+)?index\.\w+/i, - /\bindex\.(?:ts|js)\s+(?:in\s+)?every\s+(?:module|directory|folder)\b/i, - /\beach\s+(?:module|directory|folder)\s+(?:needs?|requires?|must\s+have)\s+index/i, - ], - category: 'file-structure', - verifier: 'filesystem', - description: 'Every module directory must have an index file', - severity: 'warning', - buildPattern: (line) => { - const ext = line.match(/index\.(\w+)/)?.[1] ?? 'ts'; - return { - type: 'module-index-required', - target: `index.${ext}`, - expected: true, - scope: 'project', - }; - }, - }, - { - id: 'file-structure-src-dir', - patterns: [ - /\bsource\s+(?:code|files?)\s+(?:in|under)\s+src\//i, - /\bsrc\/\s+director/i, - /\b(?:put|place|keep)\s+(?:source|code)\s+(?:in|under)\s+src\//i, - ], - category: 'file-structure', - verifier: 'filesystem', - description: 'Source code must be in src/ directory', - severity: 'warning', - buildPattern: () => ({ - type: 'directory-exists-with-files', - target: 'src', - expected: true, - scope: 'project', - }), - }, -]; diff --git a/src/parsers/rule-patterns-git-history.ts b/src/parsers/rule-patterns-git-history.ts deleted file mode 100644 index b13ddac..0000000 --- a/src/parsers/rule-patterns-git-history.ts +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Git history rule matchers. - * - * Matches instructions about commit message formats, branch naming, - * commit signing, and PR conventions. Examples: - * - "Use conventional commits" - * - "Prefix commits with [AI]" - * - "Branch names must follow feature/xxx pattern" - * - "Sign all commits" - */ - -import type { RuleMatcher, VerificationPattern } from '../types.js'; - -/** - * Extract a prefix from instruction text when matching commit prefix rules. - * - * @param text - The matched instruction text - * @param match - The regex match result - * @returns The extracted prefix string - */ -function extractPrefix(text: string, match: RegExpMatchArray): string { - // Try to find a bracketed prefix like [AI], [bot], etc. - const bracketMatch = text.match(/\[([^\]]+)\]/); - if (bracketMatch && bracketMatch[1] !== undefined) { - return `[${bracketMatch[1]}]`; - } - // Try quoted prefix - const quoteMatch = text.match(/["']([^"']+)["']/); - if (quoteMatch && quoteMatch[1] !== undefined) { - return quoteMatch[1]; - } - // Fallback: use the captured group if available - if (match[1] !== undefined) { - return match[1]; - } - return ''; -} - -/** - * Extract a branch pattern from instruction text. - * - * @param text - The matched instruction text - * @returns A regex pattern string for branch validation - */ -function extractBranchPattern(text: string): string { - // Match explicit patterns like "feature/xxx" or "type/description" - const slashPattern = text.match(/\b(feature|bugfix|hotfix|release|fix|chore|refactor|docs)\/\S*/i); - if (slashPattern) { - return `^(feature|bugfix|hotfix|release|fix|chore|refactor|docs)/`; - } - // Generic "type/description" pattern - if (/\btype\s*\/\s*description\b/i.test(text)) { - return `^[a-z]+/[a-z]`; - } - // Default conventional branch pattern - return `^(feature|bugfix|hotfix|release|fix|chore)/`; -} - -/** - * Matchers for rules verifiable via git history inspection. - */ -export const GIT_HISTORY_MATCHERS: RuleMatcher[] = [ - { - id: 'git-conventional-commits', - patterns: [ - /\bconventional\s+commit/i, - /\bcommit\s+(?:message\s+)?(?:format|convention)\b.*\b(?:feat|fix|chore)\b/i, - /\b(?:feat|fix|chore|refactor)\(?\)?:\s/i, - /\bcommit\s+messages?\s+(?:should|must)\s+follow\b.*\bformat\b/i, - ], - category: 'workflow', - verifier: 'git-history', - description: 'Commits must follow conventional commits format', - severity: 'warning', - confidence: 'high', - buildPattern: (): VerificationPattern => ({ - type: 'conventional-commits', - target: 'conventional', - expected: true, - scope: 'project', - }), - }, - { - id: 'git-commit-prefix', - patterns: [ - /\bprefix\s+commit\w*\s+(?:message\w?\s+)?(?:with|by)\s+\[?\w+\]?/i, - /\bcommit\s+message\w?\s+(?:should|must)\s+(?:be\s+)?prefix/i, - /\bcommit\w*\s+(?:should|must)\s+start\s+with\b/i, - /\b\[AI\]\b.*\bcommit/i, - /\bcommit\b.*\b\[AI\]\b/i, - ], - category: 'workflow', - verifier: 'git-history', - description: 'Commit messages must have the specified prefix', - severity: 'warning', - confidence: 'medium', - buildPattern: (text: string, match: RegExpMatchArray): VerificationPattern => ({ - type: 'commit-message-prefix', - target: extractPrefix(text, match), - expected: true, - scope: 'project', - }), - }, - { - id: 'git-branch-naming', - patterns: [ - /\bbranch\s+nam(?:e|ing)\b.*\b(?:pattern|format|convention)\b/i, - /\bbranch(?:es)?\s+(?:should|must)\s+follow\b/i, - /\bbranch\s+nam(?:e|ing)\b.*\b(?:feature|bugfix|hotfix)[\s/]/i, - /\b(?:feature|bugfix|hotfix)\/\b.*\bbranch\b/i, - ], - category: 'workflow', - verifier: 'git-history', - description: 'Branch names must follow the specified pattern', - severity: 'warning', - confidence: 'medium', - buildPattern: (text: string): VerificationPattern => ({ - type: 'branch-naming', - target: extractBranchPattern(text), - expected: true, - scope: 'project', - }), - }, - { - id: 'git-signed-commits', - patterns: [ - /\bsign\s+(?:all\s+)?commit/i, - /\bcommit\s+sign(?:ing|ed)\b/i, - /\bGPG\b.*\bcommit/i, - /\bcommit\b.*\bGPG\b/i, - /\bDCO\b.*\bsign[- ]off/i, - /\bsign[- ]off\b.*\bcommit/i, - ], - category: 'workflow', - verifier: 'git-history', - description: 'Commits must be signed (GPG, SSH, or DCO sign-off)', - severity: 'warning', - confidence: 'medium', - buildPattern: (): VerificationPattern => ({ - type: 'signed-commits', - target: 'signed', - expected: true, - scope: 'project', - }), - }, - { - id: 'git-commit-scope', - patterns: [ - /\bcommit\b.*\bsmall\b/i, - /\bsmall\s+(?:focused\s+)?commit/i, - /\batomic\s+commit/i, - /\bone\s+(?:thing|change)\s+per\s+commit/i, - ], - category: 'workflow', - verifier: 'git-history', - description: 'Commits should be small and focused', - severity: 'warning', - confidence: 'low', - buildPattern: (): VerificationPattern => ({ - type: 'commit-message-pattern', - target: '.{1,72}', - expected: true, - scope: 'project', - }), - }, -]; diff --git a/src/parsers/rule-patterns-preference.ts b/src/parsers/rule-patterns-preference.ts deleted file mode 100644 index d233c4a..0000000 --- a/src/parsers/rule-patterns-preference.ts +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Preference rule matchers. - * - * Matches "prefer X over Y" / "use X instead of Y" patterns - * in instruction files and maps them to prefer-pair verifications. - * Each matcher detects a specific prefer-pair from the PREFER_PAIRS - * definitions. - */ - -import type { RuleMatcher } from '../types.js'; - -/** - * Matchers for preference-based rules. - * - * Each matcher detects a "prefer X over Y" pattern in instruction text - * and maps it to a prefer-pair ID for verification. - */ -export const PREFERENCE_MATCHERS: RuleMatcher[] = [ - { - id: 'preference-const-over-let', - patterns: [ - /\bprefer\s+const\b.*\bover\b.*\blet\b/i, - /\bconst\s+over\s+let\b/i, - /\buse\s+const\b.*\binstead\s+of\b.*\blet\b/i, - /\bfavor\s+const\b.*\bover\b.*\blet\b/i, - /\bprefer\s+const\b(?!.*\bover\b)/i, - /\bimmutable\s+by\s+default\b/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer const over let for variable declarations', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'const-vs-let', - expected: 'const', - scope: 'project', - }), - }, - { - id: 'preference-named-over-default-exports', - patterns: [ - /\bprefer\s+named\s+exports?\b.*\bover\b.*\bdefault\b/i, - /\bnamed\s+exports?\s+over\s+default\b/i, - /\buse\s+named\s+exports?\b.*\binstead\s+of\b.*\bdefault\b/i, - /\bfavor\s+named\s+exports?\b/i, - /\bprefer\s+named\s+exports?\b/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer named exports over default exports', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'named-vs-default-exports', - expected: 'named exports', - scope: 'project', - }), - }, - { - id: 'preference-interface-over-type', - patterns: [ - /\bprefer\s+interface\b.*\bover\b.*\btype\b/i, - /\binterface\s+over\s+type\b/i, - /\buse\s+interface\b.*\binstead\s+of\b.*\btype\b/i, - /\bfavor\s+interface\b.*\b(?:for|over)\b/i, - /\binterface\b.*\bfor\s+object\s+shapes?\b/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer interface over type for object shapes', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'interface-vs-type', - expected: 'interface', - scope: 'project', - }), - }, - { - id: 'preference-async-await-over-then', - patterns: [ - /\bprefer\s+async\s*\/?\s*await\b.*\bover\b.*\.?then\b/i, - /\basync\s*\/?\s*await\s+over\b.*\.?then\b/i, - /\buse\s+async\s*\/?\s*await\b.*\binstead\s+of\b.*\.?then\b/i, - /\bprefer\s+async\s*\/?\s*await\b/i, - /\bfavor\s+async\s*\/?\s*await\b/i, - /\basync\b.*\binstead\s+of\b.*\bthen\b.*\bchain/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer async/await over .then() chains', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'async-await-vs-then', - expected: 'async/await', - scope: 'project', - }), - }, - { - id: 'preference-arrow-over-function', - patterns: [ - /\bprefer\s+arrow\s+functions?\b.*\bover\b.*\bfunction\s+declarations?\b/i, - /\barrow\s+functions?\s+over\s+function\s+declarations?\b/i, - /\buse\s+arrow\s+functions?\b.*\binstead\s+of\b.*\bfunction\b/i, - /\bfavor\s+arrow\s+functions?\b/i, - /\bprefer\s+arrow\s+functions?\b/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer arrow functions over function declarations', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'arrow-vs-function-declarations', - expected: 'arrow functions', - scope: 'project', - }), - }, - { - id: 'preference-template-literals', - patterns: [ - /\bprefer\s+template\s+(?:literals?|strings?)\b.*\bover\b.*\bconcatenation\b/i, - /\btemplate\s+(?:literals?|strings?)\s+over\b.*\bconcatenation\b/i, - /\buse\s+template\s+(?:literals?|strings?)\b.*\binstead\s+of\b.*\bconcatenation\b/i, - /\bprefer\s+template\s+(?:literals?|strings?)\b/i, - /\bprefer\s+backticks?\b/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer template literals over string concatenation', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'template-literals-vs-concatenation', - expected: 'template literals', - scope: 'project', - }), - }, - { - id: 'preference-optional-chaining', - patterns: [ - /\bprefer\s+optional\s+chaining\b/i, - /\boptional\s+chaining\s+over\b/i, - /\buse\s+optional\s+chaining\b/i, - /\bfavor\s+optional\s+chaining\b/i, - /\buse\s+\?\.\s/i, - /\boptional\s+chaining\b.*\bnullish\s+coalescing\b/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer optional chaining over nested conditionals', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'optional-chaining-vs-nested-conditionals', - expected: 'optional chaining', - scope: 'project', - }), - }, - { - id: 'preference-functional-components', - patterns: [ - /\bprefer\s+functional\s+components?\b.*\bover\b.*\bclass\b/i, - /\bfunctional\s+components?\s+over\s+class\b/i, - /\buse\s+function(?:al)?\s+components?\b.*\binstead\s+of\b.*\bclass\b/i, - /\bfavor\s+functional\s+components?\b/i, - /\bprefer\s+function\s+components?\b/i, - /\bno\s+class\s+components?\b/i, - /\buse\s+functional\s+components?\b/i, - /\bfunctional\s+components?\s+with\s+hooks\b/i, - ], - category: 'preference', - verifier: 'preference', - description: 'Prefer functional components over class components', - severity: 'warning', - buildPattern: () => ({ - type: 'prefer-pair', - target: 'functional-vs-class-components', - expected: 'functional components', - scope: 'project', - }), - }, -]; diff --git a/src/parsers/rule-patterns-project.ts b/src/parsers/rule-patterns-project.ts index 2e630fb..839c942 100644 --- a/src/parsers/rule-patterns-project.ts +++ b/src/parsers/rule-patterns-project.ts @@ -1,17 +1,14 @@ /** - * Project-level and testing rule pattern dictionary. + * Project-level rule pattern dictionary. * - * Contains matchers for test quality, import restrictions, - * project structure, dependency management, and regex-based - * code quality checks. Merged with other matcher arrays - * in rule-extractor.ts. + * Contains matchers for import restrictions and type safety. + * Merged with other matcher arrays in rule-extractor.ts. */ import type { RuleMatcher } from '../types.js'; /** - * Matchers covering testing patterns, import restrictions, - * project structure, and dependency constraints. + * Matchers covering import restrictions and type safety checks. */ export const PROJECT_RULE_MATCHERS: RuleMatcher[] = [ { @@ -31,38 +28,6 @@ export const PROJECT_RULE_MATCHERS: RuleMatcher[] = [ type: 'no-namespace-imports', target: '*.ts', expected: false, scope: 'file', }), }, - { - id: 'structure-no-barrel-files', - patterns: [ - /\bno\s+barrel\s+(?:files?|re[\s-]?exports?)\b/i, - /\bavoid\s+barrel\s+(?:files?|re[\s-]?exports?)\b/i, - /\bdon'?t\s+use\s+barrel\s+(?:files?|re[\s-]?exports?)\b/i, - /\bno\s+(?:barrel|index)\s+(?:re[\s-]?)?exports?\b/i, - ], - category: 'structure', - verifier: 'ast', - description: 'Barrel files (index.ts with only re-exports) are not allowed', - severity: 'warning', - buildPattern: () => ({ - type: 'no-barrel-files', target: 'index.ts', expected: false, scope: 'file', - }), - }, - { - id: 'test-no-settimeout', - patterns: [ - /\bno\s+setTimeout\s+in\s+tests?\b/i, - /\bavoid\s+setTimeout\s+in\s+tests?\b/i, - /\bdon'?t\s+use\s+setTimeout\s+in\s+tests?\b/i, - /\bno\s+timers?\s+in\s+tests?\b/i, - ], - category: 'test-requirement', - verifier: 'ast', - description: 'setTimeout/setInterval must not be used in test files', - severity: 'warning', - buildPattern: () => ({ - type: 'no-setTimeout-in-tests', target: '*.test.ts', expected: false, scope: 'file', - }), - }, { id: 'type-no-ts-directives', patterns: [ @@ -81,40 +46,6 @@ export const PROJECT_RULE_MATCHERS: RuleMatcher[] = [ type: 'no-ts-directives', target: '*.ts', expected: false, scope: 'file', }), }, - { - id: 'test-no-only', - patterns: [ - /\bno\s+\.only\s*\(\b/i, - /\bno\s+(?:test|describe|it)\.only\b/i, - /\bdon'?t\s+(?:use|leave)\s+\.only\b/i, - /\bavoid\s+\.only\b.*\btests?\b/i, - /\bno\s+focused\s+tests?\b/i, - ], - category: 'test-requirement', - verifier: 'regex', - description: 'Test .only() calls must not be committed', - severity: 'error', - buildPattern: () => ({ - type: 'no-test-only', target: '*.test.ts', expected: false, scope: 'file', - }), - }, - { - id: 'test-no-skip', - patterns: [ - /\bno\s+\.skip\s*\(\b/i, - /\bno\s+(?:test|describe|it)\.skip\b/i, - /\bdon'?t\s+(?:use|leave)\s+\.skip\b/i, - /\bavoid\s+\.skip\b.*\btests?\b/i, - /\bno\s+skipped\s+tests?\b/i, - ], - category: 'test-requirement', - verifier: 'regex', - description: 'Test .skip() calls must not be committed', - severity: 'warning', - buildPattern: () => ({ - type: 'no-test-skip', target: '*.test.ts', expected: false, scope: 'file', - }), - }, { id: 'style-quote-style', patterns: [ @@ -131,87 +62,4 @@ export const PROJECT_RULE_MATCHERS: RuleMatcher[] = [ type: 'quote-style', target: '*.ts', expected: 'single', scope: 'file', }), }, - { - id: 'import-banned-package', - patterns: [ - /\b(?:don'?t|do\s+not|never)\s+(?:import|use)\s+(?:the\s+)?(\w[\w.-]*)\s+(?:package|library|module)\b/i, - /\b(?:ban(?:ned)?|forbid(?:den)?|prohibit(?:ed)?)\s+(?:package|import|library):?\s+(\w[\w.-]*)\b/i, - /\bno\s+(\w[\w.-]*)\s+(?:imports?|dependency|package)\b/i, - ], - category: 'dependency', - verifier: 'regex', - description: 'Banned package must not be imported', - severity: 'error', - buildPattern: (_line: string, match: RegExpMatchArray) => ({ - type: 'banned-import', target: '*.ts', expected: match[1] ?? '', scope: 'file', - }), - }, - { - id: 'structure-readme-exists', - patterns: [ - /\bREADME\b.*\b(?:must|should|required)\b/i, - /\b(?:must|should|required)\b.*\bREADME\b/i, - /\binclude\s+(?:a\s+)?README\b/i, - /\bmaintain\s+(?:a\s+)?README\b/i, - /\bREADME\s+(?:is\s+)?required\b/i, - ], - category: 'structure', - verifier: 'filesystem', - description: 'A README file must exist in the project', - severity: 'warning', - buildPattern: () => ({ - type: 'readme-exists', target: 'README.md', expected: true, scope: 'project', - }), - }, - { - id: 'structure-changelog-exists', - patterns: [ - /\bCHANGELOG\b.*\b(?:must|should|required|exists?)\b/i, - /\b(?:must|should|required)\b.*\bCHANGELOG\b/i, - /\bmaintain\s+(?:a\s+)?CHANGELOG\b/i, - /\bkeep\s+(?:a\s+)?CHANGELOG\b/i, - ], - category: 'structure', - verifier: 'filesystem', - description: 'A CHANGELOG file must exist in the project', - severity: 'warning', - buildPattern: () => ({ - type: 'changelog-exists', target: 'CHANGELOG.md', expected: true, scope: 'project', - }), - }, - { - id: 'structure-formatter-config', - patterns: [ - /\buse\s+(?:a\s+)?(?:prettier|eslint|biome|formatter)\b/i, - /\b(?:prettier|eslint|biome)\s+(?:config|configuration)\s+(?:must|should|required)\b/i, - /\bmust\s+have\s+(?:a\s+)?formatter\s+config/i, - /\bformatter\s+(?:config|configuration)\s+(?:is\s+)?required\b/i, - /\benforce\s+(?:code\s+)?formatting\b/i, - ], - category: 'structure', - verifier: 'filesystem', - description: 'A formatter configuration file must exist', - severity: 'warning', - buildPattern: () => ({ - type: 'formatter-config-exists', target: '.prettierrc', expected: true, scope: 'project', - }), - }, - { - id: 'dependency-pinned-versions', - patterns: [ - /\bpin(?:ned)?\s+dependenc(?:y|ies)\b/i, - /\bexact\s+(?:dependency\s+)?versions?\b/i, - /\bno\s+[\^~]\s+(?:in\s+)?(?:versions?|dependenc(?:y|ies))/i, - /\bpin\s+(?:all\s+)?(?:dependency\s+)?versions?\b/i, - /\bdependenc(?:y|ies)\b.*\bpinned\b/i, - /\bno\s+(?:caret|tilde)\s+(?:in\s+)?(?:versions?|dependenc(?:y|ies))/i, - ], - category: 'dependency', - verifier: 'filesystem', - description: 'Dependencies must use pinned (exact) versions', - severity: 'warning', - buildPattern: () => ({ - type: 'pinned-dependencies', target: 'package.json', expected: true, scope: 'project', - }), - }, -]; +]; \ No newline at end of file diff --git a/src/parsers/rule-patterns-testing.ts b/src/parsers/rule-patterns-testing.ts deleted file mode 100644 index 12f3401..0000000 --- a/src/parsers/rule-patterns-testing.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Testing pattern rule matchers. - * - * Matches instructions about test colocation, test structure, - * and test organization patterns. Examples: - * - "Colocate tests with source files" - * - "Use describe/it blocks" - * - "Each component should have a test file" - */ - -import type { RuleMatcher } from '../types.js'; - -/** - * Matchers for testing organization and structure rules. - */ -export const TESTING_MATCHERS: RuleMatcher[] = [ - { - id: 'testing-colocate-tests', - patterns: [ - /\bcolocat(?:e|ed|ion)\b.*\btests?\b/i, - /\btests?\b.*\bcolocat(?:e|ed|ion)\b/i, - /\btests?\s+(?:next\s+to|beside|alongside)\s+(?:source|src|code)\b/i, - /\btest\s+files?\s+(?:in|next\s+to)\s+(?:the\s+)?same\s+(?:directory|folder)\b/i, - /\beach\s+(?:file|module|component)\s+(?:should|must)\s+have\s+(?:a\s+)?test\b/i, - ], - category: 'testing', - verifier: 'filesystem', - description: 'Test files must be colocated with their source files', - severity: 'warning', - buildPattern: () => ({ - type: 'test-colocation', - target: '*.test.ts', - expected: true, - scope: 'project', - }), - }, - { - id: 'testing-describe-it-blocks', - patterns: [ - /\buse\s+describe\s*\/?\s*it\s+blocks?\b/i, - /\borganize\s+tests?\s+(?:with|using|in)\s+describe\b/i, - /\bdescribe\s+blocks?\s+for\s+(?:each|every)\b/i, - /\bgroup\s+tests?\s+(?:with|using|in)\s+describe\b/i, - /\bnested\s+describe\s+blocks?\b/i, - ], - category: 'testing', - verifier: 'regex', - description: 'Test files must use describe/it block structure', - severity: 'warning', - buildPattern: () => ({ - type: 'describe-it-structure', - target: '*.test.ts', - expected: true, - scope: 'file', - }), - }, - { - id: 'testing-no-console-in-tests', - patterns: [ - /\bno\s+console\.\w+\s+in\s+tests?\b/i, - /\bavoid\s+console\b.*\btest\s+files?\b/i, - /\bdon'?t\s+use\s+console\b.*\btests?\b/i, - /\bremove\s+console\b.*\btests?\b/i, - ], - category: 'testing', - verifier: 'regex', - description: 'Console statements must not appear in test files', - severity: 'warning', - buildPattern: () => ({ - type: 'no-console-in-tests', - target: '*.test.ts', - expected: false, - scope: 'file', - }), - }, -]; diff --git a/src/parsers/rule-patterns-tooling.ts b/src/parsers/rule-patterns-tooling.ts deleted file mode 100644 index c170a09..0000000 --- a/src/parsers/rule-patterns-tooling.ts +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Tooling rule matchers. - * - * Matches instructions about package managers, test frameworks, - * and build tool requirements. Examples: - * - "Use pnpm, not npm" - * - "Use vitest for testing" - * - "Use eslint for linting" - */ - -import type { RuleMatcher } from '../types.js'; - -/** - * Matchers for tooling/dependency alignment rules. - */ -export const TOOLING_MATCHERS: RuleMatcher[] = [ - { - id: 'tooling-package-manager-pnpm', - patterns: [ - /\buse\s+pnpm\b/i, - /\bpnpm\b.*\bnot\s+(?:npm|yarn)\b/i, - /\bpackage\s+manager\b.*\bpnpm\b/i, - /\bpnpm\b.*\bpackage\s+manager\b/i, - /\bprefer\s+pnpm\b/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use pnpm as package manager', - severity: 'warning', - buildPattern: () => ({ - type: 'package-manager', - target: 'pnpm', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-package-manager-yarn', - patterns: [ - /\buse\s+yarn\b/i, - /\byarn\b.*\bnot\s+(?:npm|pnpm)\b/i, - /\bpackage\s+manager\b.*\byarn\b/i, - /\byarn\b.*\bpackage\s+manager\b/i, - /\bprefer\s+yarn\b/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use yarn as package manager', - severity: 'warning', - buildPattern: () => ({ - type: 'package-manager', - target: 'yarn', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-package-manager-bun', - patterns: [ - /\buse\s+bun\b(?!\s+(?:file|run))/i, - /\bbun\b.*\bpackage\s+manager\b/i, - /\bpackage\s+manager\b.*\bbun\b/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use bun as package manager', - severity: 'warning', - buildPattern: () => ({ - type: 'package-manager', - target: 'bun', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-test-framework-vitest', - patterns: [ - /\buse\s+vitest\b/i, - /\bvitest\s+for\s+test/i, - /\btest(?:ing|s)?\s+(?:with|using)\s+vitest\b/i, - /\brun\s+tests?\s+(?:with|using)\s+vitest\b/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use vitest for testing', - severity: 'warning', - buildPattern: () => ({ - type: 'test-framework', - target: 'vitest', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-test-framework-jest', - patterns: [ - /\buse\s+jest\b/i, - /\bjest\s+for\s+test/i, - /\btest(?:ing|s)?\s+(?:with|using)\s+jest\b/i, - /\brun\s+tests?\s+(?:with|using)\s+jest\b/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use jest for testing', - severity: 'warning', - buildPattern: () => ({ - type: 'test-framework', - target: 'jest', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-test-framework-pytest', - patterns: [ - /\buse\s+pytest\b/i, - /\bpytest\s+for\s+test/i, - /\btest(?:ing|s)?\s+(?:with|using)\s+pytest\b/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use pytest for testing', - severity: 'warning', - buildPattern: () => ({ - type: 'test-framework', - target: 'pytest', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-linter-eslint', - patterns: [ - /\buse\s+eslint\b/i, - /\beslint\s+for\s+lint/i, - /\blint(?:ing)?\s+(?:with|using)\s+eslint\b/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use eslint for linting', - severity: 'warning', - buildPattern: () => ({ - type: 'tool-present', - target: 'eslint', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-formatter-prettier', - patterns: [ - /\buse\s+prettier\b/i, - /\bformat\s+(?:with|using)\s+prettier\b/i, - /\bprettier\s+for\s+format/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use prettier for formatting', - severity: 'warning', - buildPattern: () => ({ - type: 'tool-present', - target: 'prettier', - expected: true, - scope: 'project', - }), - }, - { - id: 'tooling-formatter-biome', - patterns: [ - /\buse\s+biome\b/i, - /\bformat\s+(?:with|using)\s+biome\b/i, - /\blint\s+(?:with|using)\s+biome\b/i, - /\bbiome\s+for\s+(?:lint|format)/i, - ], - category: 'tooling', - verifier: 'tooling', - description: 'Project must use biome for linting/formatting', - severity: 'warning', - buildPattern: () => ({ - type: 'tool-present', - target: 'biome', - expected: true, - scope: 'project', - }), - }, -]; diff --git a/src/parsers/rule-patterns.ts b/src/parsers/rule-patterns.ts index 8acf065..b505440 100644 --- a/src/parsers/rule-patterns.ts +++ b/src/parsers/rule-patterns.ts @@ -182,39 +182,6 @@ export const RULE_MATCHERS: RuleMatcher[] = [ type: 'max-file-length', target: '*.ts', expected: match[1] ?? '300', scope: 'file', }), }, - { - id: 'test-files-exist', - patterns: [ - /\ball\s+files?\s+must\s+have\s+tests?\b/i, - /\btest\s+files?\s+(for\s+)?(every|each|all)\b/i, - /\bevery\s+(?:source\s+)?file\s+(?:must\s+|should\s+)?have\s+(?:a\s+)?(?:corresponding\s+)?test/i, - /\bco[\s-]?located\b.*\btests?\b/i, - /\btests?\b.*\bco[\s-]?located\b/i, - /\btest\s+files?:?\s+co[\s-]?located\b/i, - ], - category: 'test-requirement', - verifier: 'filesystem', - description: 'Every source file must have a corresponding test file', - severity: 'error', - buildPattern: () => ({ - type: 'test-files-exist', target: 'src/**/*.ts', expected: true, scope: 'project', - }), - }, - { - id: 'test-named-pattern', - patterns: [ - /\btest\s+files?\b.*\bnamed\b.*\b\.test\.ts\b/i, - /\bnamed\b.*\*\.test\.ts\b/i, - /\b\.test\.ts\b.*\btest\s+files?\b/i, - ], - category: 'test-requirement', - verifier: 'filesystem', - description: 'Test files must be named *.test.ts', - severity: 'error', - buildPattern: () => ({ - type: 'test-file-naming', target: 'tests/**', expected: '*.test.ts', scope: 'project', - }), - }, { id: 'import-no-deep-relative', patterns: [ @@ -263,20 +230,4 @@ export const RULE_MATCHERS: RuleMatcher[] = [ type: 'jsdoc-required', target: '*.ts', expected: true, scope: 'file', }), }, - { - id: 'structure-strict-mode', - patterns: [ - /\bTypeScript\s+strict\s+mode\b/i, - /\bstrict\s+mode\b.*\bTypeScript\b/i, - /\btsconfig\b.*\bstrict\b/i, - /\bstrict:\s*true\b/i, - ], - category: 'structure', - verifier: 'filesystem', - description: 'TypeScript strict mode must be enabled', - severity: 'error', - buildPattern: () => ({ - type: 'strict-mode', target: 'tsconfig.json', expected: true, scope: 'project', - }), - }, ]; diff --git a/src/reporter/detailed.ts b/src/reporter/detailed.ts index 50b562c..ae6e7f4 100644 --- a/src/reporter/detailed.ts +++ b/src/reporter/detailed.ts @@ -18,15 +18,9 @@ const CATEGORY_ORDER: RuleCategory[] = [ 'forbidden-pattern', 'structure', 'import-pattern', - 'test-requirement', 'error-handling', 'type-safety', 'code-style', - 'dependency', - 'preference', - 'file-structure', - 'tooling', - 'testing', ]; /** diff --git a/src/reporter/index.ts b/src/reporter/index.ts index 1eba2c9..92f53a2 100644 --- a/src/reporter/index.ts +++ b/src/reporter/index.ts @@ -16,7 +16,7 @@ import { formatCi } from './ci.js'; export { formatTextPlain, formatParseText } from './text.js'; export { formatJson } from './json.js'; -export { formatMarkdown, formatComparisonMarkdown } from './markdown.js'; +export { formatMarkdown } from './markdown.js'; export { formatRdjson } from './rdjson.js'; export { formatSummary } from './summary.js'; export { formatDetailed } from './detailed.js'; diff --git a/src/reporter/markdown.ts b/src/reporter/markdown.ts index b6c8e14..555779a 100644 --- a/src/reporter/markdown.ts +++ b/src/reporter/markdown.ts @@ -1,10 +1,8 @@ /** * Markdown report formatter. * - * Renders an AdherenceReport as publishable markdown. For single - * verify runs, produces a rule-by-rule report with code blocks - * for evidence. For compare runs (multiple reports), produces a - * comparison table matching the build guide format. + * Renders an AdherenceReport as publishable markdown with + * rule-by-rule results and a category summary table. */ import type { @@ -19,15 +17,9 @@ const CATEGORY_ORDER: RuleCategory[] = [ 'forbidden-pattern', 'structure', 'import-pattern', - 'test-requirement', 'error-handling', 'type-safety', 'code-style', - 'dependency', - 'preference', - 'file-structure', - 'tooling', - 'testing', ]; /** @@ -109,74 +101,3 @@ export function formatMarkdown(report: AdherenceReport): string { return lines.join('\n'); } - -/** - * Format a comparison of multiple AdherenceReports as markdown. - * - * Produces a table with one row per rule and one column per agent, - * plus a score summary row. Matches the build guide comparison format. - * - * @param reports - Array of reports to compare - * @param agentLabels - Display label for each report's agent - * @returns Markdown string with comparison table - */ -export function formatComparisonMarkdown( - reports: AdherenceReport[], - agentLabels: string[], -): string { - const lines: string[] = []; - - if (reports.length === 0) { - return '# RuleProbe: No reports to compare'; - } - - const firstReport = reports[0]!; - - lines.push('# RuleProbe: Agent Instruction Adherence Comparison'); - lines.push(''); - lines.push( - `Rules source: ${firstReport.ruleset.sourceFile} ` + - `(${firstReport.ruleset.rules.length} rules extracted, ` + - `${firstReport.ruleset.unparseable.length} unparseable)`, - ); - lines.push(`Task: ${firstReport.run.taskTemplateId}`); - lines.push(`Date: ${firstReport.run.timestamp.split('T')[0] ?? firstReport.run.timestamp}`); - lines.push(''); - - // Header row - const headerCells = ['Rule', ...agentLabels]; - lines.push(`| ${headerCells.join(' | ')} |`); - const dividerCells = ['------', ...agentLabels.map(() => ':------:')]; - lines.push(`| ${dividerCells.join(' | ')} |`); - - // Rule rows: use the first report's rules as the canonical list - for (const rule of firstReport.ruleset.rules) { - const cells: string[] = [rule.description || rule.id]; - - for (const report of reports) { - const result = report.results.find((r) => r.rule.id === rule.id); - if (!result) { - cells.push('-'); - } else { - cells.push(result.passed ? 'PASS' : 'FAIL'); - } - } - - lines.push(`| ${cells.join(' | ')} |`); - } - - lines.push(''); - - // Score summary table - lines.push('| Agent | Score |'); - lines.push('|-------|-------|'); - for (let i = 0; i < reports.length; i++) { - const report = reports[i]!; - const label = agentLabels[i] ?? report.run.agent; - const model = report.run.model; - const score = Math.round(report.summary.adherenceScore); - lines.push(`| ${label} (${model}) | ${score}% |`); - } - - return lines.join('\n'); -} diff --git a/src/reporter/summary.ts b/src/reporter/summary.ts index 1ecb55c..f11efa6 100644 --- a/src/reporter/summary.ts +++ b/src/reporter/summary.ts @@ -13,15 +13,9 @@ const CATEGORY_ORDER: RuleCategory[] = [ 'forbidden-pattern', 'structure', 'import-pattern', - 'test-requirement', 'error-handling', 'type-safety', 'code-style', - 'dependency', - 'preference', - 'file-structure', - 'tooling', - 'testing', ]; /** diff --git a/src/reporter/text.ts b/src/reporter/text.ts index 9e65fae..05b4cd8 100644 --- a/src/reporter/text.ts +++ b/src/reporter/text.ts @@ -16,15 +16,9 @@ const CATEGORY_ORDER: RuleCategory[] = [ 'forbidden-pattern', 'structure', 'import-pattern', - 'test-requirement', 'error-handling', 'type-safety', 'code-style', - 'dependency', - 'preference', - 'file-structure', - 'tooling', - 'testing', ]; /** diff --git a/src/runner/agent-configs.ts b/src/runner/agent-configs.ts deleted file mode 100644 index 4aff08c..0000000 --- a/src/runner/agent-configs.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Agent configuration types for automated invocation. - * - * Defines the shape of agent configs used by the run command. - * Each supported agent has an ID, SDK requirements, and - * default model settings. - */ - -/** Configuration for a specific agent invocation. */ -export interface AgentInvocationConfig { - /** Agent identifier matching CLI --agent values. */ - agentId: string; - /** Model to use for this invocation. */ - model: string; - /** Environment variable name for the API key. */ - apiKeyEnvVar: string; - /** Tools the agent is allowed to use during invocation. */ - allowedTools: string[]; -} - -/** Options passed to the run command handler. */ -export interface RunOptions { - /** Task template ID to give the agent. */ - task: string; - /** Agent identifier (e.g. "claude-code"). */ - agent: string; - /** Model name (e.g. "sonnet"). */ - model: string; - /** Report output format. */ - format: string; - /** Directory to persist agent output. If omitted, uses a temp dir. */ - outputDir?: string; - /** Watch mode: path to directory to watch for agent output. */ - watch?: string; - /** Watch mode timeout in seconds. */ - timeout: number; - /** Whether to allow symlinks outside the project. */ - allowSymlinks: boolean; - /** Path to ruleprobe config file. */ - config?: string; - /** Path to tsconfig.json for type-aware AST checks. */ - project?: string; -} - -/** - * Default agent configurations for known agents. - * - * @param agentId - Agent identifier - * @param model - Model override - * @returns Agent invocation config - */ -export function buildAgentConfig( - agentId: string, - model: string, -): AgentInvocationConfig { - switch (agentId) { - case 'claude-code': - return { - agentId: 'claude-code', - model, - apiKeyEnvVar: 'ANTHROPIC_API_KEY', - allowedTools: ['Read', 'Write', 'Edit', 'Bash'], - }; - default: - return { - agentId, - model, - apiKeyEnvVar: 'ANTHROPIC_API_KEY', - allowedTools: ['Read', 'Write', 'Edit', 'Bash'], - }; - } -} diff --git a/src/runner/agent-invoker.ts b/src/runner/agent-invoker.ts deleted file mode 100644 index 1694683..0000000 --- a/src/runner/agent-invoker.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Agent invocation via the Claude Agent SDK. - * - * Dynamically imports @anthropic-ai/claude-agent-sdk so it is only - * required when the run command is actually used. If the package is - * not installed, a clear install instruction is shown. - */ - -import { mkdtempSync, existsSync } from 'node:fs'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import type { AgentInvocationConfig } from './agent-configs.js'; - -/** Result of an agent invocation attempt. */ -export interface InvocationResult { - /** Whether the agent completed successfully. */ - success: boolean; - /** Directory containing the agent's output files. */ - outputDir: string; - /** Duration in seconds, or null if not measured. */ - durationSeconds: number | null; - /** Error message if the invocation failed. */ - error?: string; -} - -/** - * Invoke an agent using the Claude Agent SDK. - * - * Dynamically imports the SDK. If it is not installed, throws - * with a clear install instruction. Creates a temp directory - * for agent output unless outputDir is provided. - * - * @param config - Agent configuration - * @param taskPrompt - The full task prompt to give the agent - * @param outputDir - Optional directory for agent output - * @returns Invocation result with output directory and timing - * @throws Error if SDK is not installed or API key is missing - */ -export async function invokeAgent( - config: AgentInvocationConfig, - taskPrompt: string, - outputDir?: string, -): Promise { - const apiKey = process.env[config.apiKeyEnvVar]; - if (!apiKey) { - throw new Error( - `${config.apiKeyEnvVar} is not set. ` + - `Set it in your environment to use agent invocation.`, - ); - } - - const workDir = outputDir ?? mkdtempSync(join(tmpdir(), 'ruleprobe-run-')); - - let queryFn: (opts: Record) => AsyncIterable; - try { - // Dynamic import: SDK is optional, so no static type import - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const sdk: Record = await (Function('return import("@anthropic-ai/claude-agent-sdk")')() as Promise>); - queryFn = sdk['query'] as typeof queryFn; - } catch { - throw new Error( - 'Agent invocation requires @anthropic-ai/claude-agent-sdk. ' + - 'Install it with: npm install @anthropic-ai/claude-agent-sdk', - ); - } - - const startTime = Date.now(); - - try { - const stream = queryFn({ - prompt: taskPrompt, - options: { - model: config.model, - allowedTools: config.allowedTools, - settingSources: [], - workingDir: workDir, - permissionMode: 'acceptEdits', - }, - }); - - for await (const _msg of stream) { - // Consume the stream; SDK handles file writes to workDir - } - - const durationSeconds = (Date.now() - startTime) / 1000; - - return { - success: true, - outputDir: workDir, - durationSeconds, - }; - } catch (err) { - const durationSeconds = (Date.now() - startTime) / 1000; - return { - success: false, - outputDir: workDir, - durationSeconds, - error: (err as Error).message, - }; - } -} - -/** - * Check whether the Claude Agent SDK is importable. - * - * @returns true if the SDK is available, false otherwise - */ -export async function isAgentSdkAvailable(): Promise { - try { - await (Function('return import("@anthropic-ai/claude-agent-sdk")')() as Promise); - return true; - } catch { - return false; - } -} - -/** - * Verify that the agent output directory contains files. - * - * @param outputDir - Directory to check - * @returns true if the directory exists and is non-empty - */ -export function hasAgentOutput(outputDir: string): boolean { - return existsSync(outputDir); -} diff --git a/src/runner/task-templates.ts b/src/runner/task-templates.ts deleted file mode 100644 index 9020a19..0000000 --- a/src/runner/task-templates.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Task template registry. - * - * Contains metadata for standardized coding tasks that exercise - * common rule categories. Prompt text lives in markdown files - * under src/runner/task-templates/. This module provides the - * registry for listing and looking up templates. - */ - -import { readFileSync, existsSync } from 'node:fs'; -import { resolve, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import type { RuleCategory } from '../types.js'; - -/** Metadata for a task template (without the full prompt text). */ -export interface TaskTemplateMeta { - id: string; - name: string; - description: string; - expectedFiles: string[]; - exercises: RuleCategory[]; -} - -/** Registry of available task templates. */ -const TASK_TEMPLATES: TaskTemplateMeta[] = [ - { - id: 'rest-endpoint', - name: 'REST API Endpoint', - description: 'Build a REST API endpoint for managing user bookmarks (POST, GET, DELETE).', - expectedFiles: [ - 'src/routes/bookmarks.ts', - 'src/types.ts', - 'tests/routes/bookmarks.test.ts', - ], - exercises: ['naming', 'forbidden-pattern', 'structure', 'test-requirement'], - }, - { - id: 'utility-module', - name: 'Utility Module with Tests', - description: 'Build a string utility module with validation, formatting, and full test coverage.', - expectedFiles: [ - 'src/utils/string-utils.ts', - 'tests/utils/string-utils.test.ts', - ], - exercises: ['naming', 'forbidden-pattern', 'structure', 'test-requirement', 'import-pattern'], - }, - { - id: 'react-component', - name: 'React Component', - description: 'Build a reusable data table component with sorting, filtering, and pagination.', - expectedFiles: [ - 'src/components/data-table.tsx', - 'src/components/data-table.types.ts', - 'tests/components/data-table.test.tsx', - ], - exercises: ['naming', 'forbidden-pattern', 'structure', 'test-requirement'], - }, -]; - -/** - * List all available task templates. - * - * @returns Array of task template metadata - */ -export function listTaskTemplates(): TaskTemplateMeta[] { - return TASK_TEMPLATES; -} - -/** - * Find a task template by ID. - * - * @param id - Template identifier - * @returns The template metadata, or undefined if not found - */ -export function findTaskTemplate(id: string): TaskTemplateMeta | undefined { - return TASK_TEMPLATES.find((t) => t.id === id); -} - -/** - * Load the full prompt text for a task template. - * - * Reads from the markdown file under src/runner/task-templates/. - * Returns null if the template file does not exist yet (Phase 4). - * - * @param id - Template identifier - * @returns Prompt text, or null if the file is not available - */ -export function loadTaskPrompt(id: string): string | null { - const template = findTaskTemplate(id); - if (!template) { - return null; - } - - const thisDir = dirname(fileURLToPath(import.meta.url)); - const templatePath = resolve(thisDir, 'task-templates', `${id}.md`); - - if (!existsSync(templatePath)) { - return null; - } - - return readFileSync(templatePath, 'utf-8'); -} diff --git a/src/runner/task-templates/react-component.md b/src/runner/task-templates/react-component.md deleted file mode 100644 index e55926a..0000000 --- a/src/runner/task-templates/react-component.md +++ /dev/null @@ -1,50 +0,0 @@ -# Task: SearchFilter React Component - -Build a reusable SearchFilter component in TypeScript with React. - -## Component: SearchFilter - -A search and filter UI component that combines a text input with tag-based filtering and displays a result count. - -### Props - -- `items`: Array of objects to filter. Each object has at least `name` (string) and `tags` (string array). -- `onFilterChange`: Callback invoked with the filtered items array whenever the filter state changes. -- `placeholder`: Optional placeholder text for the search input (default: `"Search..."`) -- `debounceMs`: Optional debounce delay in milliseconds for the text input (default: `300`) -- `availableTags`: Array of strings representing all selectable tags. - -### Behavior - -1. **Text search**: Filter items where `name` contains the search text (case-insensitive). Debounce the input so filtering only runs after the user stops typing for `debounceMs` milliseconds. - -2. **Tag selection**: Render each available tag as a toggleable button. Clicking a tag toggles it on/off. When one or more tags are selected, only show items that include at least one selected tag. - -3. **Combined filtering**: When both text and tags are active, apply both filters (intersection). An item must match the text search AND have at least one selected tag. - -4. **Result count**: Display a count of matching items, e.g. "12 results" or "No results". - -5. **Clear all**: Provide a button to reset both the search text and tag selection. - -### Type Definitions - -Define and export: -- `FilterableItem`: The shape of items passed to the component -- `SearchFilterProps`: The component's prop types - -## Expected Output Structure - -``` -src/ - components/search-filter.tsx # Component implementation - components/search-filter.types.ts # Type definitions -tests/ - components/search-filter.test.tsx # Component tests -``` - -## Deliverables - -1. Fully typed SearchFilter component using function component syntax -2. Debounced text input using a custom `useDebounce` hook or inline logic -3. Tag toggle buttons with visual selected/unselected state -4. Tests covering: initial render, text filtering, tag filtering, combined filtering, debounce behavior, and clear functionality diff --git a/src/runner/task-templates/rest-endpoint.md b/src/runner/task-templates/rest-endpoint.md deleted file mode 100644 index 68ce34d..0000000 --- a/src/runner/task-templates/rest-endpoint.md +++ /dev/null @@ -1,59 +0,0 @@ -# Task: REST API for User Bookmarks - -Build a REST API for managing user bookmarks using TypeScript and Express. - -## Endpoints - -### POST /bookmarks - -Create a new bookmark. - -Request body: -- `url` (string, required): The bookmark URL -- `title` (string, required): Display title -- `tags` (string array, optional): Categorization tags - -Returns the created bookmark with a generated `id` and `createdAt` timestamp. -Return HTTP 400 for missing required fields or invalid URL format. - -### GET /bookmarks - -List all bookmarks. Supports optional query parameters: - -- `tag` (string): Filter bookmarks that include this tag -- `limit` (number): Maximum results to return (default 20) - -Returns an array of bookmark objects sorted by `createdAt` descending. - -### DELETE /bookmarks/:id - -Delete a bookmark by ID. - -Return HTTP 404 if the bookmark does not exist. -Return HTTP 204 on successful deletion. - -## Technical Requirements - -- TypeScript with Express -- Input validation on all endpoints -- Proper HTTP status codes for success and error cases -- Types exported for request and response shapes -- In-memory storage (no database required) - -## Expected Output Structure - -``` -src/ - routes/bookmarks.ts # Route handlers - types.ts # Shared type definitions - middleware/validation.ts # Input validation logic -tests/ - routes/bookmarks.test.ts # Endpoint tests -``` - -## Deliverables - -1. Working route handlers for all three endpoints -2. Type definitions for Bookmark, CreateBookmarkRequest, and BookmarkListQuery -3. Validation middleware that rejects malformed input -4. Unit tests covering success and error cases for each endpoint diff --git a/src/runner/task-templates/utility-module.md b/src/runner/task-templates/utility-module.md deleted file mode 100644 index dbecf17..0000000 --- a/src/runner/task-templates/utility-module.md +++ /dev/null @@ -1,64 +0,0 @@ -# Task: String Utility Module - -Build a TypeScript utility module with three string manipulation functions and full test coverage. - -## Functions - -### slugify(input: string): string - -Convert a string into a URL-friendly slug. - -- Convert to lowercase -- Replace spaces and special characters with hyphens -- Collapse consecutive hyphens into one -- Strip leading and trailing hyphens -- Handle unicode characters by removing them - -Examples: -- `"Hello World"` -> `"hello-world"` -- `" Multiple Spaces "` -> `"multiple-spaces"` -- `"Special @#$ Characters!"` -> `"special-characters"` - -### truncate(input: string, maxLength: number, suffix?: string): string - -Truncate a string to a maximum length, appending a suffix if truncated. - -- Default suffix is `"..."` -- If the string fits within maxLength, return it unchanged -- The returned string (including suffix) must not exceed maxLength -- Never break in the middle of a word; truncate at the last space before the limit - -Examples: -- `truncate("Hello World", 50)` -> `"Hello World"` -- `truncate("Hello World", 8)` -> `"Hello..."` -- `truncate("Hello World", 8, "…")` -> `"Hello W…"` - -### extractDomain(url: string): string | null - -Extract the domain name from a URL string. - -- Handle URLs with or without protocol (`https://`, `http://`) -- Strip `www.` prefix if present -- Return `null` for invalid or empty input -- Handle URLs with ports, paths, and query strings - -Examples: -- `"https://www.example.com/page"` -> `"example.com"` -- `"http://api.github.com:8080/v1"` -> `"api.github.com"` -- `"not-a-url"` -> `null` - -## Expected Output Structure - -``` -src/ - utils/string-utils.ts # All three functions -tests/ - utils/string-utils.test.ts # Tests for all three functions -``` - -## Deliverables - -1. All three functions implemented and exported -2. Type annotations on all parameters and return values -3. Edge case handling (empty strings, null-ish input, extreme lengths) -4. Unit tests with at least 5 test cases per function diff --git a/src/runner/watch-mode.ts b/src/runner/watch-mode.ts deleted file mode 100644 index 0060e44..0000000 --- a/src/runner/watch-mode.ts +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Watch mode for non-SDK agent invocation. - * - * Watches a directory for file changes and triggers verification - * when a .done marker file appears or a timeout elapses. Designed - * for agents that write output to a directory without an SDK - * (Copilot, Cursor, etc). - */ - -import { existsSync, readdirSync, statSync, watchFile, unwatchFile } from 'node:fs'; -import { join, extname } from 'node:path'; - -/** Options for watch mode. */ -export interface WatchOptions { - /** Directory to watch for agent output. */ - watchDir: string; - /** Timeout in seconds. 0 means no timeout. */ - timeoutSeconds: number; - /** How often to poll for the .done marker, in milliseconds. */ - pollIntervalMs?: number; -} - -/** Result of watch mode completion. */ -export interface WatchResult { - /** Whether the watch completed (vs timed out). */ - completed: boolean; - /** How the watch ended: .done marker found, timeout, or error. */ - reason: 'done-marker' | 'timeout' | 'error'; - /** Duration in seconds from start to completion. */ - durationSeconds: number; - /** Error message if reason is 'error'. */ - error?: string; -} - -/** - * Watch a directory for agent output completion. - * - * Polls for a .done marker file in the watch directory. Returns - * when the marker appears or the timeout elapses. - * - * @param options - Watch configuration - * @returns Watch result indicating how the watch ended - */ -export function watchForCompletion(options: WatchOptions): Promise { - const { watchDir, timeoutSeconds, pollIntervalMs = 1000 } = options; - const startTime = Date.now(); - - return new Promise((resolve) => { - const doneMarkerPath = join(watchDir, '.done'); - - const checkDone = (): boolean => { - return existsSync(doneMarkerPath); - }; - - // Check immediately - if (checkDone()) { - resolve({ - completed: true, - reason: 'done-marker', - durationSeconds: (Date.now() - startTime) / 1000, - }); - return; - } - - let timerHandle: ReturnType | null = null; - let timeoutHandle: ReturnType | null = null; - - const cleanup = (): void => { - if (timerHandle !== null) { - clearTimeout(timerHandle); - } - if (timeoutHandle !== null) { - clearTimeout(timeoutHandle); - } - }; - - const poll = (): void => { - if (checkDone()) { - cleanup(); - resolve({ - completed: true, - reason: 'done-marker', - durationSeconds: (Date.now() - startTime) / 1000, - }); - return; - } - timerHandle = setTimeout(poll, pollIntervalMs); - }; - - // Start polling - timerHandle = setTimeout(poll, pollIntervalMs); - - // Set timeout if non-zero - if (timeoutSeconds > 0) { - timeoutHandle = setTimeout(() => { - cleanup(); - resolve({ - completed: false, - reason: 'timeout', - durationSeconds: timeoutSeconds, - }); - }, timeoutSeconds * 1000); - } - }); -} - -/** - * Count code files in a directory (non-recursive for quick check). - * - * @param dir - Directory to scan - * @returns Number of code files found - */ -export function countCodeFiles(dir: string): number { - if (!existsSync(dir)) { - return 0; - } - - const codeExts = new Set(['.ts', '.tsx', '.js', '.jsx', '.py', '.go']); - let count = 0; - - const scan = (scanDir: string): void => { - const entries = readdirSync(scanDir, { withFileTypes: true }); - for (const entry of entries) { - if (entry.name.startsWith('.') || entry.name === 'node_modules') { - continue; - } - const fullPath = join(scanDir, entry.name); - if (entry.isDirectory()) { - scan(fullPath); - } else if (codeExts.has(extname(entry.name))) { - count++; - } - } - }; - - scan(dir); - return count; -} diff --git a/src/types.ts b/src/types.ts index cdad64c..1d13c06 100644 --- a/src/types.ts +++ b/src/types.ts @@ -13,21 +13,14 @@ export type RuleCategory = | 'naming' | 'forbidden-pattern' | 'structure' - | 'test-requirement' | 'import-pattern' | 'error-handling' | 'type-safety' | 'code-style' - | 'dependency' - | 'preference' - | 'file-structure' - | 'tooling' - | 'testing' - | 'workflow' | 'agent-behavior'; /** Which verification engine handles a given rule. */ -export type VerifierType = 'ast' | 'regex' | 'filesystem' | 'treesitter' | 'preference' | 'tooling' | 'config-file' | 'git-history'; +export type VerifierType = 'ast' | 'regex' | 'filesystem'; /** * Qualifier describing the strength of an instruction. @@ -161,6 +154,8 @@ export interface RuleResult { compliance: number; /** Evidence of what was checked and found. */ evidence: Evidence[]; + /** Whether this rule was skipped because it has no concrete implementation. */ + skipped?: boolean; } /** Per-category breakdown of pass/total counts. */ diff --git a/src/verifier/ast-verifier.ts b/src/verifier/ast-verifier.ts index fb7cddd..a205aed 100644 --- a/src/verifier/ast-verifier.ts +++ b/src/verifier/ast-verifier.ts @@ -38,6 +38,14 @@ import { checkConciseConditionals, } from '../ast-checks/index.js'; +/** Sentinel value returned by runAstCheck when a pattern type has no implementation. */ +const SKIP_MARKER: Evidence[] = []; + +/** Check whether evidence array is the skip marker. */ +function isSkipMarker(evidence: Evidence[]): boolean { + return evidence === SKIP_MARKER; +} + /** Create a ts-morph Project for parsing without compilation. */ export function createProject(): Project { return new Project({ @@ -145,11 +153,11 @@ export function runAstCheck(rule: Rule, filePath: string, sourceFile: SourceFile return checkUpperCaseConstants(sourceFile, filePath); case 'async-try-catch': case 'error-log-context': - // These rules are extracted and tracked but require deeper semantic analysis - // to verify accurately. Return empty evidence (no violations found). - return []; + // These rules are tracked but not yet implemented for AST verification. + // Return a marker to indicate the check was skipped. + return SKIP_MARKER; default: - return []; + return SKIP_MARKER; } } @@ -194,11 +202,16 @@ export function verifyAstRule( const project = createProject(); const allEvidence: Evidence[] = []; + let skipped = false; for (const fp of filePaths) { try { const sourceFile = project.addSourceFileAtPath(fp); const evidence = runAstCheck(rule, fp, sourceFile); + if (isSkipMarker(evidence)) { + skipped = true; + continue; + } allEvidence.push(...evidence); } catch { allEvidence.push({ @@ -211,6 +224,16 @@ export function verifyAstRule( } } + if (skipped && allEvidence.length === 0) { + return { + rule, + passed: false, + compliance: 0, + evidence: [], + skipped: true, + }; + } + return { rule, passed: allEvidence.length === 0, @@ -288,6 +311,6 @@ function runTypeAwareCheck( case 'no-unresolved-imports': return checkUnresolvedImports(sourceFile, filePath, project); default: - return []; + return SKIP_MARKER; } } diff --git a/src/verifier/file-verifier.ts b/src/verifier/file-verifier.ts index a4c83f7..6d0761d 100644 --- a/src/verifier/file-verifier.ts +++ b/src/verifier/file-verifier.ts @@ -3,6 +3,7 @@ * * Routes filesystem rules to the appropriate check function * and manages directory walking with symlink awareness. + * Unknown pattern types are skipped rather than false-passed. */ import type { Rule, RuleResult, Evidence } from '../types.js'; @@ -28,6 +29,24 @@ import { checkTestColocation, } from './file-structure-checks.js'; +/** Pattern types handled by the filesystem verifier. */ +const SUPPORTED_FILE_PATTERNS = new Set([ + 'kebab-case', + 'kebab-case-directories', + 'test-files-exist', + 'test-file-naming', + 'max-file-length', + 'strict-mode', + 'readme-exists', + 'changelog-exists', + 'formatter-config-exists', + 'pinned-dependencies', + 'directory-exists-with-files', + 'file-pattern-exists', + 'module-index-required', + 'test-colocation', +]); + /** * Collect all file paths under a directory with symlink awareness. * @@ -60,7 +79,18 @@ export function verifyFileSystemRule( files: string[], ): RuleResult { const patternType = rule.pattern.type; - let evidence: Evidence[]; + + // Skip unknown pattern types instead of false-passing + if (!SUPPORTED_FILE_PATTERNS.has(patternType)) { + return { + rule, + passed: false, + compliance: 0, + evidence: [], + skipped: true, + }; + } + let evidence: Evidence[] = []; switch (patternType) { case 'kebab-case': @@ -133,10 +163,17 @@ export function verifyFileSystemRule( evidence: colocationResult.evidence, }; } - default: - evidence = []; } + // All supported patterns are handled above; the earlier guard + // ensures we never reach this point with an unsupported pattern. + return { + rule, + passed: evidence.length === 0, + compliance: evidence.length === 0 ? 1 : 0, + evidence, + }; + return { rule, passed: evidence.length === 0, diff --git a/src/verifier/index.ts b/src/verifier/index.ts index e7e3d45..25aa91d 100644 --- a/src/verifier/index.ts +++ b/src/verifier/index.ts @@ -2,7 +2,7 @@ * Verification orchestrator. * * Takes a RuleSet and an output directory, routes each rule to - * the correct verifier (AST, filesystem, regex, or tree-sitter), + * the correct verifier (AST, filesystem, or regex), * collects all RuleResults, and returns them. Handles errors * gracefully: if a file can't be parsed, it's logged in evidence. * @@ -11,17 +11,11 @@ * checked against all AST rules, then discarded from memory. */ -import { extname } from 'node:path'; import type { Rule, RuleSet, RuleResult } from '../types.js'; import { verifyAstRulesBatch } from './ast-verifier-batch.js'; import { verifyFileSystemRule, collectFiles, filterSourceFiles } from './file-verifier.js'; export { verifyFileSystemRule } from './file-verifier.js'; import { verifyRegexRule } from './regex-verifier.js'; -import { verifyTreeSitterRule } from './treesitter-verifier.js'; -import { verifyPreferenceRule } from './preference-verifier.js'; -import { verifyToolingRule } from './tooling-verifier.js'; -import { verifyConfigFileRule } from './config-file-verifier.js'; -import { verifyGitHistoryRule } from './git-history-verifier.js'; /** Options for output verification. */ export interface VerifyOptions { @@ -58,10 +52,6 @@ export async function verifyOutput( // (sourceFiles already excludes minified files via filterSourceFiles) const codeFiles = sourceFiles; - // Filter to Python and Go files for tree-sitter checks - const treeSitterExtensions = new Set(['.py', '.go']); - const treeSitterFiles = allFiles.filter((f) => treeSitterExtensions.has(extname(f))); - // Batch all AST rules for single-pass verification const astRules = ruleSet.rules.filter((r) => r.verifier === 'ast'); const astResultMap = astRules.length > 0 @@ -75,7 +65,7 @@ export async function verifyOutput( results.push(astResultMap.get(rule)!); } else { const result = await verifyNonAstRule( - rule, outputDir, codeFiles, sourceFiles, allFiles, treeSitterFiles, projectPath, + rule, outputDir, codeFiles, sourceFiles, allFiles, projectPath, ); results.push(result); } @@ -93,30 +83,20 @@ async function verifyNonAstRule( codeFiles: string[], sourceFiles: string[], allFiles: string[], - treeSitterFiles: string[], - projectPath?: string, + _projectPath?: string, ): Promise { switch (rule.verifier) { case 'filesystem': return verifyFileSystemRule(rule, outputDir, allFiles); case 'regex': return verifyRegexRule(rule, sourceFiles, outputDir); - case 'treesitter': - return verifyTreeSitterRule(rule, treeSitterFiles); - case 'preference': - return verifyPreferenceRule(rule, codeFiles); - case 'tooling': - return verifyToolingRule(rule, outputDir, allFiles); - case 'config-file': - return verifyConfigFileRule(rule, outputDir, allFiles); - case 'git-history': - return verifyGitHistoryRule(rule, outputDir); default: return { rule, - passed: true, - compliance: 1, + passed: false, + compliance: 0, evidence: [], + skipped: true, }; } } diff --git a/src/verifier/regex-verifier.ts b/src/verifier/regex-verifier.ts index 3a999d0..6c06ded 100644 --- a/src/verifier/regex-verifier.ts +++ b/src/verifier/regex-verifier.ts @@ -3,6 +3,7 @@ * * Reads files as plain text and routes to check functions * defined in regex-checks.ts based on the rule's pattern type. + * Unknown pattern types are skipped rather than false-passed. */ import { readFileSync } from 'node:fs'; @@ -23,21 +24,48 @@ import { checkNoConsoleInTests, } from './test-regex-checks.js'; +/** Pattern types handled by the regex verifier. */ +const SUPPORTED_REGEX_PATTERNS = new Set([ + 'max-line-length', + 'no-ts-directives', + 'no-test-only', + 'no-test-skip', + 'quote-style', + 'banned-import', + 'no-todo-comments', + 'consistent-semicolons', + 'describe-it-structure', + 'no-console-in-tests', +]); + /** * Routes to the appropriate check function based on the rule's * verification pattern type. Reads each file as text and runs - * the pattern check. + * the pattern check. Returns skipped for unknown pattern types. * * @param rule - The rule to verify * @param filePaths - Absolute paths to files to check * @param outputDir - Root directory for relative path computation - * @returns A RuleResult with pass/fail and evidence + * @returns A RuleResult with pass/fail and evidence, or skipped for unsupported patterns */ export function verifyRegexRule( rule: Rule, filePaths: string[], outputDir: string, ): RuleResult { + const patternType = rule.pattern.type; + + // Skip unknown pattern types instead of false-passing + if (!SUPPORTED_REGEX_PATTERNS.has(patternType)) { + return { + rule, + passed: false, + compliance: 0, + evidence: [], + skipped: true, + }; + } + const allEvidence: Evidence[] = []; for (const fp of filePaths) { @@ -45,7 +73,6 @@ export function verifyRegexRule( const content = readFileSync(fp, 'utf-8'); const relPath = relative(outputDir, fp); const fileName = basename(fp); - const patternType = rule.pattern.type; switch (patternType) { case 'max-line-length': { @@ -96,8 +123,6 @@ export function verifyRegexRule( case 'no-console-in-tests': allEvidence.push(...checkNoConsoleInTests(content, relPath, fileName)); break; - default: - break; } } catch { // Skip files we can't read diff --git a/tests/action/comment.test.ts b/tests/action/comment.test.ts new file mode 100644 index 0000000..e3bbec1 --- /dev/null +++ b/tests/action/comment.test.ts @@ -0,0 +1,130 @@ +/** + * Tests for the PR comment formatting module. + * + * Validates that drift results are formatted as well-structured + * markdown comments with summary lines, collapsible details, and + * the dedup marker. + */ + +import { describe, it, expect } from 'vitest'; +import { formatDriftComment, formatDriftSummary } from '../../src/action/comment.js'; +import type { DriftResult } from '../../src/drift/types.js'; + +const noDriftResult: DriftResult = { + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, +}; + +const driftResult: DriftResult = { + items: [ + { + kind: 'md-only', + ruleName: '@typescript-eslint/no-explicit-any', + mdRuleId: 'no-any', + mdDescription: 'Never use any', + mdSeverity: 'error', + message: '@typescript-eslint/no-explicit-any is in CLAUDE.md but not in eslint config', + }, + { + kind: 'eslint-only', + ruleName: 'no-console', + eslintSeverity: 'warn', + message: 'no-console is in eslint config but not derived from CLAUDE.md', + }, + { + kind: 'severity-mismatch', + ruleName: 'prefer-const', + mdSeverity: 'error', + eslintSeverity: 'warn', + message: 'prefer-const: CLAUDE.md says error, eslint says warn', + }, + ], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: true, +}; + +const longDriftResult: DriftResult = { + items: [ + ...Array.from({ length: 15 }, (_, i) => ({ + kind: 'md-only' as const, + ruleName: `rule-${i}`, + mdRuleId: `rule-${i}`, + mdDescription: `Rule ${i} description`, + mdSeverity: 'error' as const, + message: `rule-${i} is in CLAUDE.md but not in eslint config`, + })), + ], + mdFile: 'CLAUDE.md', + eslintFile: 'eslint.config.js', + hasDrift: true, +}; + +describe('formatDriftSummary', () => { + it('formats zero drift', () => { + expect(formatDriftSummary(0)).toBe('No drift detected'); + }); + + it('formats single drift issue', () => { + expect(formatDriftSummary(1)).toBe('1 drift issue detected'); + }); + + it('formats multiple drift issues', () => { + expect(formatDriftSummary(5)).toBe('5 drift issues detected'); + }); +}); + +describe('formatDriftComment', () => { + it('includes the dedup marker', () => { + const comment = formatDriftComment(noDriftResult); + expect(comment).toContain(''); + }); + + it('includes "No drift detected" when no drift', () => { + const comment = formatDriftComment(noDriftResult); + expect(comment).toContain('No drift detected'); + }); + + it('includes summary line with drift count', () => { + const comment = formatDriftComment(driftResult); + expect(comment).toContain('3 drift issues detected'); + }); + + it('includes a markdown table for drift items', () => { + const comment = formatDriftComment(driftResult); + expect(comment).toContain('| Kind | Rule |'); + expect(comment).toContain('| md-only'); + expect(comment).toContain('@typescript-eslint/no-explicit-any'); + }); + + it('includes severity info in the table', () => { + const comment = formatDriftComment(driftResult); + expect(comment).toContain('severity-mismatch'); + expect(comment).toContain('prefer-const'); + }); + + it('wraps long drift results in a collapsible details block', () => { + const comment = formatDriftComment(longDriftResult); + expect(comment).toContain('
'); + expect(comment).toContain('
'); + expect(comment).toContain('15 drift issues'); + }); + + it('does not wrap short drift results in details block', () => { + const comment = formatDriftComment(driftResult); + expect(comment).not.toContain('
'); + }); + + it('includes the instruction file and eslint file paths', () => { + const comment = formatDriftComment(driftResult); + expect(comment).toContain('CLAUDE.md'); + expect(comment).toContain('.eslintrc.json'); + }); + + it('formats no-drift result without a table', () => { + const comment = formatDriftComment(noDriftResult); + expect(comment).not.toContain('| Kind |'); + }); +}); \ No newline at end of file diff --git a/tests/action/detect-changes.test.ts b/tests/action/detect-changes.test.ts new file mode 100644 index 0000000..a75a433 --- /dev/null +++ b/tests/action/detect-changes.test.ts @@ -0,0 +1,146 @@ +/** + * Tests for the detect-changes module. + * + * Validates that drift detection correctly identifies relevant file + * changes and skips when only unrelated files are modified. + */ + +import { describe, it, expect } from 'vitest'; +import { shouldRunDrift, autoDetectEslintFile } from '../../src/action/detect-changes.js'; + +describe('shouldRunDrift', () => { + it('returns true when CLAUDE.md is changed', () => { + expect(shouldRunDrift(['src/index.ts', 'CLAUDE.md', 'package.json'])).toBe(true); + }); + + it('returns true when AGENTS.md is changed', () => { + expect(shouldRunDrift(['src/verifiers/ast.ts', 'AGENTS.md'])).toBe(true); + }); + + it('returns true when .cursorrules is changed', () => { + expect(shouldRunDrift(['.cursorrules', 'src/main.ts'])).toBe(true); + }); + + it('returns true when a nested instruction file is changed', () => { + expect(shouldRunDrift(['packages/app/CLAUDE.md', 'src/app.ts'])).toBe(true); + }); + + it('returns true when .eslintrc.json is changed', () => { + expect(shouldRunDrift(['.eslintrc.json', 'src/util.ts'])).toBe(true); + }); + + it('returns true when eslint.config.js is changed', () => { + expect(shouldRunDrift(['eslint.config.js'])).toBe(true); + }); + + it('returns true when eslint.config.mjs is changed', () => { + expect(shouldRunDrift(['eslint.config.mjs', 'README.md'])).toBe(true); + }); + + it('returns true when .eslintrc.js is changed', () => { + expect(shouldRunDrift(['.eslintrc.js'])).toBe(true); + }); + + it('returns true when .eslintrc.cjs is changed', () => { + expect(shouldRunDrift(['.eslintrc.cjs'])).toBe(true); + }); + + it('returns true when .eslintrc.yml is changed', () => { + expect(shouldRunDrift(['.eslintrc.yml'])).toBe(true); + }); + + it('returns true when eslint.config.ts is changed', () => { + expect(shouldRunDrift(['eslint.config.ts'])).toBe(true); + }); + + it('returns false when only source files are changed', () => { + expect(shouldRunDrift(['src/index.ts', 'src/util.ts', 'package.json'])).toBe(false); + }); + + it('returns false for empty changed files list', () => { + expect(shouldRunDrift([])).toBe(false); + }); + + it('returns false for unrelated markdown files', () => { + expect(shouldRunDrift(['docs/api.md', 'CHANGELOG.md'])).toBe(false); + }); + + it('returns false for unrelated config files', () => { + expect(shouldRunDrift(['tsconfig.json', 'vitest.config.ts'])).toBe(false); + }); + + it('returns true when instruction file matches explicit path', () => { + expect( + shouldRunDrift(['src/index.ts', 'custom-instructions.md'], { + instructionFile: 'custom-instructions.md', + }), + ).toBe(true); + }); + + it('returns true when eslint file matches explicit path', () => { + expect( + shouldRunDrift(['src/index.ts', 'custom-eslint.json'], { + eslintFile: 'custom-eslint.json', + }), + ).toBe(true); + }); + + it('does not double-count a file that matches both patterns', () => { + const result = shouldRunDrift(['CLAUDE.md']); + expect(result).toBe(true); + }); +}); + +describe('autoDetectEslintFile', () => { + it('detects .eslintrc.json', () => { + expect(autoDetectEslintFile(['package.json', '.eslintrc.json', 'src/index.ts'])).toBe('.eslintrc.json'); + }); + + it('detects eslint.config.js', () => { + expect(autoDetectEslintFile(['eslint.config.js', 'src/index.ts'])).toBe('eslint.config.js'); + }); + + it('detects eslint.config.mjs', () => { + expect(autoDetectEslintFile(['eslint.config.mjs'])).toBe('eslint.config.mjs'); + }); + + it('detects eslint.config.ts', () => { + expect(autoDetectEslintFile(['eslint.config.ts'])).toBe('eslint.config.ts'); + }); + + it('detects .eslintrc.js', () => { + expect(autoDetectEslintFile(['.eslintrc.js'])).toBe('.eslintrc.js'); + }); + + it('detects .eslintrc.cjs', () => { + expect(autoDetectEslintFile(['.eslintrc.cjs'])).toBe('.eslintrc.cjs'); + }); + + it('prefers eslint.config.js over .eslintrc.json', () => { + expect(autoDetectEslintFile(['.eslintrc.json', 'eslint.config.js'])).toBe('eslint.config.js'); + }); + + it('prefers eslint.config.mjs over .eslintrc.json', () => { + expect(autoDetectEslintFile(['.eslintrc.json', 'eslint.config.mjs'])).toBe('eslint.config.mjs'); + }); + + it('returns undefined when no eslint config is found', () => { + expect(autoDetectEslintFile(['package.json', 'tsconfig.json', 'src/index.ts'])).toBeUndefined(); + }); + + it('returns undefined for empty file list', () => { + expect(autoDetectEslintFile([])).toBeUndefined(); + }); + + it('ignores eslint config in subdirectories', () => { + expect(autoDetectEslintFile(['packages/app/.eslintrc.json', 'src/index.ts'])).toBeUndefined(); + }); + + it('detects .eslintrc.yml', () => { + expect(autoDetectEslintFile(['.eslintrc.yml'])).toBe('.eslintrc.yml'); + }); + + it('detects .eslintrc (no extension)', () => { + expect(autoDetectEslintFile(['.eslintrc'])).toBe('.eslintrc'); + }); +}); \ No newline at end of file diff --git a/tests/action/regenerate.test.ts b/tests/action/regenerate.test.ts new file mode 100644 index 0000000..84da1ec --- /dev/null +++ b/tests/action/regenerate.test.ts @@ -0,0 +1,60 @@ +/** + * Tests for the regenerate module. + * + * Validates that regeneration produces the correct branch name, + * commit message, and PR title from the instruction file path. + * The actual git/gh operations are tested via the runner integration + * tests with injected deps. + */ + +import { describe, it, expect } from 'vitest'; +import { branchNameFor, commitMessageFor, prTitleFor } from '../../src/action/regenerate.js'; + +describe('branchNameFor', () => { + it('produces a deterministic branch name from instruction file', () => { + const branch = branchNameFor('CLAUDE.md'); + expect(branch).toMatch(/^ruleprobe\/sync-[a-f0-9]+$/); + }); + + it('produces different branch names for different files', () => { + expect(branchNameFor('CLAUDE.md')).not.toBe(branchNameFor('AGENTS.md')); + }); + + it('produces the same branch name for the same file', () => { + expect(branchNameFor('CLAUDE.md')).toBe(branchNameFor('CLAUDE.md')); + }); + + it('handles paths with directories', () => { + const branch = branchNameFor('packages/app/CLAUDE.md'); + expect(branch).toMatch(/^ruleprobe\/sync-[a-f0-9]+$/); + }); +}); + +describe('commitMessageFor', () => { + it('includes the instruction file name', () => { + const msg = commitMessageFor('CLAUDE.md'); + expect(msg).toContain('CLAUDE.md'); + }); + + it('starts with chore:', () => { + const msg = commitMessageFor('CLAUDE.md'); + expect(msg).toMatch(/^chore:/); + }); + + it('includes "sync eslint config"', () => { + const msg = commitMessageFor('CLAUDE.md'); + expect(msg).toContain('sync eslint config'); + }); +}); + +describe('prTitleFor', () => { + it('includes the instruction file name', () => { + const title = prTitleFor('CLAUDE.md'); + expect(title).toContain('CLAUDE.md'); + }); + + it('mentions drift sync', () => { + const title = prTitleFor('CLAUDE.md'); + expect(title).toMatch(/drift/i); + }); +}); \ No newline at end of file diff --git a/tests/action/runner.test.ts b/tests/action/runner.test.ts new file mode 100644 index 0000000..14b324e --- /dev/null +++ b/tests/action/runner.test.ts @@ -0,0 +1,300 @@ +/** + * Tests for the action runner. + * + * Tests the orchestration logic with injected dependencies, confirming + * that the right commands run based on inputs and PR context. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { runAction } from '../../src/action/runner.js'; +import type { ActionInputs, ActionDeps, GitHubContext } from '../../src/action/types.js'; + +function makeContext(overrides?: Partial): GitHubContext { + return { + repository: 'owner/repo', + apiUrl: 'https://api.github.com', + prNumber: 42, + token: 'test-token', + workspace: '/home/runner/work/repo', + eventPath: '/home/runner/work/_temp/event.json', + eventName: 'pull_request', + ...overrides, + }; +} + +function makeDeps(overrides?: Partial): ActionDeps { + return { + runCommand: vi.fn().mockResolvedValue(0), + getChangedFiles: vi.fn().mockResolvedValue(['CLAUDE.md', '.eslintrc.json']), + postComment: vi.fn().mockResolvedValue(undefined), + exec: vi.fn().mockResolvedValue({ stdout: '', exitCode: 0 }), + writeFile: vi.fn().mockResolvedValue(undefined), + readFile: vi.fn().mockResolvedValue('{}'), + info: vi.fn(), + warn: vi.fn(), + setOutput: vi.fn(), + setFailed: vi.fn(), + ...overrides, + }; +} + +function makeInputs(overrides?: Partial): ActionInputs { + return { + mode: 'drift', + instructionFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + regenerateOnDrift: false, + commentOnPr: true, + failOnDrift: false, + ...overrides, + }; +} + +describe('runAction (drift mode)', () => { + it('skips when no relevant files changed', async () => { + const deps = makeDeps({ + getChangedFiles: vi.fn().mockResolvedValue(['src/index.ts', 'package.json']), + }); + + await runAction(makeInputs(), makeContext(), deps); + + expect(deps.runCommand).not.toHaveBeenCalled(); + expect(deps.postComment).not.toHaveBeenCalled(); + expect(deps.info).toHaveBeenCalledWith( + expect.stringContaining('skipping'), + ); + }); + + it('runs drift when CLAUDE.md is changed', async () => { + const deps = makeDeps({ + runCommand: vi.fn().mockResolvedValue(0), + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, + })), + }); + + await runAction(makeInputs(), makeContext(), deps); + + expect(deps.runCommand).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['drift']), + ); + }); + + it('posts a PR comment when drift runs and commentOnPr is true', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, + })), + }); + + await runAction(makeInputs({ commentOnPr: true }), makeContext(), deps); + + expect(deps.postComment).toHaveBeenCalledWith( + expect.any(Object), + 42, + expect.stringContaining('ruleprobe-drift'), + expect.any(String), + ); + }); + + it('skips PR comment when commentOnPr is false', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, + })), + }); + + await runAction(makeInputs({ commentOnPr: false }), makeContext(), deps); + + expect(deps.postComment).not.toHaveBeenCalled(); + }); + + it('skips PR comment when not in a PR context', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, + })), + }); + const context = makeContext({ prNumber: undefined, eventName: 'push' }); + + await runAction(makeInputs({ commentOnPr: true }), context, deps); + + expect(deps.postComment).not.toHaveBeenCalled(); + }); + + it('sets drift-count and has-drift outputs', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [ + { kind: 'md-only', ruleName: 'no-any', message: 'missing' }, + ], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: true, + })), + }); + + await runAction(makeInputs(), makeContext(), deps); + + expect(deps.setOutput).toHaveBeenCalledWith('drift-count', '1'); + expect(deps.setOutput).toHaveBeenCalledWith('has-drift', 'true'); + }); + + it('fails the action when drift detected and failOnDrift is true', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [ + { kind: 'md-only', ruleName: 'no-any', message: 'missing' }, + ], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: true, + })), + }); + + await runAction(makeInputs({ failOnDrift: true }), makeContext(), deps); + + expect(deps.setFailed).toHaveBeenCalledWith( + expect.stringContaining('drift'), + ); + }); + + it('does not fail when no drift and failOnDrift is true', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, + })), + }); + + await runAction(makeInputs({ failOnDrift: true }), makeContext(), deps); + + expect(deps.setFailed).not.toHaveBeenCalled(); + }); + + it('auto-detects eslint config when eslintFile is not specified', async () => { + const deps = makeDeps({ + getChangedFiles: vi.fn().mockResolvedValue(['CLAUDE.md', '.eslintrc.json']), + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, + })), + }); + + await runAction( + makeInputs({ eslintFile: undefined }), + makeContext(), + deps, + ); + + expect(deps.runCommand).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['.eslintrc.json']), + ); + }); + + it('regenerates config when drift detected and regenerateOnDrift is true', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [ + { kind: 'md-only', ruleName: 'no-any', message: 'missing' }, + ], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: true, + })), + }); + + await runAction( + makeInputs({ regenerateOnDrift: true }), + makeContext(), + deps, + ); + + // Should call exec to run lint-config, git checkout, etc. + expect(deps.exec).toHaveBeenCalled(); + }); + + it('does not regenerate when no drift even if regenerateOnDrift is true', async () => { + const deps = makeDeps({ + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + items: [], + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: false, + })), + }); + + await runAction( + makeInputs({ regenerateOnDrift: true }), + makeContext(), + deps, + ); + + // exec should not be called for git/gh operations + const execCalls = (deps.exec as ReturnType).mock.calls; + const gitCalls = execCalls.filter((call: string[]) => + call[0] === 'git' || call[0] === 'gh', + ); + expect(gitCalls.length).toBe(0); + }); +}); + +describe('runAction (verify mode)', () => { + it('runs verify command in verify mode', async () => { + const deps = makeDeps({ + runCommand: vi.fn().mockResolvedValue(0), + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + summary: { adherenceScore: 100, passed: 5, failed: 0, totalRules: 5 }, + })), + }); + + await runAction( + makeInputs({ mode: 'verify', outputDir: 'src' }), + makeContext(), + deps, + ); + + expect(deps.runCommand).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['verify']), + ); + }); + + it('sets verify outputs in verify mode', async () => { + const deps = makeDeps({ + runCommand: vi.fn().mockResolvedValue(0), + readFile: vi.fn().mockResolvedValue(JSON.stringify({ + summary: { adherenceScore: 80, passed: 4, failed: 1, totalRules: 5 }, + })), + }); + + await runAction( + makeInputs({ mode: 'verify', outputDir: 'src' }), + makeContext(), + deps, + ); + + expect(deps.setOutput).toHaveBeenCalledWith('score', '80'); + expect(deps.setOutput).toHaveBeenCalledWith('passed', '4'); + expect(deps.setOutput).toHaveBeenCalledWith('failed', '1'); + expect(deps.setOutput).toHaveBeenCalledWith('total', '5'); + }); +}); \ No newline at end of file diff --git a/tests/analyzers/project-analyzer.test.ts b/tests/analyzers/project-analyzer.test.ts index 48acf5f..a26881c 100644 --- a/tests/analyzers/project-analyzer.test.ts +++ b/tests/analyzers/project-analyzer.test.ts @@ -82,11 +82,11 @@ describe('analyzeProject', () => { }); it('detects conflicting rules across files', () => { - writeFileSync(join(tempDir, 'CLAUDE.md'), '# Tooling\n- Use pnpm, not npm'); - writeFileSync(join(tempDir, 'AGENTS.md'), '# Tooling\n- Use yarn, not npm'); + writeFileSync(join(tempDir, 'CLAUDE.md'), '# Rules\n- Maximum line length: 80 characters'); + writeFileSync(join(tempDir, 'AGENTS.md'), '# Rules\n- Maximum line length: 120 characters'); const result = analyzeProject(tempDir); expect(result.conflicts.length).toBeGreaterThan(0); - expect(result.conflicts[0]!.topic).toBe('package-manager'); + expect(result.conflicts[0]!.topic).toBe('max-line-length'); }); it('builds coverage map by category', () => { diff --git a/tests/cli/cli.test.ts b/tests/cli/cli.test.ts index 40812aa..c155e55 100644 --- a/tests/cli/cli.test.ts +++ b/tests/cli/cli.test.ts @@ -1,10 +1,10 @@ // Integration tests for the ruleprobe CLI. Spawns the real CLI via tsx -// and verifies parse, verify, tasks, task, compare, and help commands, -// including the summary statistics line in verification output. +// and verifies parse, verify, lint-config, drift, extract, and help commands. import { describe, it, expect } from 'vitest'; import { execSync, type ExecSyncOptionsWithStringEncoding } from 'node:child_process'; import { resolve } from 'node:path'; +import { readFileSync, writeFileSync, unlinkSync } from 'node:fs'; const ROOT = resolve(import.meta.dirname, '..', '..'); const CLI = 'npx tsx src/cli.ts'; @@ -44,6 +44,53 @@ function runFail(args: string): { stderr: string; status: number } { } } +// ── lint-config command ───────────────────────────────────────── + +describe('CLI: lint-config command', () => { + it('emits flat config by default', () => { + const output = run(`lint-config ${CLAUDE_FIXTURE}`); + expect(output).toContain('export default ['); + expect(output).toContain('rules: {'); + }); + + it('emits legacy config with --format legacy', () => { + const output = run(`lint-config ${CLAUDE_FIXTURE} --format legacy`); + expect(output).toContain('"rules"'); + expect(output).toContain('"plugins"'); + }); + + it('includes unmappable rules as comments when present', () => { + // Use a temp file with an instruction that has no ESLint equivalent + const tmpMd = resolve(ROOT, 'tmp-unmappable-test.md'); + try { + writeFileSync(tmpMd, '# Rules\n\nAlways review code before merging.\n'); + const output = run(`lint-config ${tmpMd} --format flat`); + // Unmatched instructions go to unparseable, not unmappable in the ESLint config + // The output should still be a valid flat config + expect(output).toContain('export default ['); + } finally { + unlinkSync(tmpMd); + } + }); + + it('writes output to file with --output', () => { + const tmpFile = resolve(ROOT, 'tmp-lint-config-test.js'); + try { + run(`lint-config ${CLAUDE_FIXTURE} --output ${tmpFile}`); + const content = readFileSync(tmpFile, 'utf-8'); + expect(content).toContain('export default ['); + } finally { + unlinkSync(tmpFile); + } + }); + + it('fails with actionable error for missing instruction file', () => { + const result = runFail('lint-config nonexistent.md'); + expect(result.status).toBe(2); + expect(result.stderr).toContain('Failed to parse instruction file'); + }); +}); + // ── parse command ────────────────────────────────────────────── describe('CLI: parse command', () => { @@ -161,51 +208,70 @@ describe('CLI: verify command', () => { }); }); -// ── tasks command ────────────────────────────────────────────── +// ── drift command ────────────────────────────────────────────── -describe('CLI: tasks command', () => { - it('lists available task templates', () => { - const output = run('tasks'); - expect(output).toContain('rest-endpoint'); - expect(output).toContain('utility-module'); - expect(output).toContain('react-component'); - }); +describe('CLI: drift command', () => { + const ESLINT_FIXTURE = 'tests/drift/fixtures/eslintrc-basic.json'; + const ESLINT_EMPTY = 'tests/drift/fixtures/eslintrc-empty.json'; - it('shows template descriptions', () => { - const output = run('tasks'); - expect(output).toContain('REST API'); + it('reports drift between CLAUDE.md and eslint config', () => { + const result = runFail(`drift ${CLAUDE_FIXTURE} ${ESLINT_EMPTY}`); + expect(result.status).toBe(1); + // runFail captures stderr; drift writes to stdout, so check either }); -}); -// ── task command ─────────────────────────────────────────────── + it('outputs markdown with --format markdown when drift exists', () => { + const result = runFail(`drift ${CLAUDE_FIXTURE} ${ESLINT_FIXTURE} --format markdown`); + expect(result.status).toBe(1); + }); -describe('CLI: task command', () => { - it('outputs the full prompt for a valid template', () => { - const output = run('task rest-endpoint'); - expect(output).toContain('REST API'); - expect(output).toContain('bookmarks'); + it('fails with actionable error for missing instruction file', () => { + const { stderr, status } = runFail(`drift nonexistent.md ${ESLINT_FIXTURE}`); + expect(status).toBe(2); + expect(stderr).toContain('Failed to parse instruction file'); }); - it('fails for unknown template', () => { - const { stderr, status } = runFail('task nonexistent-template'); + it('fails with actionable error for missing eslint config', () => { + const { stderr, status } = runFail(`drift ${CLAUDE_FIXTURE} nonexistent.json`); expect(status).toBe(2); - expect(stderr).toContain('Unknown task template'); + expect(stderr).toContain('Failed to parse ESLint config'); }); }); -// ── compare command ──────────────────────────────────────────── +// ── extract command ────────────────────────────────────────────── -describe('CLI: compare command', () => { - it('produces markdown comparison table', () => { - const output = run( - `compare ${CLAUDE_FIXTURE} ${PASSING_DIR} ${FAILING_DIR} ` + - '--agents passing,failing --format markdown', - ); - expect(output).toContain('Adherence Comparison'); - expect(output).toContain('passing'); - expect(output).toContain('failing'); - expect(output).toContain('PASS'); - expect(output).toContain('FAIL'); +describe('CLI: extract command', () => { + const ESLINT_FIXTURE = 'tests/drift/fixtures/eslintrc-basic.json'; + + it('extracts rules from an eslint config', () => { + const result = run(`extract ${ESLINT_FIXTURE}`); + expect(result).toContain('## Rules'); + expect(result).toContain('`any`'); + }); + + it('skips stylistic rules in output', () => { + const STYLISTIC_FIXTURE = 'tests/extractor/fixtures/eslintrc-stylistic.json'; + const result = run(`extract ${STYLISTIC_FIXTURE}`); + expect(result).toContain('## Rules'); + expect(result).toContain('Skipped rules'); + expect(result).toContain('semi'); + }); + + it('writes output to file with --output', () => { + const outPath = resolve(ROOT, 'tests/extractor/fixtures/extract-output.md'); + try { + run(`extract ${ESLINT_FIXTURE} --output ${outPath}`); + const content = readFileSync(outPath, 'utf-8'); + expect(content).toContain('## Rules'); + } finally { + try { unlinkSync(outPath); } catch { /* already gone */ } + } + }); + + it('fails with actionable error for missing eslint config', () => { + const { stderr, status } = runFail('extract nonexistent.json'); + expect(status).toBe(2); + expect(stderr).toContain('Failed to parse ESLint config'); }); }); @@ -217,7 +283,7 @@ describe('CLI: help', () => { expect(output).toContain('ruleprobe'); expect(output).toContain('parse'); expect(output).toContain('verify'); - expect(output).toContain('tasks'); - expect(output).toContain('compare'); + expect(output).toContain('lint-config'); + expect(output).toContain('drift'); }); }); diff --git a/tests/dataset/collect.test.ts b/tests/dataset/collect.test.ts new file mode 100644 index 0000000..5ad56ba --- /dev/null +++ b/tests/dataset/collect.test.ts @@ -0,0 +1,352 @@ +import { describe, it, expect, beforeEach, afterAll } from 'vitest'; +import { mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { cacheKey, readCache, writeCache } from '../../src/dataset/cache.js'; +import { + parseFileContent, + computeMedian, + computePercentile, + buildHistogram, + clusterUnparseable, + generateSummary, + type PerFileResult, +} from '../../src/dataset/summary.js'; + +const FIXTURE_DIR = join(__dirname, '__cache_fixtures__'); + +describe('collect.ts', () => { + beforeEach(() => { + if (existsSync(FIXTURE_DIR)) { + rmSync(FIXTURE_DIR, { recursive: true, force: true }); + } + mkdirSync(FIXTURE_DIR, { recursive: true }); + }); + + afterAll(() => { + if (existsSync(FIXTURE_DIR)) { + rmSync(FIXTURE_DIR, { recursive: true, force: true }); + } + }); + + // --------------------------------------------------------------------------- + // Cache helpers + // --------------------------------------------------------------------------- + + describe('cacheKey', () => { + it('produces a safe filesystem path from a prefix and identifier', () => { + const key = cacheKey('repo', 'moonrunnerkc-ruleprobe'); + expect(key).toContain('repo-moonrunnerkc-ruleprobe.json'); + }); + + it('replaces slashes in identifier with underscores', () => { + const key = cacheKey('search', 'path/to/something'); + expect(key).toContain('search-path_to_something.json'); + }); + }); + + describe('readCache / writeCache', () => { + it('round-trips data through the cache', () => { + const key = join(FIXTURE_DIR, 'test-entry.json'); + const data = { stars: 42, name: 'ruleprobe' }; + + writeCache(key, data); + const result = readCache(key, 60_000); + + expect(result).toEqual(data); + }); + + it('returns null when the cache key does not exist', () => { + const key = join(FIXTURE_DIR, 'nonexistent.json'); + expect(readCache(key, 60_000)).toBeNull(); + }); + + it('returns null when the cache entry has expired', () => { + const key = join(FIXTURE_DIR, 'expired-entry.json'); + writeFileSync(key, JSON.stringify({ timestamp: Date.now() - 100_000, data: { old: true } })); + + const result = readCache(key, 1); + expect(result).toBeNull(); + }); + + it('returns null for malformed cache entries', () => { + const key = join(FIXTURE_DIR, 'bad-entry.json'); + writeFileSync(key, 'not json at all'); + + const result = readCache(key, 60_000); + expect(result).toBeNull(); + }); + }); + + // --------------------------------------------------------------------------- + // Statistics + // --------------------------------------------------------------------------- + + describe('computeMedian', () => { + it('returns 0 for an empty array', () => { + expect(computeMedian([])).toBe(0); + }); + + it('computes median for odd-length arrays', () => { + expect(computeMedian([3, 1, 2])).toBe(2); + }); + + it('computes median for even-length arrays', () => { + expect(computeMedian([4, 1, 3, 2])).toBe(2.5); + }); + + it('handles single-element arrays', () => { + expect(computeMedian([7])).toBe(7); + }); + }); + + describe('computePercentile', () => { + it('returns 0 for an empty array', () => { + expect(computePercentile([], 75)).toBe(0); + }); + + it('computes the 75th percentile', () => { + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + const values = Array.from({ length: 10 }, (_, i) => i + 1); + const p75 = computePercentile(values, 75); + expect(p75).toBe(8); + }); + + it('computes the 90th percentile', () => { + const values = Array.from({ length: 10 }, (_, i) => i + 1); + const p90 = computePercentile(values, 90); + expect(p90).toBe(9); + }); + }); + + // --------------------------------------------------------------------------- + // Histogram + // --------------------------------------------------------------------------- + + describe('buildHistogram', () => { + it('returns "No data" for empty input', () => { + expect(buildHistogram([])).toBe('No data'); + }); + + it('groups values into correct buckets', () => { + const values = [0, 0, 3, 7, 12, 25, 60]; + const histogram = buildHistogram(values); + expect(histogram).toContain('0 |'); + expect(histogram).toContain('1-4 |'); + expect(histogram).toContain('5-9 |'); + expect(histogram).toContain('10-19 |'); + expect(histogram).toContain('20-49 |'); + expect(histogram).toContain('50+ |'); + }); + + it('draws bars proportional to counts', () => { + const values = [5, 5, 5, 5, 5]; + const histogram = buildHistogram(values); + const line5_9 = histogram.split('\n').find((l) => l.includes('5-9')); + expect(line5_9).toBeTruthy(); + expect(line5_9!.includes('#####')).toBe(true); + }); + }); + + // --------------------------------------------------------------------------- + // Unparseable clustering + // --------------------------------------------------------------------------- + + describe('clusterUnparseable', () => { + it('groups similar lines by normalized form', () => { + const lines = [ + 'Use conventional commits', + 'use conventional commits', + ' Use conventional commits ', + 'Always write tests', + ]; + + const clusters = clusterUnparseable(lines); + expect(clusters.length).toBe(2); + + const commitsCluster = clusters.find((c) => c.pattern.includes('conventional')); + expect(commitsCluster).toBeTruthy(); + expect(commitsCluster!.count).toBe(3); + }); + + it('sorts by count descending', () => { + const lines = [ + 'rare pattern', + 'common pattern', + 'common pattern', + 'common pattern', + ]; + + const clusters = clusterUnparseable(lines); + expect(clusters[0].pattern).toContain('common'); + expect(clusters[0].count).toBe(3); + }); + + it('skips lines shorter than 3 characters', () => { + const lines = ['a', 'ab', 'abc', 'valid line here']; + const clusters = clusterUnparseable(lines); + expect(clusters.every((c) => c.pattern.length >= 3)).toBe(true); + }); + + it('strips leading markdown characters', () => { + const lines = ['- Always use strict mode']; + const clusters = clusterUnparseable(lines); + expect(clusters[0].pattern.startsWith('-')).toBe(false); + }); + }); + + // --------------------------------------------------------------------------- + // parseFileContent + // --------------------------------------------------------------------------- + + describe('parseFileContent', () => { + it('parses a CLAUDE.md file and extracts rules', () => { + const content = [ + '# CLAUDE.md', + '', + '## Code Style', + '', + '- Always use TypeScript strict mode.', + '- Never use `any`. Use `unknown` and narrow.', + '- Use camelCase for variables and functions.', + '', + '## Testing', + '', + '- Every new function requires at least one test.', + '- Test names describe behavior, not implementation.', + ].join('\n'); + + const result = parseFileContent(content, 'CLAUDE.md'); + expect(result.sourceType).toBe('claude.md'); + expect(result.parseableRuleCount).toBeGreaterThan(0); + expect(result.parseError).toBeNull(); + }); + + it('handles empty content gracefully', () => { + const result = parseFileContent('', 'CLAUDE.md'); + expect(result.parseableRuleCount).toBe(0); + expect(result.unparseableLines).toEqual([]); + }); + + it('populates categoryBreakdown from parsed rules', () => { + const content = [ + '# Test', + '', + '- Always use TypeScript strict mode.', + '- Never use `any`.', + '- Every function requires a test.', + ].join('\n'); + + const result = parseFileContent(content, 'CLAUDE.md'); + const totalFromBreakdown = Object.values(result.categoryBreakdown).reduce( + (sum, count) => sum + count, + 0, + ); + expect(totalFromBreakdown).toBe(result.parseableRuleCount); + }); + }); + + // --------------------------------------------------------------------------- + // generateSummary + // --------------------------------------------------------------------------- + + describe('generateSummary', () => { + function makeResult(overrides: Partial = {}): PerFileResult { + return { + repoUrl: 'https://github.com/test/repo', + repoStars: 100, + filePath: 'CLAUDE.md', + sourceType: 'claude.md', + parseableRuleCount: 0, + categoryBreakdown: {}, + unparseableLines: [], + parseError: null, + ...overrides, + }; + } + + it('renders a GO verdict when median >= 5 and P75 >= 10', () => { + const results: PerFileResult[] = [ + makeResult({ parseableRuleCount: 12, categoryBreakdown: { naming: 5, 'code-style': 7 } }), + makeResult({ parseableRuleCount: 15, categoryBreakdown: { naming: 8, structure: 7 } }), + makeResult({ parseableRuleCount: 8, categoryBreakdown: { naming: 8 } }), + makeResult({ parseableRuleCount: 20, categoryBreakdown: { naming: 10, 'code-style': 10 } }), + ]; + + const summary = generateSummary(results); + expect(summary).toContain('GO'); + expect(summary).toContain('Median'); + expect(summary).toContain('75th percentile'); + }); + + it('renders a NO-GO verdict when median < 5', () => { + const results: PerFileResult[] = [ + makeResult({ parseableRuleCount: 2 }), + makeResult({ parseableRuleCount: 3 }), + makeResult({ parseableRuleCount: 1 }), + ]; + + const summary = generateSummary(results); + expect(summary).toContain('NO-GO'); + }); + + it('renders a NO-GO verdict when P75 < 10', () => { + const results: PerFileResult[] = [ + makeResult({ parseableRuleCount: 7 }), + makeResult({ parseableRuleCount: 6 }), + makeResult({ parseableRuleCount: 5 }), + makeResult({ parseableRuleCount: 5 }), + ]; + + const summary = generateSummary(results); + expect(summary).toContain('NO-GO'); + }); + + it('includes top rule categories', () => { + const results: PerFileResult[] = [ + makeResult({ + parseableRuleCount: 10, + categoryBreakdown: { naming: 5, 'code-style': 3, testing: 2 }, + }), + makeResult({ + parseableRuleCount: 5, + categoryBreakdown: { naming: 3, testing: 2 }, + }), + ]; + + const summary = generateSummary(results); + expect(summary).toContain('naming'); + expect(summary).toContain('8'); + }); + + it('includes unparseable patterns', () => { + const results: PerFileResult[] = [ + makeResult({ + parseableRuleCount: 5, + unparseableLines: ['Use conventional commits', 'Use conventional commits', 'Be nice to people'], + }), + ]; + + const summary = generateSummary(results); + expect(summary).toContain('conventional'); + }); + + it('renders histogram for rule count distribution', () => { + const results: PerFileResult[] = [ + makeResult({ parseableRuleCount: 0 }), + makeResult({ parseableRuleCount: 3 }), + makeResult({ parseableRuleCount: 7 }), + makeResult({ parseableRuleCount: 15 }), + ]; + + const summary = generateSummary(results); + expect(summary).toContain('1-4'); + expect(summary).toContain('5-9'); + }); + + it('handles zero results gracefully', () => { + const summary = generateSummary([]); + expect(summary).toContain('NO-GO'); + expect(summary).toContain('No data'); + }); + }); +}); \ No newline at end of file diff --git a/tests/drift/compareConfigs.test.ts b/tests/drift/compareConfigs.test.ts new file mode 100644 index 0000000..cafbd40 --- /dev/null +++ b/tests/drift/compareConfigs.test.ts @@ -0,0 +1,303 @@ +/** + * Tests for the drift comparison logic. + * + * Validates that compareConfigs correctly identifies md-only rules, + * eslint-only rules, severity mismatches, config-arg mismatches, and + * that unparseable rules from CLAUDE.md are excluded from drift. + */ + +import { describe, it, expect } from 'vitest'; +import { compareConfigs } from '../../src/drift/compareConfigs.js'; +import type { EslintConfig, EslintRuleEntry } from '../../src/mapper/types.js'; +import type { ParsedEslintConfig } from '../../src/drift/types.js'; + +/** Build a minimal EslintConfig for testing. */ +function makeMdConfig(rules: EslintRuleEntry[], overrides?: Partial): EslintConfig { + return { + rules, + unmappable: [], + plugins: [], + sourceFile: 'test-claude.md', + ...overrides, + }; +} + +/** Build a single EslintRuleEntry. */ +function makeMdRule(overrides: Partial & { ruleName: string }): EslintRuleEntry { + return { + severity: 'error', + sourceRuleId: 'test-rule-1', + description: 'test rule', + ...overrides, + }; +} + +/** Build a minimal ParsedEslintConfig for testing. */ +function makeFileConfig(rules: { ruleName: string; severity: 'error' | 'warn' | 'off'; options?: unknown[] }[]): ParsedEslintConfig { + return { + rules: rules.map((r) => ({ + ruleName: r.ruleName, + severity: r.severity, + options: r.options ?? [], + })), + sourceFile: '.eslintrc.json', + }; +} + +describe('compareConfigs', () => { + it('returns empty drift when configs are in sync', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'error' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.items).toHaveLength(0); + expect(result.hasDrift).toBe(false); + }); + + it('detects md-only rules (in CLAUDE.md but not in eslint config)', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + makeMdRule({ + ruleName: '@typescript-eslint/no-explicit-any', + severity: 'error', + sourceRuleId: 'forbidden-no-any-type-1', + description: 'No any type', + }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'error' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(true); + const mdOnly = result.items.filter((i) => i.kind === 'md-only'); + expect(mdOnly).toHaveLength(1); + expect(mdOnly[0].ruleName).toBe('@typescript-eslint/no-explicit-any'); + expect(mdOnly[0].mdRuleId).toBe('forbidden-no-any-type-1'); + expect(mdOnly[0].mdDescription).toBe('No any type'); + }); + + it('detects eslint-only rules (in eslint config but not in CLAUDE.md)', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'error' }, + { ruleName: 'sonarjs/no-identical-conditions', severity: 'error' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(true); + const eslintOnly = result.items.filter((i) => i.kind === 'eslint-only'); + expect(eslintOnly).toHaveLength(1); + expect(eslintOnly[0].ruleName).toBe('sonarjs/no-identical-conditions'); + }); + + it('detects severity mismatches (error vs warn)', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'warn' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(true); + const severityDrift = result.items.filter((i) => i.kind === 'severity-mismatch'); + expect(severityDrift).toHaveLength(1); + expect(severityDrift[0].ruleName).toBe('no-console'); + expect(severityDrift[0].mdSeverity).toBe('error'); + expect(severityDrift[0].eslintSeverity).toBe('warn'); + }); + + it('detects config-arg mismatches (different options)', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ + ruleName: 'max-lines', + severity: 'error', + options: [{ max: 300, skipBlankLines: true, skipComments: true }], + }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'max-lines', severity: 'error', options: [{ max: 500 }] }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(true); + const argDrift = result.items.filter((i) => i.kind === 'config-arg-mismatch'); + expect(argDrift).toHaveLength(1); + expect(argDrift[0].ruleName).toBe('max-lines'); + expect(argDrift[0].mdOptions).toEqual([{ max: 300, skipBlankLines: true, skipComments: true }]); + expect(argDrift[0].eslintOptions).toEqual([{ max: 500 }]); + }); + + it('does not report unparseable md rules as drift', () => { + const mdConfig = makeMdConfig( + [makeMdRule({ ruleName: 'no-console', severity: 'error' })], + { unmappable: [{ sourceRuleId: 'test-files-exist-1', sourceText: 'Every file needs a test', reason: 'No ESLint rule enforces test file existence' }] }, + ); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'error' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(false); + expect(result.items).toHaveLength(0); + }); + + it('detects md-only rules that are disabled (off) in eslint config', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'off' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(true); + const mdOnly = result.items.filter((i) => i.kind === 'md-only'); + expect(mdOnly).toHaveLength(1); + expect(mdOnly[0].ruleName).toBe('no-console'); + }); + + it('reports a rule disabled in eslint but present in md as md-only, not severity-mismatch', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'off' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + const severityDrift = result.items.filter((i) => i.kind === 'severity-mismatch'); + expect(severityDrift).toHaveLength(0); + }); + + it('detects both md-only and eslint-only in the same comparison', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + makeMdRule({ ruleName: 'prefer-const', severity: 'warn' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'error' }, + { ruleName: 'sonarjs/no-identical-conditions', severity: 'error' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(true); + const mdOnly = result.items.filter((i) => i.kind === 'md-only'); + const eslintOnly = result.items.filter((i) => i.kind === 'eslint-only'); + expect(mdOnly).toHaveLength(1); + expect(mdOnly[0].ruleName).toBe('prefer-const'); + expect(eslintOnly).toHaveLength(1); + expect(eslintOnly[0].ruleName).toBe('sonarjs/no-identical-conditions'); + }); + + it('includes source file paths in the result', () => { + const mdConfig = makeMdConfig([]); + const fileConfig = makeFileConfig([]); + + const result = compareConfigs( + { ...mdConfig, sourceFile: 'path/to/CLAUDE.md' }, + { ...fileConfig, sourceFile: 'path/to/.eslintrc.json' }, + ); + expect(result.mdFile).toBe('path/to/CLAUDE.md'); + expect(result.eslintFile).toBe('path/to/.eslintrc.json'); + }); + + it('detects config-arg mismatch when md has options but eslint has none', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ + ruleName: 'max-lines', + severity: 'error', + options: [{ max: 300 }], + }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'max-lines', severity: 'error', options: [] }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + const argDrift = result.items.filter((i) => i.kind === 'config-arg-mismatch'); + expect(argDrift).toHaveLength(1); + }); + + it('reports eslint-only for plugin rules never mentioned in CLAUDE.md', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'no-console', severity: 'error' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'no-console', severity: 'error' }, + { ruleName: 'sonarjs/no-identical-conditions', severity: 'error' }, + { ruleName: 'import/no-cycle', severity: 'warn' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + const eslintOnly = result.items.filter((i) => i.kind === 'eslint-only'); + expect(eslintOnly).toHaveLength(2); + const names = eslintOnly.map((i) => i.ruleName).sort(); + expect(names).toEqual(['import/no-cycle', 'sonarjs/no-identical-conditions']); + }); + + it('detects severity mismatch when md says warn but eslint says error', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ ruleName: 'prefer-const', severity: 'warn' }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'prefer-const', severity: 'error' }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + const severityDrift = result.items.filter((i) => i.kind === 'severity-mismatch'); + expect(severityDrift).toHaveLength(1); + expect(severityDrift[0].mdSeverity).toBe('warn'); + expect(severityDrift[0].eslintSeverity).toBe('error'); + }); + + it('considers same severity and same options as in-sync', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ + ruleName: 'max-lines', + severity: 'error', + options: [{ max: 300 }], + }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'max-lines', severity: 'error', options: [{ max: 300 }] }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + expect(result.hasDrift).toBe(false); + expect(result.items).toHaveLength(0); + }); + + it('does not double-count a rule as both severity and config-arg mismatch', () => { + const mdConfig = makeMdConfig([ + makeMdRule({ + ruleName: 'max-lines', + severity: 'warn', + options: [{ max: 300 }], + }), + ]); + const fileConfig = makeFileConfig([ + { ruleName: 'max-lines', severity: 'error', options: [{ max: 500 }], + }, + ]); + + const result = compareConfigs(mdConfig, fileConfig); + // Should report as config-arg mismatch since options differ (which subsumes severity) + // OR report both separately. The spec says "config-arg mismatch" covers option differences. + // Both severity and options differ, so we report a config-arg-mismatch that includes severity info. + const mismatches = result.items.filter( + (i) => i.kind === 'severity-mismatch' || i.kind === 'config-arg-mismatch', + ); + expect(mismatches).toHaveLength(1); + expect(mismatches[0].kind).toBe('config-arg-mismatch'); + expect(mismatches[0].mdSeverity).toBe('warn'); + expect(mismatches[0].eslintSeverity).toBe('error'); + }); +}); \ No newline at end of file diff --git a/tests/drift/fixtures/eslintrc-basic.json b/tests/drift/fixtures/eslintrc-basic.json new file mode 100644 index 0000000..57a778b --- /dev/null +++ b/tests/drift/fixtures/eslintrc-basic.json @@ -0,0 +1,8 @@ +{ + "rules": { + "no-console": "error", + "prefer-const": "warn", + "max-lines": ["error", { "max": 300 }], + "@typescript-eslint/no-explicit-any": "error" + } +} \ No newline at end of file diff --git a/tests/drift/fixtures/eslintrc-disabled.json b/tests/drift/fixtures/eslintrc-disabled.json new file mode 100644 index 0000000..d00c026 --- /dev/null +++ b/tests/drift/fixtures/eslintrc-disabled.json @@ -0,0 +1,6 @@ +{ + "rules": { + "no-console": "off", + "prefer-const": "warn" + } +} \ No newline at end of file diff --git a/tests/drift/fixtures/eslintrc-empty.json b/tests/drift/fixtures/eslintrc-empty.json new file mode 100644 index 0000000..58db1a2 --- /dev/null +++ b/tests/drift/fixtures/eslintrc-empty.json @@ -0,0 +1,3 @@ +{ + "rules": {} +} \ No newline at end of file diff --git a/tests/drift/fixtures/eslintrc-extended.json b/tests/drift/fixtures/eslintrc-extended.json new file mode 100644 index 0000000..4ab7d7d --- /dev/null +++ b/tests/drift/fixtures/eslintrc-extended.json @@ -0,0 +1,8 @@ +{ + "rules": { + "no-console": ["error", { "allow": ["warn", "error"] }], + "@typescript-eslint/no-explicit-any": ["error", { "fixToUnknown": true }], + "@typescript-eslint/naming-convention": ["error", { "selector": "variable", "format": ["camelCase"] }], + "sonarjs/no-identical-conditions": "error" + } +} \ No newline at end of file diff --git a/tests/drift/formatDriftReport.test.ts b/tests/drift/formatDriftReport.test.ts new file mode 100644 index 0000000..2e980ef --- /dev/null +++ b/tests/drift/formatDriftReport.test.ts @@ -0,0 +1,152 @@ +/** + * Tests for drift report formatting. + * + * Validates text, JSON, and markdown output formats. + */ + +import { describe, it, expect } from 'vitest'; +import { formatDriftReport } from '../../src/drift/formatDriftReport.js'; +import type { DriftResult, DriftItem } from '../../src/drift/types.js'; + +/** Build a minimal DriftItem. */ +function makeDriftItem(overrides: Partial & { kind: DriftItem['kind']; ruleName: string }): DriftItem { + return { message: 'test drift', ...overrides }; +} + +/** Build a minimal DriftResult. */ +function makeDriftResult(items: DriftItem[]): DriftResult { + return { + items, + mdFile: 'CLAUDE.md', + eslintFile: '.eslintrc.json', + hasDrift: items.length > 0, + }; +} + +describe('formatDriftReport', () => { + const mdOnlyItem: DriftItem = makeDriftItem({ + kind: 'md-only', + ruleName: 'no-console', + mdRuleId: 'forbidden-no-console-log-1', + mdDescription: 'No console.log', + mdSeverity: 'error', + message: 'no-console is in CLAUDE.md but not in eslint config', + }); + + const eslintOnlyItem: DriftItem = makeDriftItem({ + kind: 'eslint-only', + ruleName: 'sonarjs/no-identical-conditions', + eslintSeverity: 'error', + message: 'sonarjs/no-identical-conditions is in eslint config but not derived from CLAUDE.md', + }); + + const severityItem: DriftItem = makeDriftItem({ + kind: 'severity-mismatch', + ruleName: 'prefer-const', + mdSeverity: 'warn', + eslintSeverity: 'error', + message: 'prefer-const: CLAUDE.md says warn, eslint says error', + }); + + const configArgItem: DriftItem = makeDriftItem({ + kind: 'config-arg-mismatch', + ruleName: 'max-lines', + mdSeverity: 'error', + eslintSeverity: 'error', + mdOptions: [{ max: 300 }], + eslintOptions: [{ max: 500 }], + message: 'max-lines: CLAUDE.md says [{"max":300}], eslint says [{"max":500}]', + }); + + describe('text format', () => { + it('reports no drift when items are empty', () => { + const result = makeDriftResult([]); + const output = formatDriftReport(result, 'text'); + expect(output).toContain('No drift detected'); + expect(output).toContain('CLAUDE.md'); + expect(output).toContain('.eslintrc.json'); + }); + + it('formats md-only items', () => { + const result = makeDriftResult([mdOnlyItem]); + const output = formatDriftReport(result, 'text'); + expect(output).toContain('md-only'); + expect(output).toContain('no-console'); + expect(output).toContain('No console.log'); + }); + + it('formats eslint-only items', () => { + const result = makeDriftResult([eslintOnlyItem]); + const output = formatDriftReport(result, 'text'); + expect(output).toContain('eslint-only'); + expect(output).toContain('sonarjs/no-identical-conditions'); + }); + + it('formats severity mismatches', () => { + const result = makeDriftResult([severityItem]); + const output = formatDriftReport(result, 'text'); + expect(output).toContain('severity-mismatch'); + expect(output).toContain('prefer-const'); + }); + + it('formats config-arg mismatches', () => { + const result = makeDriftResult([configArgItem]); + const output = formatDriftReport(result, 'text'); + expect(output).toContain('config-arg-mismatch'); + expect(output).toContain('max-lines'); + }); + + it('includes a summary line with counts', () => { + const result = makeDriftResult([mdOnlyItem, eslintOnlyItem, severityItem, configArgItem]); + const output = formatDriftReport(result, 'text'); + expect(output).toContain('1 md-only'); + expect(output).toContain('1 eslint-only'); + expect(output).toContain('1 severity-mismatch'); + expect(output).toContain('1 config-arg-mismatch'); + }); + }); + + describe('json format', () => { + it('produces valid JSON', () => { + const result = makeDriftResult([mdOnlyItem]); + const output = formatDriftReport(result, 'json'); + const parsed = JSON.parse(output); + expect(parsed.items).toHaveLength(1); + expect(parsed.hasDrift).toBe(true); + }); + + it('includes all drift fields', () => { + const result = makeDriftResult([configArgItem]); + const output = formatDriftReport(result, 'json'); + const parsed = JSON.parse(output); + expect(parsed.items[0].kind).toBe('config-arg-mismatch'); + expect(parsed.items[0].mdOptions).toEqual([{ max: 300 }]); + expect(parsed.items[0].eslintOptions).toEqual([{ max: 500 }]); + }); + }); + + describe('markdown format', () => { + it('reports no drift when items are empty', () => { + const result = makeDriftResult([]); + const output = formatDriftReport(result, 'markdown'); + expect(output).toContain('No drift detected'); + }); + + it('formats a markdown table with drift items', () => { + const result = makeDriftResult([mdOnlyItem, eslintOnlyItem]); + const output = formatDriftReport(result, 'markdown'); + expect(output).toContain('md-only'); + expect(output).toContain('eslint-only'); + expect(output).toContain('no-console'); + expect(output).toContain('sonarjs/no-identical-conditions'); + }); + + it('includes severity and options in markdown', () => { + const result = makeDriftResult([configArgItem]); + const output = formatDriftReport(result, 'markdown'); + expect(output).toContain('max-lines'); + expect(output).toContain('300'); + expect(output).toContain('500'); + }); + }); +}); \ No newline at end of file diff --git a/tests/drift/parseEslintConfig.test.ts b/tests/drift/parseEslintConfig.test.ts new file mode 100644 index 0000000..867c08c --- /dev/null +++ b/tests/drift/parseEslintConfig.test.ts @@ -0,0 +1,156 @@ +/** + * Tests for ESLint config file parsing. + * + * Validates that parseEslintConfig correctly reads .eslintrc.json + * files and extracts rule entries with severity and options. + */ + +import { describe, it, expect } from 'vitest'; +import { parseEslintConfig } from '../../src/drift/parseEslintConfig.js'; +import { join } from 'node:path'; +import { writeFileSync, unlinkSync } from 'node:fs'; + +const fixturesDir = join(import.meta.dirname, 'fixtures'); + +describe('parseEslintConfig', () => { + it('parses a basic .eslintrc.json with severity-only rules', () => { + const result = parseEslintConfig( + join(fixturesDir, 'eslintrc-basic.json'), + ); + expect(result.sourceFile).toContain('eslintrc-basic.json'); + expect(result.rules).toHaveLength(4); + + const consoleRule = result.rules.find((r) => r.ruleName === 'no-console'); + expect(consoleRule).toBeDefined(); + expect(consoleRule!.severity).toBe('error'); + expect(consoleRule!.options).toEqual([]); + + const constRule = result.rules.find((r) => r.ruleName === 'prefer-const'); + expect(constRule).toBeDefined(); + expect(constRule!.severity).toBe('warn'); + expect(constRule!.options).toEqual([]); + + const maxLines = result.rules.find((r) => r.ruleName === 'max-lines'); + expect(maxLines).toBeDefined(); + expect(maxLines!.severity).toBe('error'); + expect(maxLines!.options).toEqual([{ max: 300 }]); + + const noAny = result.rules.find((r) => r.ruleName === '@typescript-eslint/no-explicit-any'); + expect(noAny).toBeDefined(); + expect(noAny!.severity).toBe('error'); + expect(noAny!.options).toEqual([]); + }); + + it('parses a config with off (disabled) rules', () => { + const result = parseEslintConfig( + join(fixturesDir, 'eslintrc-disabled.json'), + ); + expect(result.rules).toHaveLength(2); + + const consoleRule = result.rules.find((r) => r.ruleName === 'no-console'); + expect(consoleRule!.severity).toBe('off'); + + const constRule = result.rules.find((r) => r.ruleName === 'prefer-const'); + expect(constRule!.severity).toBe('warn'); + }); + + it('parses a config with plugin rules and extended options', () => { + const result = parseEslintConfig( + join(fixturesDir, 'eslintrc-extended.json'), + ); + expect(result.rules).toHaveLength(4); + + const namingRule = result.rules.find( + (r) => r.ruleName === '@typescript-eslint/naming-convention', + ); + expect(namingRule).toBeDefined(); + expect(namingRule!.severity).toBe('error'); + expect(namingRule!.options).toEqual([{ selector: 'variable', format: ['camelCase'] }]); + + const sonarRule = result.rules.find( + (r) => r.ruleName === 'sonarjs/no-identical-conditions', + ); + expect(sonarRule).toBeDefined(); + expect(sonarRule!.severity).toBe('error'); + }); + + it('parses an empty rules config', () => { + const result = parseEslintConfig( + join(fixturesDir, 'eslintrc-empty.json'), + ); + expect(result.rules).toHaveLength(0); + }); + + it('throws on a nonexistent file', () => { + expect(() => + parseEslintConfig(join(fixturesDir, 'nonexistent.json')), + ).toThrow(); + }); + + it('throws on an unparseable JSON file', () => { + const invalidPath = join(fixturesDir, 'eslintrc-invalid.json'); + writeFileSync(invalidPath, '{ invalid json !!!'); + try { + expect(() => parseEslintConfig(invalidPath)).toThrow(); + } finally { + unlinkSync(invalidPath); + } + }); + + it('handles numeric severity values (0, 1, 2)', () => { + const numericPath = join(fixturesDir, 'eslintrc-numeric.json'); + writeFileSync(numericPath, JSON.stringify({ + rules: { + 'no-console': 2, + 'prefer-const': 1, + 'no-var': 0, + }, + })); + try { + const result = parseEslintConfig(numericPath); + const consoleRule = result.rules.find((r) => r.ruleName === 'no-console'); + expect(consoleRule!.severity).toBe('error'); + const constRule = result.rules.find((r) => r.ruleName === 'prefer-const'); + expect(constRule!.severity).toBe('warn'); + const varRule = result.rules.find((r) => r.ruleName === 'no-var'); + expect(varRule!.severity).toBe('off'); + } finally { + unlinkSync(numericPath); + } + }); + + it('parses rules with arrays where severity is the first element', () => { + const arrayPath = join(fixturesDir, 'eslintrc-array-severity.json'); + writeFileSync(arrayPath, JSON.stringify({ + rules: { + 'no-console': ['warn', { allow: ['warn', 'error'] }], + }, + })); + try { + const result = parseEslintConfig(arrayPath); + const consoleRule = result.rules.find((r) => r.ruleName === 'no-console'); + expect(consoleRule!.severity).toBe('warn'); + expect(consoleRule!.options).toEqual([{ allow: ['warn', 'error'] }]); + } finally { + unlinkSync(arrayPath); + } + }); + + it('parses a flat config JSON array', () => { + const flatPath = join(fixturesDir, 'eslint-config-flat.json'); + writeFileSync(flatPath, JSON.stringify([ + { rules: { 'no-console': 'error' } }, + { rules: { 'prefer-const': 'warn' } }, + ])); + try { + const result = parseEslintConfig(flatPath); + expect(result.rules).toHaveLength(2); + const consoleRule = result.rules.find((r) => r.ruleName === 'no-console'); + expect(consoleRule!.severity).toBe('error'); + const constRule = result.rules.find((r) => r.ruleName === 'prefer-const'); + expect(constRule!.severity).toBe('warn'); + } finally { + unlinkSync(flatPath); + } + }); +}); \ No newline at end of file diff --git a/tests/emitter/eslint.test.ts b/tests/emitter/eslint.test.ts new file mode 100644 index 0000000..449eb14 --- /dev/null +++ b/tests/emitter/eslint.test.ts @@ -0,0 +1,296 @@ +/** + * Tests for the ESLint config emitter. + * + * Validates that EslintConfig objects are correctly serialized to + * both flat and legacy ESLint config formats, that generated configs + * parse and validate correctly, and that unmappable rules appear + * as commented sections with reasons. + */ + +import { describe, it, expect } from 'vitest'; +import { emitEslintConfig } from '../../src/emitter/eslint.js'; +import type { EslintConfig } from '../../src/mapper/types.js'; + +function makeConfig(overrides: Partial = {}): EslintConfig { + return { + rules: [], + unmappable: [], + plugins: [], + sourceFile: 'CLAUDE.md', + ...overrides, + }; +} + +describe('emitEslintConfig', () => { + describe('flat config format', () => { + it('emits a valid flat config with no rules', () => { + const config = makeConfig(); + const output = emitEslintConfig(config, 'flat'); + expect(output).toContain('export default ['); + expect(output).toContain('rules: {'); + expect(output).toContain('Source: CLAUDE.md'); + }); + + it('emits rules as object properties with string severity values', () => { + const config = makeConfig({ + rules: [ + { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'error', + sourceRuleId: 'forbidden-no-any-type-1', + description: 'The "any" type must not be used', + }, + { + ruleName: 'prefer-const', + severity: 'warn', + options: [{ destructuring: 'all' }], + sourceRuleId: 'style-prefer-const-1', + description: 'Prefer const', + }, + ], + plugins: ['@typescript-eslint'], + }); + const output = emitEslintConfig(config, 'flat'); + + // Rules must be object properties with quoted keys, not array entries + expect(output).toContain("'@typescript-eslint/no-explicit-any': ["); + expect(output).toContain("'prefer-const': ["); + + // Severity must be a string value, not a bare identifier + expect(output).toContain("'error'"); + expect(output).toContain("'warn'"); + + // Options must be present + expect(output).toContain('destructuring'); + }); + + it('emits plugin imports and plugin object for flat config', () => { + const config = makeConfig({ + rules: [ + { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'error', + sourceRuleId: 'test-1', + description: 'test', + }, + ], + plugins: ['@typescript-eslint'], + }); + const output = emitEslintConfig(config, 'flat'); + expect(output).toContain('@typescript-eslint/eslint-plugin'); + expect(output).toContain('plugins: {'); + expect(output).toContain("'@typescript-eslint': _typescript_eslintPlugin"); + }); + + it('emits unmappable rules as commented sections', () => { + const config = makeConfig({ + unmappable: [ + { + sourceRuleId: 'test-files-exist-1', + sourceText: 'Every source file must have a corresponding test file', + reason: 'No ESLint rule enforces test file existence', + }, + ], + }); + const output = emitEslintConfig(config, 'flat'); + expect(output).toContain('Unmappable rules'); + expect(output).toContain('test-files-exist-1'); + expect(output).toContain('No ESLint rule enforces test file existence'); + expect(output).toContain('Every source file must have a corresponding test file'); + }); + + it('emits numeric options correctly', () => { + const config = makeConfig({ + rules: [ + { + ruleName: 'max-lines', + severity: 'warn', + options: [{ max: 300, skipBlankLines: true, skipComments: true }], + sourceRuleId: 'structure-max-file-length-1', + description: 'Max file length', + }, + ], + }); + const output = emitEslintConfig(config, 'flat'); + expect(output).toContain('300'); + expect(output).toContain('skipBlankLines'); + }); + + it('produces syntactically valid flat config JavaScript', () => { + const config = makeConfig({ + rules: [ + { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'error', + sourceRuleId: 'test-1', + description: 'No any', + }, + { + ruleName: 'no-console', + severity: 'warn', + sourceRuleId: 'test-2', + description: 'No console', + }, + ], + plugins: ['@typescript-eslint'], + }); + const output = emitEslintConfig(config, 'flat'); + + // The output must be parseable as ES module JS + // Check for required structural elements + expect(output).toMatch(/import\s+\w+Plugin\s+from\s+['"]/); + expect(output).toContain('export default ['); + expect(output).toContain('plugins: {'); + expect(output).toContain('rules: {'); + + // Rule entries must use object property syntax, not array syntax + expect(output).toContain("'@typescript-eslint/no-explicit-any': ["); + expect(output).toContain("'no-console': ["); + + // Severity must be a quoted string, not a bare identifier + expect(output).toContain("'error'"); + expect(output).toContain("'warn'"); + }); + + it('handles multiple plugins in flat config', () => { + const config = makeConfig({ + rules: [ + { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'error', + sourceRuleId: 'test-1', + description: 'test', + }, + { + ruleName: 'import/no-namespace', + plugin: 'import', + severity: 'warn', + sourceRuleId: 'test-2', + description: 'test', + }, + ], + plugins: ['@typescript-eslint', 'import'], + }); + const output = emitEslintConfig(config, 'flat'); + expect(output).toContain('@typescript-eslint/eslint-plugin'); + expect(output).toContain('eslint-plugin-import'); + expect(output).toContain('_typescript_eslintPlugin'); + expect(output).toContain('importPlugin'); + }); + }); + + describe('legacy config format', () => { + it('emits valid JSON for .eslintrc.json with no rules', () => { + const config = makeConfig(); + const output = emitEslintConfig(config, 'legacy'); + // Must be valid JSON + const parsed = JSON.parse(output); + expect(parsed).toHaveProperty('rules'); + expect(parsed.rules).toEqual({}); + }); + + it('emits rules with severity and options in legacy format', () => { + const config = makeConfig({ + rules: [ + { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'error', + sourceRuleId: 'forbidden-no-any-type-1', + description: 'No any type', + }, + ], + plugins: ['@typescript-eslint'], + }); + const output = emitEslintConfig(config, 'legacy'); + const parsed = JSON.parse(output); + expect(parsed.rules['@typescript-eslint/no-explicit-any']).toBe('error'); + expect(parsed.plugins).toEqual(['@typescript-eslint']); + expect(parsed.extends).toEqual([ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + ]); + }); + + it('emits rules with options in legacy format', () => { + const config = makeConfig({ + rules: [ + { + ruleName: 'max-lines', + severity: 'warn', + options: [{ max: 300, skipBlankLines: true }], + sourceRuleId: 'structure-max-file-length-1', + description: 'Max file length', + }, + ], + }); + const output = emitEslintConfig(config, 'legacy'); + const parsed = JSON.parse(output); + expect(parsed.rules['max-lines']).toEqual(['warn', { max: 300, skipBlankLines: true }]); + }); + + it('produces valid JSON that can be parsed by JSON.parse', () => { + const config = makeConfig({ + rules: [ + { + ruleName: '@typescript-eslint/no-explicit-any', + plugin: '@typescript-eslint', + severity: 'error', + sourceRuleId: 'test-1', + description: 'No any', + }, + { + ruleName: 'no-console', + severity: 'warn', + sourceRuleId: 'test-2', + description: 'No console', + }, + { + ruleName: 'prefer-const', + severity: 'warn', + options: [{ destructuring: 'all' }], + sourceRuleId: 'test-3', + description: 'Prefer const', + }, + ], + plugins: ['@typescript-eslint'], + }); + const output = emitEslintConfig(config, 'legacy'); + + // Must be valid JSON - no trailing commas, no comments, no bare identifiers + const parsed = JSON.parse(output); + expect(Object.keys(parsed.rules)).toHaveLength(3); + expect(parsed.rules['@typescript-eslint/no-explicit-any']).toBe('error'); + expect(parsed.rules['no-console']).toBe('warn'); + expect(parsed.rules['prefer-const']).toEqual(['warn', { destructuring: 'all' }]); + }); + + it('emits unmappable rules in a comment block after the JSON', () => { + const config = makeConfig({ + unmappable: [ + { + sourceRuleId: 'strict-mode-1', + sourceText: 'Use TypeScript strict mode', + reason: 'TypeScript strict mode is a tsconfig setting', + }, + ], + }); + const output = emitEslintConfig(config, 'legacy'); + + // Find the JSON portion (ends before the unmappable comment block, or at end) + const commentMarker = '// Unmappable rules'; + const commentStart = output.indexOf(commentMarker); + const jsonPart = commentStart >= 0 ? output.substring(0, commentStart).trimEnd() : output; + const parsed = JSON.parse(jsonPart); + expect(parsed).toHaveProperty('rules'); + + // The unmappable section must appear after the JSON + expect(output).toContain('// [strict-mode-1]'); + expect(output).toContain('TypeScript strict mode is a tsconfig setting'); + }); + }); +}); \ No newline at end of file diff --git a/tests/extractor/extract.test.ts b/tests/extractor/extract.test.ts new file mode 100644 index 0000000..9a56896 --- /dev/null +++ b/tests/extractor/extract.test.ts @@ -0,0 +1,285 @@ +/** + * Tests for the ESLint config to CLAUDE.md rules extractor. + * + * Validates that extractRules correctly reverse-maps ESLint config + * entries to prose instructions, skips stylistic rules, handles + * unknown rules, and formats markdown output. + */ + +import { describe, it, expect } from 'vitest'; +import { parseEslintConfig } from '../../src/drift/parseEslintConfig.js'; +import { extractRules, formatRulesMarkdown } from '../../src/extractor/index.js'; +import type { ParsedEslintConfig } from '../../src/drift/types.js'; +import { join } from 'node:path'; + +const fixturesDir = join(import.meta.dirname, 'fixtures'); + +describe('extractRules', () => { + it('extracts prose for basic mappable rules', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-basic.json')); + const result = extractRules(config); + + expect(result.rules.length).toBeGreaterThan(0); + + const noAny = result.rules.find((r) => r.eslintRuleName === '@typescript-eslint/no-explicit-any'); + expect(noAny).toBeDefined(); + expect(noAny!.prose).toContain('`any`'); + expect(noAny!.patternType).toBe('no-any'); + + const noConsole = result.rules.find((r) => r.eslintRuleName === 'no-console'); + expect(noConsole).toBeDefined(); + expect(noConsole!.prose).toContain('console'); + + const noVar = result.rules.find((r) => r.eslintRuleName === 'no-var'); + expect(noVar).toBeDefined(); + expect(noVar!.prose).toContain('`var`'); + }); + + it('skips stylistic rules with no prose equivalent', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-stylistic.json')); + const result = extractRules(config); + + const stylisticRules = result.skipped.filter((s) => s.reason === 'stylistic'); + expect(stylisticRules.length).toBeGreaterThanOrEqual(2); + + const semiSkip = result.skipped.find((s) => s.eslintRuleName === 'semi'); + expect(semiSkip).toBeDefined(); + expect(semiSkip!.reason).toBe('stylistic'); + + const quotesSkip = result.skipped.find((s) => s.eslintRuleName === 'quotes'); + expect(quotesSkip).toBeDefined(); + expect(quotesSkip!.reason).toBe('stylistic'); + + // Non-stylistic rules should still be extracted + const noVar = result.rules.find((r) => r.eslintRuleName === 'no-var'); + expect(noVar).toBeDefined(); + }); + + it('handles naming convention rules with complex options', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-naming.json')); + const result = extractRules(config); + + const namingRules = result.rules.filter((r) => r.eslintRuleName === '@typescript-eslint/naming-convention'); + expect(namingRules.length).toBeGreaterThanOrEqual(2); + + const pascalRule = namingRules.find((r) => r.prose.includes('PascalCase')); + expect(pascalRule).toBeDefined(); + + const camelRule = namingRules.find((r) => r.prose.includes('camelCase')); + expect(camelRule).toBeDefined(); + + // Each selector should produce distinct prose + const proseValues = namingRules.map((r) => r.prose); + const uniqueProse = new Set(proseValues); + expect(uniqueProse.size).toBe(proseValues.length); + }); + + it('skips unknown rules with no mapping', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-unknown.json')); + const result = extractRules(config); + + const unknownSkips = result.skipped.filter((s) => s.reason === 'no-mapping'); + expect(unknownSkips.length).toBe(2); + + const unknown1 = result.skipped.find((s) => s.eslintRuleName === 'some-plugin/unknown-rule'); + expect(unknown1).toBeDefined(); + expect(unknown1!.reason).toBe('no-mapping'); + + // Known rules should still be extracted + const noConsole = result.rules.find((r) => r.eslintRuleName === 'no-console'); + expect(noConsole).toBeDefined(); + }); + + it('skips disabled (off) rules', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-off.json')); + const result = extractRules(config); + + // no-console is "off" and should not appear in extracted rules + const noConsole = result.rules.find((r) => r.eslintRuleName === 'no-console'); + expect(noConsole).toBeUndefined(); + + // no-var should still be extracted + const noVar = result.rules.find((r) => r.eslintRuleName === 'no-var'); + expect(noVar).toBeDefined(); + }); + + it('extracts prose with interpolated config args', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-with-args.json')); + const result = extractRules(config); + + const maxLines = result.rules.find((r) => r.eslintRuleName === 'max-lines'); + expect(maxLines).toBeDefined(); + expect(maxLines!.prose).toContain('300'); + + const maxLen = result.rules.find((r) => r.eslintRuleName === 'max-len'); + expect(maxLen).toBeDefined(); + expect(maxLen!.prose).toContain('120'); + + const maxParams = result.rules.find((r) => r.eslintRuleName === 'max-params'); + expect(maxParams).toBeDefined(); + expect(maxParams!.prose).toContain('4'); + + const maxLinesPerFn = result.rules.find((r) => r.eslintRuleName === 'max-lines-per-function'); + expect(maxLinesPerFn).toBeDefined(); + expect(maxLinesPerFn!.prose).toContain('50'); + + const todoComments = result.rules.find((r) => r.eslintRuleName === 'no-warning-comments'); + expect(todoComments).toBeDefined(); + expect(todoComments!.prose).toContain('todo'); + expect(todoComments!.prose).toContain('fixme'); + }); + + it('returns the source file path', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-basic.json')); + const result = extractRules(config); + expect(result.sourceFile).toContain('eslintrc-basic.json'); + }); +}); + +describe('formatRulesMarkdown', () => { + it('formats extracted rules as markdown with heading and bullets', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-basic.json')); + const result = extractRules(config); + const markdown = formatRulesMarkdown(result); + + expect(markdown).toContain('## Rules'); + expect(markdown).toContain('- '); + expect(markdown).toContain('`any`'); + }); + + it('includes skipped rules in an HTML comment block', () => { + const config = parseEslintConfig(join(fixturesDir, 'eslintrc-stylistic.json')); + const result = extractRules(config); + const markdown = formatRulesMarkdown(result); + + expect(markdown).toContain(''); + }); + + it('does not include skipped block when no rules are skipped', () => { + const config: ParsedEslintConfig = { + rules: [{ ruleName: 'no-var', severity: 'error', options: [] }], + sourceFile: 'test.json', + }; + const result = extractRules(config); + const markdown = formatRulesMarkdown(result); + + expect(markdown).toContain('## Rules'); + expect(markdown).not.toContain('