Use the workflow

This doesn't use the published version, but instead compiles the current
version which is, itself, a form of QA

Author: slifty

.github/workflows/qa.yml

@@ -0,0 +1,33 @@
+name: QA Instructions
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+permissions:
+  pull-requests: write
+  models: read
+
+jobs:
+  qa-instructions:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: ".node-version"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Generate QA Instructions
+        uses: ./
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

src/constants.ts

@@ -9,11 +9,29 @@ export const DEFAULT_GITHUB_MODELS_MODEL = "openai/gpt-4o";
 export const GITHUB_MODELS_BASE_URL =
 	"https://models.github.ai/inference/chat/completions";
 
-export const MAX_DIFF_CHARS = 80_000;
-export const MAX_CHANGED_FILES_CHARS = 60_000;
-export const MAX_FILE_CHARS = 10_000;
-export const MAX_FILE_TREE_CHARS = 20_000;
-export const MAX_TOTAL_CHARS = 180_000;
+export interface ContextLimits {
+	maxDiffChars: number;
+	maxChangedFilesChars: number;
+	maxFileChars: number;
+	maxFileTreeChars: number;
+	maxTotalChars: number;
+}
+
+export const ANTHROPIC_CONTEXT_LIMITS: ContextLimits = {
+	maxDiffChars: 80_000,
+	maxChangedFilesChars: 60_000,
+	maxFileChars: 10_000,
+	maxFileTreeChars: 20_000,
+	maxTotalChars: 180_000,
+};
+
+export const GITHUB_MODELS_CONTEXT_LIMITS: ContextLimits = {
+	maxDiffChars: 16_000,
+	maxChangedFilesChars: 12_000,
+	maxFileChars: 5_000,
+	maxFileTreeChars: 5_000,
+	maxTotalChars: 40_000,
+};
 
 export const SYSTEM_PROMPT = `You are an expert QA engineer reviewing a pull request. Your job is to generate clear, actionable QA testing instructions that a human tester can follow.
 

src/context-builder.test.ts

@@ -1,7 +1,9 @@
 import { describe, it, expect } from "vitest";
 import { buildPromptContext } from "./context-builder.js";
 import type { PrData } from "./types.js";
-import { MAX_DIFF_CHARS, MAX_FILE_CHARS } from "./constants.js";
+import { ANTHROPIC_CONTEXT_LIMITS } from "./constants.js";
+
+const limits = ANTHROPIC_CONTEXT_LIMITS;
 
 function makePrData(overrides: Partial<PrData> = {}): PrData {
 	return {
@@ -16,7 +18,7 @@ function makePrData(overrides: Partial<PrData> = {}): PrData {
 
 describe("buildPromptContext", () => {
 	it("includes PR title and description", () => {
-		const result = buildPromptContext(makePrData(), "");
+		const result = buildPromptContext(makePrData(), "", limits);
 		expect(result).toContain("**Title:** Test PR");
 		expect(result).toContain("Test body");
 	});
@@ -25,7 +27,7 @@ describe("buildPromptContext", () => {
 		const data = makePrData({
 			metadata: { title: "Test", body: "", headSha: "abc" },
 		});
-		const result = buildPromptContext(data, "");
+		const result = buildPromptContext(data, "", limits);
 		expect(result).not.toContain("**Description:**");
 	});
 
@@ -36,25 +38,29 @@ describe("buildPromptContext", () => {
 				{ sha: "1234567abcdef", message: "Fix bug" },
 			],
 		});
-		const result = buildPromptContext(data, "");
+		const result = buildPromptContext(data, "", limits);
 		expect(result).toContain("## Commits");
 		expect(result).toContain("- abcdef1 Initial commit");
 		expect(result).toContain("- 1234567 Fix bug");
 	});
 
 	it("includes diff content", () => {
 		const data = makePrData({ diff: "+added line\n-removed line" });
-		const result = buildPromptContext(data, "");
+		const result = buildPromptContext(data, "", limits);
 		expect(result).toContain("## Diff");
 		expect(result).toContain("+added line\n-removed line");
 	});
 
 	it("truncates long diffs at line boundaries", () => {
 		const lines = Array.from({ length: 10000 }, (_, i) => `+line ${i}`);
 		const longDiff = lines.join("\n");
-		expect(longDiff.length).toBeGreaterThan(MAX_DIFF_CHARS);
+		expect(longDiff.length).toBeGreaterThan(limits.maxDiffChars);
 
-		const result = buildPromptContext(makePrData({ diff: longDiff }), "");
+		const result = buildPromptContext(
+			makePrData({ diff: longDiff }),
+			"",
+			limits,
+		);
 		expect(result).toContain("[Content truncated]");
 	});
 
@@ -65,15 +71,15 @@ describe("buildPromptContext", () => {
 				{ filename: "big.ts", content: "x".repeat(500) },
 			],
 		});
-		const result = buildPromptContext(data, "");
+		const result = buildPromptContext(data, "", limits);
 		expect(result).toContain("## Changed File Contents");
 		// Big file should appear before small file
 		const bigIdx = result.indexOf("big.ts");
 		const smallIdx = result.indexOf("small.ts");
 		expect(bigIdx).toBeLessThan(smallIdx);
 	});
 
-	it("truncates individual files exceeding MAX_FILE_CHARS", () => {
+	it("truncates individual files exceeding maxFileChars", () => {
 		const data = makePrData({
 			changedFiles: [
 				{
@@ -84,21 +90,22 @@ describe("buildPromptContext", () => {
 				},
 			],
 		});
-		// The content is larger than MAX_FILE_CHARS
-		expect(data.changedFiles[0].content.length).toBeGreaterThan(MAX_FILE_CHARS);
+		expect(data.changedFiles[0].content.length).toBeGreaterThan(
+			limits.maxFileChars,
+		);
 
-		const result = buildPromptContext(data, "");
+		const result = buildPromptContext(data, "", limits);
 		expect(result).toContain("[Content truncated]");
 	});
 
-	it("stops adding files when total exceeds MAX_CHANGED_FILES_CHARS", () => {
+	it("stops adding files when total exceeds maxChangedFilesChars", () => {
 		const files = Array.from({ length: 20 }, (_, i) => ({
 			filename: `file${i}.ts`,
-			content: "x".repeat(MAX_FILE_CHARS - 100),
+			content: "x".repeat(limits.maxFileChars - 100),
 		}));
 		const data = makePrData({ changedFiles: files });
 
-		const result = buildPromptContext(data, "");
+		const result = buildPromptContext(data, "", limits);
 
 		// Not all 20 files should be included
 		const includedCount = (result.match(/### file\d+\.ts/g) || []).length;
@@ -110,7 +117,7 @@ describe("buildPromptContext", () => {
 		const data = makePrData({
 			fileTree: ["src/index.ts", "src/utils.ts", "package.json"],
 		});
-		const result = buildPromptContext(data, "");
+		const result = buildPromptContext(data, "", limits);
 		expect(result).toContain("## Repository File Tree");
 		expect(result).toContain("src/index.ts");
 	});
@@ -119,26 +126,25 @@ describe("buildPromptContext", () => {
 		const result = buildPromptContext(
 			makePrData(),
 			"Focus on accessibility testing",
+			limits,
 		);
 		expect(result).toContain("## Additional Instructions");
 		expect(result).toContain("Focus on accessibility testing");
 	});
 
 	it("omits additional instructions when custom prompt is empty", () => {
-		const result = buildPromptContext(makePrData(), "");
+		const result = buildPromptContext(makePrData(), "", limits);
 		expect(result).not.toContain("## Additional Instructions");
 	});
 
 	it("respects overall character cap", () => {
-		// Create data that would exceed MAX_TOTAL_CHARS
+		// Create data that would exceed maxTotalChars
 		const data = makePrData({
 			diff: "x".repeat(80_000),
 			changedFiles: [{ filename: "big.ts", content: "y".repeat(60_000) }],
 			fileTree: Array.from({ length: 5000 }, (_, i) => `path/${i}.ts`),
 		});
-		const result = buildPromptContext(data, "");
-		// The result includes truncation markers, so it will be slightly over
-		// but the core content should be capped
+		const result = buildPromptContext(data, "", limits);
 		expect(result).toContain("[Content truncated]");
 	});
 });

src/context-builder.ts

@@ -1,11 +1,5 @@
 import type { PrData } from "./types.js";
-import {
-	MAX_DIFF_CHARS,
-	MAX_CHANGED_FILES_CHARS,
-	MAX_FILE_CHARS,
-	MAX_FILE_TREE_CHARS,
-	MAX_TOTAL_CHARS,
-} from "./constants.js";
+import type { ContextLimits } from "./constants.js";
 
 function truncateAtLineBreak(text: string, maxChars: number): string {
 	if (text.length <= maxChars) return text;
@@ -15,7 +9,11 @@ function truncateAtLineBreak(text: string, maxChars: number): string {
 	return truncated.slice(0, cutPoint) + "\n\n[Content truncated]";
 }
 
-export function buildPromptContext(data: PrData, customPrompt: string): string {
+export function buildPromptContext(
+	data: PrData,
+	customPrompt: string,
+	limits: ContextLimits,
+): string {
 	const sections: string[] = [];
 
 	// Pr title + description + commits (always included in full)
@@ -31,13 +29,13 @@ export function buildPromptContext(data: PrData, customPrompt: string): string {
 		sections.push(`## Commits\n\n${commitLines}`);
 	}
 
-	// Diff (up to MAX_DIFF_CHARS)
+	// Diff (up to maxDiffChars)
 	if (data.diff) {
-		const truncatedDiff = truncateAtLineBreak(data.diff, MAX_DIFF_CHARS);
+		const truncatedDiff = truncateAtLineBreak(data.diff, limits.maxDiffChars);
 		sections.push(`## Diff\n\n\`\`\`diff\n${truncatedDiff}\n\`\`\``);
 	}
 
-	// Changed file contents (up to MAX_CHANGED_FILES_CHARS total, MAX_FILE_CHARS per file)
+	// Changed file contents (up to maxChangedFilesChars total, maxFileChars per file)
 	// Sort by content length descending (most-changed files first)
 	if (data.changedFiles.length > 0) {
 		const sorted = [...data.changedFiles].sort(
@@ -47,10 +45,10 @@ export function buildPromptContext(data: PrData, customPrompt: string): string {
 		let totalFileChars = 0;
 
 		for (const file of sorted) {
-			if (totalFileChars >= MAX_CHANGED_FILES_CHARS) break;
+			if (totalFileChars >= limits.maxChangedFilesChars) break;
 
-			const remaining = MAX_CHANGED_FILES_CHARS - totalFileChars;
-			const perFileLimit = Math.min(MAX_FILE_CHARS, remaining);
+			const remaining = limits.maxChangedFilesChars - totalFileChars;
+			const perFileLimit = Math.min(limits.maxFileChars, remaining);
 			const content = truncateAtLineBreak(file.content, perFileLimit);
 
 			fileBlocks.push(`### ${file.filename}\n\n\`\`\`\n${content}\n\`\`\``);
@@ -60,10 +58,13 @@ export function buildPromptContext(data: PrData, customPrompt: string): string {
 		sections.push(`## Changed File Contents\n\n${fileBlocks.join("\n\n")}`);
 	}
 
-	// File tree (lowest priority, up to MAX_FILE_TREE_CHARS)
+	// File tree (lowest priority, up to maxFileTreeChars)
 	if (data.fileTree.length > 0) {
 		const treeText = data.fileTree.join("\n");
-		const truncatedTree = truncateAtLineBreak(treeText, MAX_FILE_TREE_CHARS);
+		const truncatedTree = truncateAtLineBreak(
+			treeText,
+			limits.maxFileTreeChars,
+		);
 		sections.push(
 			`## Repository File Tree\n\n\`\`\`\n${truncatedTree}\n\`\`\``,
 		);
@@ -77,8 +78,8 @@ export function buildPromptContext(data: PrData, customPrompt: string): string {
 	let result = sections.join("\n\n");
 
 	// Overall cap
-	if (result.length > MAX_TOTAL_CHARS) {
-		result = truncateAtLineBreak(result, MAX_TOTAL_CHARS);
+	if (result.length > limits.maxTotalChars) {
+		result = truncateAtLineBreak(result, limits.maxTotalChars);
 	}
 
 	return result;

src/index.test.ts

@@ -20,6 +20,7 @@ import * as ghModule from "./github.js";
 import * as contextBuilder from "./context-builder.js";
 import * as providerFactory from "./provider-factory.js";
 import * as comment from "./comment.js";
+import { GITHUB_MODELS_CONTEXT_LIMITS } from "./constants.js";
 
 const mockGenerateQAInstructions = vi.fn();
 
@@ -46,6 +47,9 @@ describe("run", () => {
 		vi.mocked(providerFactory.createProvider).mockReturnValue({
 			generateQAInstructions: mockGenerateQAInstructions,
 		});
+		vi.mocked(providerFactory.getContextLimits).mockReturnValue(
+			GITHUB_MODELS_CONTEXT_LIMITS,
+		);
 
 		vi.mocked(ghModule.getPrMetadata).mockResolvedValue({
 			title: "Test PR",
@@ -75,6 +79,9 @@ describe("run", () => {
 			githubToken: "fake-token",
 			octokit: "mock-octokit",
 		});
+		expect(providerFactory.getContextLimits).toHaveBeenCalledWith(
+			"github-models",
+		);
 		expect(ghModule.getPrMetadata).toHaveBeenCalledWith(
 			"mock-octokit",
 			"test-owner",
@@ -105,6 +112,7 @@ describe("run", () => {
 				commits: [{ sha: "abc123", message: "commit" }],
 			},
 			"",
+			GITHUB_MODELS_CONTEXT_LIMITS,
 		);
 		expect(mockGenerateQAInstructions).toHaveBeenCalledWith(
 			expect.any(String),
@@ -179,11 +187,7 @@ describe("run", () => {
 		contextAny.payload = originalPayload;
 	});
 
-	it("calls setFailed when createProvider throws", async () => {
-		vi.mocked(providerFactory.createProvider).mockImplementation(() => {
-			throw new Error('Invalid provider "bad"');
-		});
-
+	it("fails for invalid provider", async () => {
 		vi.mocked(core.getInput).mockImplementation((name: string) => {
 			const inputs: Record<string, string> = {
 				"github-token": "fake-token",
@@ -200,6 +204,7 @@ describe("run", () => {
 		expect(core.setFailed).toHaveBeenCalledWith(
 			expect.stringContaining('Invalid provider "bad"'),
 		);
+		expect(providerFactory.createProvider).not.toHaveBeenCalled();
 	});
 
 	it("calls setFailed when an error occurs", async () => {