|
| 1 | +import assert from "node:assert/strict" |
| 2 | +import { execFileSync } from "node:child_process" |
| 3 | +import { readFile } from "node:fs/promises" |
| 4 | +import { resolve } from "node:path" |
| 5 | + |
| 6 | +const root = resolve(new URL("..", import.meta.url).pathname) |
| 7 | +const docsAgentRevision = "5344a4bfbda4a0553cc92636258e46a715b1c72d" |
| 8 | +const wpCodeboxRevision = "54c2f9a7bc3cd1fe20055d496c83efcfb99afb41" |
| 9 | +const docsAgentDir = process.env.DOCS_AGENT_DIR |
| 10 | +const wpCodeboxDir = process.env.WP_CODEBOX_DIR |
| 11 | + |
| 12 | +assert.ok(docsAgentDir, "DOCS_AGENT_DIR must point to the pinned Docs Agent checkout") |
| 13 | +assert.ok(wpCodeboxDir, "WP_CODEBOX_DIR must point to the pinned WP Codebox checkout") |
| 14 | + |
| 15 | +const revision = (directory) => execFileSync("git", ["-C", directory, "rev-parse", "HEAD"], { encoding: "utf8" }).trim() |
| 16 | +const readJson = async (directory, path) => JSON.parse(await readFile(resolve(directory, path), "utf8")) |
| 17 | + |
| 18 | +assert.equal(revision(docsAgentDir), docsAgentRevision, "Docs Agent checkout must match the declared producer revision") |
| 19 | +assert.equal(revision(wpCodeboxDir), wpCodeboxRevision, "WP Codebox checkout must match the Docs Agent producer revision") |
| 20 | + |
| 21 | +const docsAgentWorkflow = await readFile(resolve(docsAgentDir, ".github/workflows/maintain-docs.yml"), "utf8") |
| 22 | +const wpCodeboxContract = await readJson(wpCodeboxDir, "contracts/run-agent-task-reusable-workflow-interface.v1.json") |
| 23 | +assert.equal(wpCodeboxContract.schema, "wp-codebox/reusable-workflow-interface/v1") |
| 24 | +assert.match(docsAgentWorkflow, new RegExp(`uses: Automattic/wp-codebox/.github/workflows/run-agent-task.yml@${wpCodeboxRevision}`)) |
| 25 | +assert.match(docsAgentWorkflow, /DOCS_AGENT_REVISION: \$\{\{ github\.job_workflow_sha \}\}/) |
| 26 | +assert.match(docsAgentWorkflow, /OPENAI_API_KEY: \$\{\{ secrets\.OPENAI_API_KEY \}\}/) |
| 27 | +assert.match(docsAgentWorkflow, /ACCESS_TOKEN: \$\{\{ github\.token \}\}/) |
| 28 | +assert.match(docsAgentWorkflow, /EXTERNAL_PACKAGE_SOURCE_POLICY: \$\{\{ secrets\.EXTERNAL_PACKAGE_SOURCE_POLICY \}\}/) |
| 29 | + |
| 30 | +const docsAgentInputsBlock = docsAgentWorkflow.match(/ inputs:\n([\s\S]*?) secrets:/)?.[1] |
| 31 | +const docsAgentSecretsBlock = docsAgentWorkflow.match(/ secrets:\n([\s\S]*?) outputs:/)?.[1] |
| 32 | +assert.ok(docsAgentInputsBlock, "Docs Agent must declare reusable-workflow inputs") |
| 33 | +assert.ok(docsAgentSecretsBlock, "Docs Agent must declare reusable-workflow secrets") |
| 34 | +const docsAgentInputs = [...docsAgentInputsBlock.matchAll(/^ ([a-z_]+):/gm)].map((match) => match[1]) |
| 35 | +const docsAgentSecrets = [...docsAgentSecretsBlock.matchAll(/^ ([A-Z_]+):/gm)].map((match) => match[1]) |
| 36 | +const producerSecrets = Object.keys(wpCodeboxContract.secrets) |
| 37 | +const requiredProducerSecrets = Object.entries(wpCodeboxContract.secrets) |
| 38 | + .filter(([, definition]) => definition.required) |
| 39 | + .map(([name]) => name) |
| 40 | +const workflows = [ |
| 41 | + { |
| 42 | + path: ".github/workflows/developer-docs-agent.yml", |
| 43 | + audience: "technical", |
| 44 | + runKind: true, |
| 45 | + writablePaths: "README.md,docs/**,plugins/**/README.md", |
| 46 | + }, |
| 47 | + { |
| 48 | + path: ".github/workflows/skills-agent.yml", |
| 49 | + audience: "skills", |
| 50 | + runKind: false, |
| 51 | + writablePaths: "skills/**,plugins/**/skills/**,plugins/**/README.md", |
| 52 | + }, |
| 53 | +] |
| 54 | + |
| 55 | +for (const workflow of workflows) { |
| 56 | + const source = await readFile(resolve(root, workflow.path), "utf8") |
| 57 | + const withBlock = source.match(/ with:\n([\s\S]*?) secrets:/)?.[1] |
| 58 | + const secretsBlock = source.match(/ secrets:\n([\s\S]*)$/)?.[1] |
| 59 | + assert.ok(withBlock, `${workflow.path} must call the producer with inputs`) |
| 60 | + assert.ok(secretsBlock, `${workflow.path} must forward producer secrets`) |
| 61 | + const usedInputs = [...withBlock.matchAll(/^ ([a-z_]+):/gm)].map((match) => match[1]) |
| 62 | + const usedSecrets = [...secretsBlock.matchAll(/^ ([A-Z_]+):/gm)].map((match) => match[1]) |
| 63 | + |
| 64 | + assert.match(source, /uses: Automattic\/docs-agent\/.github\/workflows\/maintain-docs.yml@main/) |
| 65 | + assert.deepEqual(usedInputs, [...new Set(usedInputs)], `${workflow.path} must not declare an input twice`) |
| 66 | + assert.ok(usedInputs.every((input) => docsAgentInputs.includes(input)), `${workflow.path} uses an input absent from the Docs Agent schema`) |
| 67 | + assert.deepEqual(usedSecrets, ["OPENAI_API_KEY", "EXTERNAL_PACKAGE_SOURCE_POLICY"], `${workflow.path} must forward the Docs Agent secrets`) |
| 68 | + assert.ok(usedSecrets.every((secret) => docsAgentSecrets.includes(secret)), `${workflow.path} forwards a secret absent from the Docs Agent schema`) |
| 69 | + assert.ok(usedSecrets.every((secret) => producerSecrets.includes(secret) || secret === "EXTERNAL_PACKAGE_SOURCE_POLICY"), `${workflow.path} forwards a secret absent from the producer schema`) |
| 70 | + assert.ok(requiredProducerSecrets.includes("EXTERNAL_PACKAGE_SOURCE_POLICY"), "WP Codebox must require the external package policy") |
| 71 | + assert.match(source, new RegExp(`audience: ${workflow.audience}`)) |
| 72 | + assert.match(source, new RegExp(`writable_paths: ${workflow.writablePaths.replaceAll("*", "\\*")}`)) |
| 73 | + assert.match(source, /base_ref: trunk/) |
| 74 | + assert.match(source, /docs_branch: docs-agent\/build-with-wordpress-/) |
| 75 | + assert.match(source, /pnpm install --frozen-lockfile/) |
| 76 | + assert.match(source, /pnpm build/) |
| 77 | + assert.match(source, /pnpm verify/) |
| 78 | + assert.match(source, /git diff --exit-code/) |
| 79 | + assert.match(source, /permissions:\n contents: write\n pull-requests: write\n issues: write/) |
| 80 | + assert.match(source, /OPENAI_API_KEY: \$\{\{ secrets\.OPENAI_API_KEY \}\}/) |
| 81 | + assert.match(source, /EXTERNAL_PACKAGE_SOURCE_POLICY: \$\{\{ secrets\.EXTERNAL_PACKAGE_SOURCE_POLICY \}\}/) |
| 82 | + assert.doesNotMatch(source, /ACCESS_TOKEN/) |
| 83 | + assert.doesNotMatch(source, /docs_agent_ref/) |
| 84 | + assert.doesNotMatch(source, /runtime-agent-full-run|runtime_(?:provider|profile|profiles|execution|dependencies|task|config)|datamachine|homeboy/i) |
| 85 | + |
| 86 | + if (workflow.runKind) { |
| 87 | + assert.match(source, /run_kind: \$\{\{ github\.event_name == 'workflow_dispatch' && 'bootstrap' \|\| 'maintenance' \}\}/) |
| 88 | + } else { |
| 89 | + assert.doesNotMatch(source, /^ run_kind:/m) |
| 90 | + } |
| 91 | +} |
| 92 | + |
| 93 | +console.log("Docs Agent and WP Codebox producer contract validation passed") |
0 commit comments